From eab245f61d27014ba38036c0f392997c0d3919f8 Mon Sep 17 00:00:00 2001 From: Miles Ward Date: Mon, 5 Oct 2026 14:24:29 -0400 Subject: [PATCH] OpenShift: pull runtime image from the internal ATT GitLab registry - runtime/03-deployment.yaml: image is now att-gitlab.apps.evd.geaerospace.net:5050/att/infrastructure/vshell:latest with imagePullSecrets; the internal ImageStream trigger is gone - runtime/00-serviceaccount.yaml: carries att-gitlab-pull - runtime/06-att-gitlab-pull-secret.yaml: registry credentials (placeholders) - cluster/00-insecure-registry.yaml: Image/cluster insecureRegistries entry, the cluster-side equivalent of 'podman pull -k' - 02/03 build manifests marked optional, with the GitLab push-output variant documented inline - openshift/README.md split into run-from-registry and build-in-cluster paths - all 14 manifests decode cleanly with 'oc create --dry-run=client' --- README.md | 8 +- openshift/02-vshell-imagestream.yaml | 5 +- openshift/03-buildconfig.yaml | 24 +++- openshift/README.md | 120 ++++++++++-------- openshift/cluster/00-insecure-registry.yaml | 28 ++++ openshift/runtime/00-serviceaccount.yaml | 5 +- openshift/runtime/03-deployment.yaml | 21 +-- .../runtime/06-att-gitlab-pull-secret.yaml | 24 ++++ 8 files changed, 164 insertions(+), 71 deletions(-) create mode 100644 openshift/cluster/00-insecure-registry.yaml create mode 100644 openshift/runtime/06-att-gitlab-pull-secret.yaml diff --git a/README.md b/README.md index b2e4dc8..24db452 100644 --- a/README.md +++ b/README.md @@ -98,9 +98,11 @@ HTTP→HTTPS redirect, after checking `-version`, `-test_config` and `-license`. ## OpenShift -`openshift/` holds drop-in manifests that build this image in-cluster -(`BuildConfig` + `ImageStream` + Gitea source secret) and, optionally, run it -(custom SCC, PVC, Deployment, Service, passthrough Route). See +`openshift/` holds drop-in manifests. The runtime Deployment pulls +`att-gitlab.apps.evd.geaerospace.net:5050/att/infrastructure/vshell:latest` from +the internal GitLab registry (pull secret, custom SCC, PVC, Service, passthrough +Route, plus a cluster-scoped insecure-registry entry for the `-k` equivalent); +the `BuildConfig`/`ImageStream` chain for building in-cluster is optional. See [openshift/README.md](openshift/README.md) for the apply order. ## Licensing diff --git a/openshift/02-vshell-imagestream.yaml b/openshift/02-vshell-imagestream.yaml index 9e484dc..a824578 100644 --- a/openshift/02-vshell-imagestream.yaml +++ b/openshift/02-vshell-imagestream.yaml @@ -1,5 +1,6 @@ -# Output image stream for the built image. The build pushes vshell:latest here -# and the runtime Deployment pulls from it. +# Output image stream for in-cluster builds (03-buildconfig.yaml). +# Only needed if OpenShift builds the image; the runtime Deployment pulls +# att-gitlab.apps.evd.geaerospace.net:5050/att/infrastructure/vshell:latest instead. apiVersion: image.openshift.io/v1 kind: ImageStream metadata: diff --git a/openshift/03-buildconfig.yaml b/openshift/03-buildconfig.yaml index ea823a3..debfd61 100644 --- a/openshift/03-buildconfig.yaml +++ b/openshift/03-buildconfig.yaml @@ -1,12 +1,18 @@ # In-cluster build of the VShell image. # -# The repository already vendors the vendor .deb, so the only things the build -# pod fetches from the network are: +# OPTIONAL when you already have the image in the internal GitLab registry: +# runtime/03-deployment.yaml pulls +# att-gitlab.apps.evd.geaerospace.net:5050/att/infrastructure/vshell:latest +# so this BuildConfig and 02-vshell-imagestream.yaml are only needed if you want +# OpenShift to build the image too. +# +# The repository vendors the vendor .deb, so the only things the build pod +# fetches from the network are: # - gitea.milesward.dev (this git repo, authenticated by gitea-vshell-source) # - registry-1.docker.io (base image ubuntu:24.04) # - archive.ubuntu.com (apt-get install openssl in the Dockerfile) -# On an air-gapped cluster, mirror ubuntu:24.04 and openssl into an internal -# registry first; see 04-base-imagestream.yaml and openshift/README.md. +# On an air-gapped cluster, mirror ubuntu:24.04 and openssl first; see +# 04-base-imagestream.yaml. apiVersion: build.openshift.io/v1 kind: BuildConfig metadata: @@ -32,10 +38,20 @@ spec: name: docker.io/library/ubuntu:24.04 # Pin the base by digest instead of tag when you need reproducible builds: # name: docker.io/library/ubuntu@sha256:a60ef8ec184edb5ed91f5983f00261359fb2fe20f603e8d488b9477d32fcad70 + # Default: push the build into the in-cluster image stream (02-vshell-imagestream.yaml). output: to: kind: ImageStreamTag name: vshell:latest + # Alternative: push the build straight to the internal GitLab registry, so the + # Deployment picks it up with imagePullPolicy: Always. Needs push credentials + # on the builder service account, and the nodes must trust the registry + # (cluster/00-insecure-registry.yaml): + # oc create secret docker-registry att-gitlab-push -n vshell \ + # --docker-server=att-gitlab.apps.evd.geaerospace.net:5050 \ + # --docker-username= --docker-password= + # oc secrets link builder att-gitlab-push -n vshell + # oc patch bc/vshell -n vshell --type=merge -p '{"spec":{"output":{"to":{"kind":"DockerImage","name":"att-gitlab.apps.evd.geaerospace.net:5050/att/infrastructure/vshell:latest"}}}}' resources: requests: cpu: 500m diff --git a/openshift/README.md b/openshift/README.md index 9dcf36d..796faf5 100644 --- a/openshift/README.md +++ b/openshift/README.md @@ -1,76 +1,90 @@ # OpenShift manifests for VShell -Everything needed to build `vshell:latest` inside an OpenShift project and, if you -want, to run it there. Write-tested against the OpenShift 4.22 client scheme and -matching the 4.21 cluster described in `/workspace/openshift-win11/FINDINGS.md`. +Two independent halves: -## Build (required) +- **run** (the normal path): pull `att-gitlab.apps.evd.geaerospace.net:5050/att/infrastructure/vshell:latest` + from the internal ATT GitLab registry and run it in the `vshell` project. +- **build** (optional): build the image inside OpenShift from the private Gitea + repo, for when you want the cluster to produce the image itself. + +All manifests have been decode-tested with `oc create --dry-run=client` against +the OpenShift 4.22 client scheme. + +## Run from the internal registry | File | What it does | |---|---| | [`00-namespace.yaml`](00-namespace.yaml) | Project `vshell`. | +| [`cluster/00-insecure-registry.yaml`](cluster/00-insecure-registry.yaml) | Cluster-scoped: mark `att-gitlab.apps.evd.geaerospace.net:5050` insecure (the `-k` equivalent). | +| [`runtime/00-serviceaccount.yaml`](runtime/00-serviceaccount.yaml) | `ServiceAccount/vshell`, carries the pull secret. | +| [`runtime/01-scc.yaml`](runtime/01-scc.yaml) | Least-privilege SCC allowing UID 0 + Role/RoleBinding so only this SA may use it. | +| [`runtime/02-pvc.yaml`](runtime/02-pvc.yaml) | `PersistentVolumeClaim/vshell-data` for config + TLS certificate. | +| [`runtime/03-deployment.yaml`](runtime/03-deployment.yaml) | Deployment pulling from the GitLab registry. | +| [`runtime/04-service.yaml`](runtime/04-service.yaml) | ClusterIP service for ports 22/21/990/80/443. | +| [`runtime/05-route.yaml`](runtime/05-route.yaml) | HTTPS Route, **passthrough** termination. | +| [`runtime/06-att-gitlab-pull-secret.yaml`](runtime/06-att-gitlab-pull-secret.yaml) | Registry credentials (placeholders - create from the CLI). | + +```sh +# 1. registry credentials (deploy token / PAT with read_registry) +oc create secret docker-registry att-gitlab-pull -n vshell \ + --docker-server=att-gitlab.apps.evd.geaerospace.net:5050 \ + --docker-username= --docker-password= + +# 2. let the nodes pull from it over plain/insecure TLS (cluster-admin; rolls all nodes) +oc apply -f cluster/00-insecure-registry.yaml + +# 3. project + workload +oc apply -f 00-namespace.yaml +oc apply -f runtime/ +oc get pods,route -n vshell +``` + +`cluster/00-insecure-registry.yaml` is the pod-world replacement for +`podman pull -k`: a pod spec cannot opt out of TLS verification, so the node CRI +must be told the registry is exempt. Editing `Image/cluster` makes the Machine +Config Operator drain and reboot every node — do it in a maintenance window, and +remember the rollback rolls them again. + +If the cluster sets `registrySources.allowedRegistries`, add +`att-gitlab.apps.evd.geaerospace.net:5050` **and** the internal registry +`image-registry.openshift-image-registry.svc:5000` to that list, or other pulls +start failing. If the registry has a private CA, prefer +`spec.additionalTrustedCA` over `insecureRegistries`. + +`restricted-v2` forbids UID 0 and VShell needs root (PAM reads `/etc/shadow`, +writes `/var/run/*.pid`, creates root-owned host keys), hence the SCC — or +`oc adm policy add-scc-to-user anyuid -z vshell -n vshell` for the stock one. +The Route is passthrough because `vshell-httpsd` terminates TLS itself with the +PKCS#12 certificate the entrypoint generates; SSH (22), explicit FTPS (21) and +implicit FTPS (990) cannot use a Route, so expose the Service with a +NodePort/LoadBalancer. Never scale the Deployment past one replica: fixed ports +and a single host key. + +## Build in OpenShift (optional) + +| File | What it does | +|---|---| | [`01-gitea-source-secret.yaml`](01-gitea-source-secret.yaml) | `kubernetes.io/basic-auth` secret for the private Gitea repo. | | [`02-vshell-imagestream.yaml`](02-vshell-imagestream.yaml) | Output `ImageStream/vshell` (`latest`). | -| [`03-buildconfig.yaml`](03-buildconfig.yaml) | `BuildConfig/vshell` — Docker strategy, git source, `ubuntu:24.04` base. | -| [`04-base-imagestream.yaml`](04-base-imagestream.yaml) | Optional: keep the Ubuntu base inside the cluster instead of pulling Docker Hub on every build. | +| [`03-buildconfig.yaml`](03-buildconfig.yaml) | `BuildConfig/vshell` — Docker strategy, git source, `ubuntu:24.04` base. Can also be pointed at the GitLab registry (see the comments in the file). | +| [`04-base-imagestream.yaml`](04-base-imagestream.yaml) | Optional: keep the Ubuntu base inside the cluster instead of pulling Docker Hub each build. | | [`05-dockerhub-pull-secret.yaml`](05-dockerhub-pull-secret.yaml) | Optional: Docker Hub credentials so base pulls are not rate limited. | ```sh oc apply -f 00-namespace.yaml - -# password = the Gitea token from ~/.dsh/AGENTS.md (do not commit it into 01-*.yaml) oc create secret generic gitea-vshell-source -n vshell \ --type=kubernetes.io/basic-auth \ --from-literal=username=mward4 \ --from-literal=password= - -oc apply -f 02-vshell-imagestream.yaml -oc apply -f 03-buildconfig.yaml - +oc apply -f 02-vshell-imagestream.yaml -f 03-buildconfig.yaml oc start-build vshell -n vshell --follow -oc get is vshell -n vshell ``` -The repository vendors the vendor `.deb`, so the build pod only needs egress to -`gitea.milesward.dev`, `registry-1.docker.io` and `archive.ubuntu.com` -(`apt-get install openssl`). Air-gapped clusters: import `ubuntu:24.04` with -`04-base-imagestream.yaml`, point `strategy.dockerStrategy.from` at -`ImageStreamTag ubuntu:24.04`, and add an internal apt mirror. - -`replicas: 1` is not a build concern but a runtime one: the daemons own fixed -ports and a single host key, so never scale the Deployment. - -## Runtime (optional) - -| File | What it does | -|---|---| -| [`runtime/00-serviceaccount.yaml`](runtime/00-serviceaccount.yaml) | `ServiceAccount/vshell`. | -| [`runtime/01-scc.yaml`](runtime/01-scc.yaml) | Least-privilege SCC that allows UID 0 + Role/RoleBinding so only this SA may use it. | -| [`runtime/02-pvc.yaml`](runtime/02-pvc.yaml) | `PersistentVolumeClaim/vshell-data` for config + TLS certificate. | -| [`runtime/03-deployment.yaml`](runtime/03-deployment.yaml) | Deployment, auto-updated by the `image.openshift.io/triggers` annotation. | -| [`runtime/04-service.yaml`](runtime/04-service.yaml) | ClusterIP service for ports 22/21/990/80/443. | -| [`runtime/05-route.yaml`](runtime/05-route.yaml) | HTTPS Route, **passthrough** termination. | - -```sh -oc apply -f runtime/ -oc get route vshell-https -n vshell -``` - -`restricted-v2` forbids UID 0, so the SCC (or `oc adm policy add-scc-to-user anyuid -z vshell -n vshell`) -is required — VShell reads `/etc/shadow` through PAM, writes `/var/run/*.pid` -and creates root-owned host keys. The custom SCC grants no extra capabilities -and no host access. - -The Route is passthrough because `vshell-httpsd` terminates TLS itself with the -PKCS#12 certificate the entrypoint generated; clients therefore see a -self-signed certificate until you mount a real one at `$VSHELL_TLS_P12`. SSH -(22), explicit FTPS (21) and implicit FTPS (990) cannot go through a Route — -expose the Service with a NodePort/LoadBalancer or an external load balancer. - -The Deployment points `VSHELL_CONFIG` and `VSHELL_TLS_P12` at the PVC-mounted -`/data`, which keeps the vendor defaults and the `HttpsFiles/` web assets -visible. The SSH host key stays at `/usr/local/etc/vshelld_hostkey` unless you -set `HostKeyFilenames /data/vshelld_hostkey` in the generated config. +The repository vendors the vendor `.deb`, so a build pod needs egress only to +`gitea.milesward.dev`, `registry-1.docker.io` and `archive.ubuntu.com`. For +air-gapped clusters import `ubuntu:24.04` with `04-base-imagestream.yaml`, point +`strategy.dockerStrategy.from` at `ImageStreamTag ubuntu:24.04`, and add an +internal apt mirror. ## Caveats diff --git a/openshift/cluster/00-insecure-registry.yaml b/openshift/cluster/00-insecure-registry.yaml new file mode 100644 index 0000000..2f985f7 --- /dev/null +++ b/openshift/cluster/00-insecure-registry.yaml @@ -0,0 +1,28 @@ +# Insecure-registry configuration for the internal ATT GitLab registry. +# +# This is the cluster equivalent of `podman pull -k` / `skopeo --src-tls-verify=false`. +# A pod spec cannot skip TLS verification, so the node container runtime has to be +# told the registry is exempt. Cluster-scoped: requires cluster-admin. +# +# WARNING: editing spec.registrySources makes the Machine Config Operator roll +# every node (drain + reboot) to apply the new CRI config. Do it in a maintenance +# window. Reverting this file rolls the nodes again. +# +# If your cluster also sets spec.registrySources.allowedRegistries (a whitelist), +# that list must then contain att-gitlab.apps.evd.geaerospace.net:5050 AND the +# OpenShift internal registry (image-registry.openshift-image-registry.svc:5000) +# AND any other registry the cluster pulls from, or pulls start failing. +# +# Preferred over insecure TLS when the registry has a private CA: keep the CA in +# a ConfigMap in the openshift-config namespace and reference it instead: +# spec: +# additionalTrustedCA: +# name: att-gitlab-ca +apiVersion: config.openshift.io/v1 +kind: Image +metadata: + name: cluster +spec: + registrySources: + insecureRegistries: + - att-gitlab.apps.evd.geaerospace.net:5050 diff --git a/openshift/runtime/00-serviceaccount.yaml b/openshift/runtime/00-serviceaccount.yaml index 3095c6a..e674d50 100644 --- a/openshift/runtime/00-serviceaccount.yaml +++ b/openshift/runtime/00-serviceaccount.yaml @@ -1,6 +1,7 @@ # Service account the VShell pod runs as. It is bound to the custom SCC in # 01-scc.yaml, because VShell must run as root (PAM reads /etc/shadow, the -# daemons write /var/run/*.pid and generate host keys owned by root). +# daemons write /var/run/*.pid and generate host keys owned by root), and it +# carries the pull secret for the internal ATT GitLab registry. apiVersion: v1 kind: ServiceAccount metadata: @@ -8,3 +9,5 @@ metadata: namespace: vshell labels: app.kubernetes.io/name: vshell +imagePullSecrets: + - name: att-gitlab-pull diff --git a/openshift/runtime/03-deployment.yaml b/openshift/runtime/03-deployment.yaml index 6d74d2b..1561996 100644 --- a/openshift/runtime/03-deployment.yaml +++ b/openshift/runtime/03-deployment.yaml @@ -1,8 +1,14 @@ -# VShell runtime. Single replica: the daemons own fixed ports and one host key, -# so they are not horizontally scalable. +# VShell runtime, pulling the image from the internal ATT GitLab registry. # -# The image tag is kept in sync by the image.openshift.io/triggers annotation, -# which fires whenever a new vshell:latest build finishes. +# Single replica: the daemons own fixed ports and one host key, so they are not +# horizontally scalable. +# +# This registry is outside OpenShift, so three things make the pull work: +# 1. 06-att-gitlab-pull-secret.yaml (credentials, linked via the ServiceAccount) +# 2. cluster/00-insecure-registry.yaml on the nodes - the registry equivalent +# of `podman pull -k`, since a pod spec cannot opt out of TLS verification +# 3. DNS/route reachability of att-gitlab.apps.evd.geaerospace.net:5050 from +# every node apiVersion: apps/v1 kind: Deployment metadata: @@ -10,9 +16,6 @@ metadata: namespace: vshell labels: app.kubernetes.io/name: vshell - annotations: - image.openshift.io/triggers: | - [{"from":{"kind":"ImageStreamTag","name":"vshell:latest"},"fieldPath":"spec.template.spec.containers[0].image"}] spec: replicas: 1 strategy: @@ -26,13 +29,15 @@ spec: app.kubernetes.io/name: vshell spec: serviceAccountName: vshell + imagePullSecrets: + - name: att-gitlab-pull securityContext: runAsUser: 0 runAsGroup: 0 fsGroup: 0 containers: - name: vshell - image: vshell:latest + image: att-gitlab.apps.evd.geaerospace.net:5050/att/infrastructure/vshell:latest imagePullPolicy: Always env: - name: VSHELL_CONFIG diff --git a/openshift/runtime/06-att-gitlab-pull-secret.yaml b/openshift/runtime/06-att-gitlab-pull-secret.yaml new file mode 100644 index 0000000..9cd1bd8 --- /dev/null +++ b/openshift/runtime/06-att-gitlab-pull-secret.yaml @@ -0,0 +1,24 @@ +# Credentials for the internal ATT GitLab container registry. +# +# Create it with your own GitLab credentials (deploy token, PAT with read_registry, +# or a robot account) - do not commit them: +# +# oc create secret docker-registry att-gitlab-pull -n vshell \ +# --docker-server=att-gitlab.apps.evd.geaerospace.net:5050 \ +# --docker-username= \ +# --docker-password= +# +# Already referenced by runtime/00-serviceaccount.yaml and runtime/03-deployment.yaml, +# so nothing else needs to be linked. +apiVersion: v1 +kind: Secret +metadata: + name: att-gitlab-pull + namespace: vshell + labels: + app.kubernetes.io/name: vshell + app.kubernetes.io/component: registry-auth +type: kubernetes.io/dockerconfigjson +stringData: + .dockerconfigjson: | + {"auths":{"att-gitlab.apps.evd.geaerospace.net:5050":{"username":"REPLACE","password":"REPLACE","auth":"REPLACE_BASE64_USER_COLON_TOKEN"}}}