Skip to content

feat: observe installed Zitadel instances through native references #32

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
patrickleet merged 2 commits into from
Oct 6, 2026
Merged

feat: observe installed Zitadel instances through native references #32

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
45c5c66
feat: observe installed instances through native references
patrickleet Oct 2, 2026
d83edde
test: align CI validation with all local examples
patrickleet Oct 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 20 additions & 1 deletion .github/workflows/on-pr.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,28 +34,47 @@ jobs:
[
{ "example": "examples/authstacks/minimal.yaml", "api_path": "apis/authstacks" },
{ "example": "examples/authstacks/standard.yaml", "api_path": "apis/authstacks" },
{ "example": "examples/authstacks/with-smtp.yaml", "api_path": "apis/authstacks" },
{ "example": "examples/authstacks/with-instance.yaml", "api_path": "apis/authstacks" },
{ "example": "examples/authstacks/local-colima.yaml", "api_path": "apis/authstacks" },
{ "example": "examples/machineusers/minimal.yaml", "api_path": "apis/machineusers" },
{ "example": "examples/machineusers/with-pat.yaml", "api_path": "apis/machineusers" },
{ "example": "examples/machineusers/with-pat-push.yaml","api_path": "apis/machineusers" },
{ "example": "examples/humanusers/explicit-org.yaml", "api_path": "apis/humanusers" },
{ "example": "examples/humanusers/with-org-ref.yaml", "api_path": "apis/humanusers" },
{ "example": "examples/oidcclients/storybook-preview.yaml", "api_path": "apis/oidcclients" },
{ "example": "examples/oidcclients/storybook-preview.yaml", "api_path": "apis/oidcclients", "observed_resources": "tests/test-oidcclient/observed/ready.yaml" },
{ "example": "examples/grants/referenced-same-org.yaml", "api_path": "apis/grants" },
{ "example": "examples/grants/same-org.yaml", "api_path": "apis/grants" },
{ "example": "examples/grants/cross-org.yaml", "api_path": "apis/grants" }
{ "example": "examples/grants/cross-org.yaml", "api_path": "apis/grants" },
{ "example": "examples/grants/cross-org.yaml", "api_path": "apis/grants", "observed_resources": "tests/test-grant/observed/cross-org-iter2.yaml" }
]
error_on_missing_schemas: true

test:
uses: hops-ops/workflows-crossplane/.github/workflows/test.yaml@v3.2.0

instance-observation:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- uses: upbound/action-up@v1.2.0
with:
skip-login: true
- uses: upbound/action-up-project@v1.2.0
with:
skip-login-check: true
- run: python3 -m pip install PyYAML==6.0.3
- run: make test-security

e2e:
uses: hops-ops/workflows-crossplane/.github/workflows/e2e.yaml@v3.2.0

publish:
needs:
- validate
- test
- instance-observation
uses: hops-ops/workflows-crossplane/.github/workflows/publish.yaml@v3.2.0
secrets:
GH_PAT: ${{ secrets.GH_PAT }}
Expand Down
21 changes: 20 additions & 1 deletion .github/workflows/on-push-main.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,21 +29,39 @@ jobs:
[
{ "example": "examples/authstacks/minimal.yaml", "api_path": "apis/authstacks" },
{ "example": "examples/authstacks/standard.yaml", "api_path": "apis/authstacks" },
{ "example": "examples/authstacks/with-smtp.yaml", "api_path": "apis/authstacks" },
{ "example": "examples/authstacks/with-instance.yaml", "api_path": "apis/authstacks" },
{ "example": "examples/authstacks/local-colima.yaml", "api_path": "apis/authstacks" },
{ "example": "examples/machineusers/minimal.yaml", "api_path": "apis/machineusers" },
{ "example": "examples/machineusers/with-pat.yaml", "api_path": "apis/machineusers" },
{ "example": "examples/machineusers/with-pat-push.yaml","api_path": "apis/machineusers" },
{ "example": "examples/humanusers/explicit-org.yaml", "api_path": "apis/humanusers" },
{ "example": "examples/humanusers/with-org-ref.yaml", "api_path": "apis/humanusers" },
{ "example": "examples/oidcclients/storybook-preview.yaml", "api_path": "apis/oidcclients" },
{ "example": "examples/oidcclients/storybook-preview.yaml", "api_path": "apis/oidcclients", "observed_resources": "tests/test-oidcclient/observed/ready.yaml" },
{ "example": "examples/grants/referenced-same-org.yaml", "api_path": "apis/grants" },
{ "example": "examples/grants/same-org.yaml", "api_path": "apis/grants" },
{ "example": "examples/grants/cross-org.yaml", "api_path": "apis/grants" }
{ "example": "examples/grants/cross-org.yaml", "api_path": "apis/grants" },
{ "example": "examples/grants/cross-org.yaml", "api_path": "apis/grants", "observed_resources": "tests/test-grant/observed/cross-org-iter2.yaml" }
]
error_on_missing_schemas: true

test:
uses: hops-ops/workflows-crossplane/.github/workflows/test.yaml@v3.2.0

instance-observation:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- uses: upbound/action-up@v1.2.0
with:
skip-login: true
- uses: upbound/action-up-project@v1.2.0
with:
skip-login-check: true
- run: python3 -m pip install PyYAML==6.0.3
- run: make test-security

e2e:
uses: hops-ops/workflows-crossplane/.github/workflows/e2e.yaml@v3.2.0

Expand All @@ -52,6 +70,7 @@ jobs:
needs:
- validate
- test
- instance-observation
- e2e
uses: unbounded-tech/workflow-vnext-tag/.github/workflows/workflow.yaml@v1.22.3
secrets:
Expand Down
97 changes: 30 additions & 67 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ EXAMPLES := \
examples/authstacks/standard.yaml:: \
examples/authstacks/local-colima.yaml:: \
examples/authstacks/with-smtp.yaml:: \
examples/authstacks/with-instance.yaml:: \
examples/machineusers/minimal.yaml:: \
examples/machineusers/with-pat.yaml:: \
examples/machineusers/with-pat-push.yaml:: \
Expand All @@ -40,75 +41,37 @@ EXAMPLES := \
examples/grants/cross-org.yaml:: \
examples/grants/cross-org.yaml::tests/test-grant/observed/cross-org-iter2.yaml

# Render all examples (parallel execution, output shown per-job when complete)
# Render serially: up updates shared project metadata/schema caches during build.
render\:all:
@tmpdir=$$(mktemp -d); \
pids=""; \
@set -e; \
for entry in $(EXAMPLES); do \
example=$${entry%%::*}; \
observed=$${entry#*::}; \
example=$${entry%%::*}; observed=$${entry#*::}; \
api_dir=$$(echo "$$example" | awk -F/ '{print "apis/" $$2}'); \
composition="$$api_dir/composition.yaml"; \
definition="$$api_dir/definition.yaml"; \
outfile="$$tmpdir/$$(echo $$entry | tr '/:' '__')"; \
( \
if [ -n "$$observed" ]; then \
echo "=== Rendering $$example with observed-resources $$observed ==="; \
up composition render --xrd=$$definition $$composition $$example --observed-resources=$$observed; \
else \
echo "=== Rendering $$example (api=$$api_dir) ==="; \
up composition render --xrd=$$definition $$composition $$example; \
fi; \
echo "" \
) > "$$outfile" 2>&1 & \
pids="$$pids $$!:$$outfile"; \
done; \
failed=0; \
for pair in $$pids; do \
pid=$${pair%%:*}; \
outfile=$${pair#*:}; \
if ! wait $$pid; then failed=1; fi; \
cat "$$outfile"; \
done; \
rm -rf "$$tmpdir"; \
exit $$failed
echo "Rendering $$example"; \
if [ -n "$$observed" ]; then \
up composition render --xrd=$$api_dir/definition.yaml $$api_dir/composition.yaml $$example --observed-resources=$$observed; \
else \
up composition render --xrd=$$api_dir/definition.yaml $$api_dir/composition.yaml $$example; \
fi; \
done

# Validate all examples (parallel execution, output shown per-job when complete)
validate\:all:
@tmpdir=$$(mktemp -d); \
pids=""; \
# Include dependency metadata and all XRDs when loading validation schemas.
.PHONY: generate-configuration
generate-configuration:
hops validate generate-configuration --path . --api-path $(XRD_DIR) --no-gitignore-update

validate\:all: generate-configuration
@set -e -o pipefail; \
for entry in $(EXAMPLES); do \
example=$${entry%%::*}; \
observed=$${entry#*::}; \
example=$${entry%%::*}; observed=$${entry#*::}; \
api_dir=$$(echo "$$example" | awk -F/ '{print "apis/" $$2}'); \
composition="$$api_dir/composition.yaml"; \
definition="$$api_dir/definition.yaml"; \
outfile="$$tmpdir/$$(echo $$entry | tr '/:' '__')"; \
( \
if [ -n "$$observed" ]; then \
echo "=== Validating $$example with observed-resources $$observed ==="; \
up composition render --xrd=$$definition $$composition $$example \
--observed-resources=$$observed --include-full-xr --quiet | \
crossplane resource validate $$api_dir --error-on-missing-schemas -; \
else \
echo "=== Validating $$example (api=$$api_dir) ==="; \
up composition render --xrd=$$definition $$composition $$example \
--include-full-xr --quiet | \
crossplane resource validate $$api_dir --error-on-missing-schemas -; \
fi; \
echo "" \
) > "$$outfile" 2>&1 & \
pids="$$pids $$!:$$outfile"; \
done; \
failed=0; \
for pair in $$pids; do \
pid=$${pair%%:*}; \
outfile=$${pair#*:}; \
if ! wait $$pid; then failed=1; fi; \
cat "$$outfile"; \
done; \
rm -rf "$$tmpdir"; \
exit $$failed
echo "Validating $$example"; \
if [ -n "$$observed" ]; then \
up composition render --xrd=$$api_dir/definition.yaml $$api_dir/composition.yaml $$example --observed-resources=$$observed --include-full-xr --quiet; \
else \
up composition render --xrd=$$api_dir/definition.yaml $$api_dir/composition.yaml $$example --include-full-xr --quiet; \
fi | crossplane resource validate apis --error-on-missing-schemas -; \
done

# Shorthand aliases
.PHONY: clean build test e2e publish render validate
Expand All @@ -120,16 +83,16 @@ render\:%:
@example="examples/authstacks/$*.yaml"; \
up composition render --xrd=$(DEFINITION) $(COMPOSITION) $$example

validate\:%:
@example="examples/authstacks/$*.yaml"; \
validate\:%: generate-configuration
@set -e -o pipefail; example="examples/authstacks/$*.yaml"; \
up composition render --xrd=$(DEFINITION) $(COMPOSITION) $$example \
--include-full-xr --quiet | \
crossplane resource validate $(XRD_DIR) --error-on-missing-schemas -
crossplane resource validate apis --error-on-missing-schemas -

test:
up test run $(RENDER_TESTS)

# Includes rejection cases that CompositionTest cannot express.
# Native observation readiness and migration regressions; requires PyYAML.
.PHONY: test-security
test-security:
python3 -m unittest discover -s tests/security -v
Expand Down
106 changes: 63 additions & 43 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,46 +179,66 @@ private keys outside AuthStack and deliver them to provider consumers via ESO.
No user or hostname is enabled by default. Omitting memberships grants Zitadel's
SYSTEM_OWNER default; specify memberships explicitly when narrower access fits.

Set `instanceDiscovery.enabled: true` with an explicit `internalURL` to publish
`status.instanceId`. A retry-bounded Job reads the chart-generated IAM PAT, queries
instance metadata, and patches only its named ConfigMap. It never mutates
Zitadel. `firstInstance.enabled` must remain enabled for this discovery mode.
The ConfigMap is observed without Update management so reconcile cannot erase
the probe's result. Recreating an instance requires recreating the discovery
Job/ConfigMap alongside it. The default discovery image is Python 3.13.7 Alpine;
set `instanceDiscovery.image` to an approved mirror/digest when required.
Discovery configuration (including its image and chart overrides) is an
administrator-level interface and must not be delegated to untrusted tenants.

`internalURL` must match the chart's actual Zitadel Service and configured port,
using `<service>.<namespace>.svc` or `<service>.<namespace>.svc.cluster.local`,
without credentials, path, query or fragment. For example:
`https://identity-zitadel.identity.svc:8080`. The origin is validated before
rendering the credential-consuming Job. Redirects and environment proxies are
disabled. HTTPS verifies the certificate chain and Service hostname; configure
server TLS through chart values and optionally set
`caCertSecretRef: {name: zitadel-ca, key: ca.crt}` for a private CA.

Only a trusted local cluster with plaintext Service traffic should set
`allowInsecureHTTP: true`. This explicit exception sends the PAT in cleartext
inside the cluster; browser/gateway HTTPS does not encrypt that hop. It is false
by default and is not a cloud default. Enabling HTTPS requires the Service itself
to support TLS, not merely TLS termination at the ingress.

The pod can wait for the chart-generated PAT without a Job wall-clock deadline.
Once started it makes at most 110 attempts (two 10-second request timeouts and a
5-second sleep per attempt), with Kubernetes backoffLimit 3. A terminal API
failure still requires retrying the Job after fixing the cause; waiting for a
missing first-install Secret no longer consumes that retry budget. PAT rotation
alone does not recreate the Job. Endpoint, image, domain, admin username, or CA
reference changes do.

These additions let consumers use provider CustomDomain/TrustedDomain resources
without carrying instance IDs in git. Project domains and browser ingress remain
consumer/platform responsibilities. `spec.domain` is always supplied by the
installation: cloud examples use real domains; only local CLI templates use
localhost. Existing installations have both features disabled unless requested.

Discovery security regressions: run `make test-security` with the same Docker/
`up` setup as render tests. It checks rejected origins and the actual Job
transport code (TLS identity verification, no redirects, no environment proxies).
Set `instanceDiscovery.enabled: true` and select an existing Zitadel
ProviderConfig. AuthStack composes a namespaced, read-only `Instance` using
provider-upjet-zitadel **v0.3.0 or newer**:

```yaml
spec:
instanceDiscovery:
enabled: true
providerConfigRef:
name: zitadel-admin
kind: ClusterProviderConfig
```

The ProviderConfig owns authentication, endpoint, TLS and instance headers. It
must target the instance installed by this AuthStack; AuthStack cannot verify
that an arbitrary consumer-owned endpoint belongs to its Helm release. Provision
credentials with ESO, and publish chart-generated PATs through PushSecret before
waiting for native observation. For first install, wait for the composed Helm
Release, establish the ProviderConfig, then wait for AuthStack readiness. Do not
wait for AuthStack readiness before creating its observer's credentials.

Successful, current-generation observation publishes `status.instanceId` and
`status.instanceRef: {name, namespace}`. Domain resources can use the named
observer directly, without copying the ID or importing an external name:

```yaml
apiVersion: instance.zitadel.m.crossplane.io/v1alpha1
kind: TrustedDomain
metadata:
name: browser
namespace: preview
spec:
forProvider:
instanceIdRef:
name: identity-instance # AuthStack metadata.name + "-instance"
namespace: platform # AuthStack namespace
policy: {resolve: Always}
domain: preview.example.com
providerConfigRef:
name: zitadel-admin
kind: ClusterProviderConfig
```

`instanceIdSelector` is also supported by the provider; use a unique label match
when consumers should discover the observer rather than name it explicitly.
The observer retries API failures and reads rotated credentials on reconciliation.
It binds the first observed ID and refuses a different ID until deliberately
recreated. Deleting it does not delete Zitadel; Helm still owns that lifecycle.
A Usage keeps the Helm release available until the observer has been deleted.

### Migrating existing instance discovery

This changes only the **opt-in discovery API** introduced previously. Replace
`internalURL`, `allowInsecureHTTP`, `caCertSecretRef` and `image` with
`providerConfigRef`. Configure transport in that ProviderConfig instead; an
old discovery configuration without the new reference is rejected. The Job,
ConfigMap, ServiceAccount and RBAC are removed by composition reconciliation.
`enabled` and `status.instanceId` remain; `status.instanceRef` is additive.
AuthStack consumers with discovery disabled retain their installation behavior.
No localhost, plaintext transport or secret backend is defaulted by this API.
The package dependency now requires provider v0.3.0+, so upgrade that provider
before applying this configuration. Recreate only the observer when intentionally
replacing an installed instance; preserve Helm/database resources otherwise.
33 changes: 13 additions & 20 deletions apis/authstacks/definition.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,33 +69,20 @@ spec:
minItems: 1
items: {type: string}
instanceDiscovery:
description: Opt-in read-only discovery of the bootstrapped instance ID. Uses the chart-generated IAM admin PAT and publishes only non-secret metadata.
description: Opt-in continuous, read-only observation of the installed Zitadel instance using provider-upjet-zitadel v0.3.0 or newer. The supplied ProviderConfig must authenticate to this stack's instance.
type: object
properties:
enabled: {type: boolean, default: false}
internalURL:
description: Origin of this stack's Zitadel Service (HTTPS by default), with explicit service port and no path. Host must match the rendered chart Service in the install namespace.
type: string
pattern: '^https?://[a-z0-9-]+[.][a-z0-9-]+[.]svc([.]cluster[.]local)?:[0-9]+$'
allowInsecureHTTP:
description: Explicit opt-in for trusted local clusters with plaintext Service traffic. Never enable on an untrusted network; the IAM PAT is transmitted in cleartext.
type: boolean
default: false
caCertSecretRef:
description: Optional CA bundle for verifying the Zitadel Service certificate. The certificate must cover the Service hostname. Secret must be in the install namespace.
providerConfigRef:
description: Existing Zitadel ProviderConfig in the XR namespace, or ClusterProviderConfig. Credentials and transport are owned by that provider configuration.
type: object
required: [name, key]
required: [name, kind]
properties:
name: {type: string, minLength: 1}
key: {type: string, minLength: 1}
image:
type: string
default: python:3.13.7-alpine3.22
kind: {type: string, enum: [ProviderConfig, ClusterProviderConfig]}
x-kubernetes-validations:
- rule: '!self.enabled || has(self.internalURL)'
message: instanceDiscovery.internalURL is required when enabled
- rule: '!has(self.internalURL) || self.internalURL.startsWith("https://") || (has(self.allowInsecureHTTP) && self.allowInsecureHTTP)'
message: instanceDiscovery requires HTTPS unless allowInsecureHTTP is explicitly enabled
- rule: '!self.enabled || has(self.providerConfigRef)'
message: instanceDiscovery.providerConfigRef is required when enabled
clusterName:
description: Name of the target cluster. Used as default for provider config refs.
type: string
Expand Down Expand Up @@ -538,6 +525,12 @@ spec:
description: AuthStackStatus surfaces the OIDC contract and chart-managed bootstrap secret refs.
type: object
properties:
instanceRef:
description: Observed namespaced Instance resource for CustomDomain/TrustedDomain instanceIdRef. Populated together with instanceId after a successful current-generation observation.
type: object
properties:
name: {type: string}
namespace: {type: string}
instanceId:
description: Instance ID observed through the Zitadel API, populated when instanceDiscovery is enabled.
type: string
Expand Down
Loading
Loading