diff --git a/.github/workflows/gha-e2e-tests.yaml b/.github/workflows/gha-e2e-tests.yaml index 16b9093dc6..30789c2ec0 100644 --- a/.github/workflows/gha-e2e-tests.yaml +++ b/.github/workflows/gha-e2e-tests.yaml @@ -29,7 +29,8 @@ jobs: default-setup: runs-on: ubuntu-latest timeout-minutes: 20 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id + if: github.event_name != 'pull_request' || + github.event.pull_request.head.repo.id == github.repository_id steps: - uses: actions/checkout@v6 with: @@ -52,7 +53,8 @@ jobs: default-setup-v2: runs-on: ubuntu-latest timeout-minutes: 20 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id + if: github.event_name != 'pull_request' || + github.event.pull_request.head.repo.id == github.repository_id steps: - uses: actions/checkout@v6 with: @@ -74,7 +76,8 @@ jobs: single-namespace-setup: runs-on: ubuntu-latest timeout-minutes: 20 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id + if: github.event_name != 'pull_request' || + github.event.pull_request.head.repo.id == github.repository_id steps: - uses: actions/checkout@v6 with: @@ -96,7 +99,8 @@ jobs: single-namespace-setup-v2: runs-on: ubuntu-latest timeout-minutes: 20 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id + if: github.event_name != 'pull_request' || + github.event.pull_request.head.repo.id == github.repository_id steps: - uses: actions/checkout@v6 with: @@ -118,7 +122,8 @@ jobs: dind-mode-setup: runs-on: ubuntu-latest timeout-minutes: 20 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id + if: github.event_name != 'pull_request' || + github.event.pull_request.head.repo.id == github.repository_id steps: - uses: actions/checkout@v6 with: @@ -132,7 +137,7 @@ jobs: application_private_key: ${{ secrets.E2E_TESTS_ACCESS_PK }} organization: ${{ env.TARGET_ORG }} - name: Run dind mode setup test - run: hack/e2e-test.sh dind-mode-setup-v2 + run: hack/e2e-test.sh dind-mode-setup env: GITHUB_TOKEN: "${{steps.config-token.outputs.token}}" shell: bash @@ -140,7 +145,8 @@ jobs: dind-mode-setup-v2: runs-on: ubuntu-latest timeout-minutes: 20 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id + if: github.event_name != 'pull_request' || + github.event.pull_request.head.repo.id == github.repository_id steps: - uses: actions/checkout@v6 with: @@ -154,7 +160,7 @@ jobs: application_private_key: ${{ secrets.E2E_TESTS_ACCESS_PK }} organization: ${{ env.TARGET_ORG }} - name: Run dind mode setup test - run: hack/e2e-test.sh dind-mode-setup + run: hack/e2e-test.sh dind-mode-setup-v2 env: GITHUB_TOKEN: "${{steps.config-token.outputs.token}}" shell: bash @@ -162,7 +168,8 @@ jobs: kubernetes-mode-setup: runs-on: ubuntu-latest timeout-minutes: 20 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id + if: github.event_name != 'pull_request' || + github.event.pull_request.head.repo.id == github.repository_id steps: - uses: actions/checkout@v6 with: @@ -176,7 +183,7 @@ jobs: application_private_key: ${{ secrets.E2E_TESTS_ACCESS_PK }} organization: ${{ env.TARGET_ORG }} - name: Run kubernetes mode setup test - run: hack/e2e-test.sh kubernetes-mode-setup-v2 + run: hack/e2e-test.sh kubernetes-mode-setup env: GITHUB_TOKEN: "${{steps.config-token.outputs.token}}" shell: bash @@ -184,7 +191,8 @@ jobs: kubernetes-mode-setup-v2: runs-on: ubuntu-latest timeout-minutes: 20 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id + if: github.event_name != 'pull_request' || + github.event.pull_request.head.repo.id == github.repository_id steps: - uses: actions/checkout@v6 with: @@ -198,7 +206,7 @@ jobs: application_private_key: ${{ secrets.E2E_TESTS_ACCESS_PK }} organization: ${{ env.TARGET_ORG }} - name: Run kubernetes mode setup test - run: hack/e2e-test.sh kubernetes-mode-setup + run: hack/e2e-test.sh kubernetes-mode-setup-v2 env: GITHUB_TOKEN: "${{steps.config-token.outputs.token}}" shell: bash @@ -206,7 +214,8 @@ jobs: auth-proxy-setup: runs-on: ubuntu-latest timeout-minutes: 20 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id + if: github.event_name != 'pull_request' || + github.event.pull_request.head.repo.id == github.repository_id steps: - uses: actions/checkout@v6 with: @@ -220,7 +229,7 @@ jobs: application_private_key: ${{ secrets.E2E_TESTS_ACCESS_PK }} organization: ${{ env.TARGET_ORG }} - name: Run single namespace setup test - run: hack/e2e-test.sh single-namespace-setup-v2 + run: hack/e2e-test.sh single-namespace-setup env: GITHUB_TOKEN: "${{steps.config-token.outputs.token}}" shell: bash @@ -228,7 +237,8 @@ jobs: auth-proxy-setup-v2: runs-on: ubuntu-latest timeout-minutes: 20 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id + if: github.event_name != 'pull_request' || + github.event.pull_request.head.repo.id == github.repository_id steps: - uses: actions/checkout@v6 with: @@ -242,7 +252,7 @@ jobs: application_private_key: ${{ secrets.E2E_TESTS_ACCESS_PK }} organization: ${{ env.TARGET_ORG }} - name: Run single namespace setup test - run: hack/e2e-test.sh single-namespace-setup + run: hack/e2e-test.sh single-namespace-setup-v2 env: GITHUB_TOKEN: "${{steps.config-token.outputs.token}}" shell: bash @@ -250,7 +260,8 @@ jobs: anonymous-proxy-setup: runs-on: ubuntu-latest timeout-minutes: 20 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id + if: github.event_name != 'pull_request' || + github.event.pull_request.head.repo.id == github.repository_id steps: - uses: actions/checkout@v6 with: @@ -264,7 +275,7 @@ jobs: application_private_key: ${{ secrets.E2E_TESTS_ACCESS_PK }} organization: ${{ env.TARGET_ORG }} - name: Run anonymous proxy setup test - run: hack/e2e-test.sh anonymous-proxy-setup-v2 + run: hack/e2e-test.sh anonymous-proxy-setup env: GITHUB_TOKEN: "${{steps.config-token.outputs.token}}" shell: bash @@ -272,7 +283,8 @@ jobs: anonymous-proxy-setup-v2: runs-on: ubuntu-latest timeout-minutes: 20 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id + if: github.event_name != 'pull_request' || + github.event.pull_request.head.repo.id == github.repository_id steps: - uses: actions/checkout@v6 with: @@ -286,7 +298,7 @@ jobs: application_private_key: ${{ secrets.E2E_TESTS_ACCESS_PK }} organization: ${{ env.TARGET_ORG }} - name: Run anonymous proxy setup test - run: hack/e2e-test.sh anonymous-proxy-setup + run: hack/e2e-test.sh anonymous-proxy-setup-v2 env: GITHUB_TOKEN: "${{steps.config-token.outputs.token}}" shell: bash @@ -294,7 +306,8 @@ jobs: self-signed-ca-setup: runs-on: ubuntu-latest timeout-minutes: 20 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id + if: github.event_name != 'pull_request' || + github.event.pull_request.head.repo.id == github.repository_id steps: - uses: actions/checkout@v6 with: @@ -308,7 +321,7 @@ jobs: application_private_key: ${{ secrets.E2E_TESTS_ACCESS_PK }} organization: ${{ env.TARGET_ORG }} - name: Run self signed CA setup test - run: hack/e2e-test.sh self-signed-ca-setup-v2 + run: hack/e2e-test.sh self-signed-ca-setup env: GITHUB_TOKEN: "${{steps.config-token.outputs.token}}" shell: bash @@ -316,7 +329,8 @@ jobs: self-signed-ca-setup-v2: runs-on: ubuntu-latest timeout-minutes: 20 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id + if: github.event_name != 'pull_request' || + github.event.pull_request.head.repo.id == github.repository_id steps: - uses: actions/checkout@v6 with: @@ -330,15 +344,16 @@ jobs: application_private_key: ${{ secrets.E2E_TESTS_ACCESS_PK }} organization: ${{ env.TARGET_ORG }} - name: Run self signed CA setup test - run: hack/e2e-test.sh self-signed-ca-setup + run: hack/e2e-test.sh self-signed-ca-setup-v2 env: GITHUB_TOKEN: "${{steps.config-token.outputs.token}}" shell: bash - update-strategy-tests: + update-gha-runner-scale-set-tests: runs-on: ubuntu-latest timeout-minutes: 20 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id + if: github.event_name != 'pull_request' || + github.event.pull_request.head.repo.id == github.repository_id steps: - uses: actions/checkout@v6 with: @@ -352,15 +367,16 @@ jobs: application_private_key: ${{ secrets.E2E_TESTS_ACCESS_PK }} organization: ${{ env.TARGET_ORG }} - name: Run update strategy test - run: hack/e2e-test.sh update-strategy-v2 + run: hack/e2e-test.sh update-gha-runner-scale-set env: GITHUB_TOKEN: "${{steps.config-token.outputs.token}}" shell: bash - update-strategy-tests-v2: + update-gha-runner-scale-set-tests-v2: runs-on: ubuntu-latest timeout-minutes: 20 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id + if: github.event_name != 'pull_request' || + github.event.pull_request.head.repo.id == github.repository_id steps: - uses: actions/checkout@v6 with: @@ -374,7 +390,7 @@ jobs: application_private_key: ${{ secrets.E2E_TESTS_ACCESS_PK }} organization: ${{ env.TARGET_ORG }} - name: Run update strategy test - run: hack/e2e-test.sh update-strategy + run: hack/e2e-test.sh update-gha-runner-scale-set-v2 env: GITHUB_TOKEN: "${{steps.config-token.outputs.token}}" shell: bash @@ -382,7 +398,8 @@ jobs: init-with-min-runners: runs-on: ubuntu-latest timeout-minutes: 20 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id + if: github.event_name != 'pull_request' || + github.event.pull_request.head.repo.id == github.repository_id steps: - uses: actions/checkout@v6 with: @@ -396,7 +413,7 @@ jobs: application_private_key: ${{ secrets.E2E_TESTS_ACCESS_PK }} organization: ${{ env.TARGET_ORG }} - name: Run init with min runners test - run: hack/e2e-test.sh init-with-min-runners-v2 + run: hack/e2e-test.sh init-with-min-runners env: GITHUB_TOKEN: "${{steps.config-token.outputs.token}}" shell: bash @@ -404,7 +421,8 @@ jobs: init-with-min-runners-v2: runs-on: ubuntu-latest timeout-minutes: 20 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id + if: github.event_name != 'pull_request' || + github.event.pull_request.head.repo.id == github.repository_id steps: - uses: actions/checkout@v6 with: @@ -418,8 +436,7 @@ jobs: application_private_key: ${{ secrets.E2E_TESTS_ACCESS_PK }} organization: ${{ env.TARGET_ORG }} - name: Run init with min runners test - run: hack/e2e-test.sh init-with-min-runners + run: hack/e2e-test.sh init-with-min-runners-v2 env: GITHUB_TOKEN: "${{steps.config-token.outputs.token}}" shell: bash - diff --git a/apis/actions.github.com/v1alpha1/autoscalinglistener_types.go b/apis/actions.github.com/v1alpha1/autoscalinglistener_types.go index 7009369288..1ae599f1fb 100644 --- a/apis/actions.github.com/v1alpha1/autoscalinglistener_types.go +++ b/apis/actions.github.com/v1alpha1/autoscalinglistener_types.go @@ -17,6 +17,7 @@ limitations under the License. package v1alpha1 import ( + "github.com/actions/actions-runner-controller/hash" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -24,13 +25,13 @@ import ( // AutoscalingListenerSpec defines the desired state of AutoscalingListener type AutoscalingListenerSpec struct { // Required - GitHubConfigUrl string `json:"githubConfigUrl,omitempty"` + GitHubConfigURL string `json:"githubConfigUrl,omitempty"` // Required GitHubConfigSecret string `json:"githubConfigSecret,omitempty"` // Required - RunnerScaleSetId int `json:"runnerScaleSetId,omitempty"` + RunnerScaleSetID int `json:"runnerScaleSetId,omitempty"` // Required AutoscalingRunnerSetNamespace string `json:"autoscalingRunnerSetNamespace,omitempty"` @@ -83,6 +84,10 @@ type AutoscalingListenerSpec struct { RoleBindingMetadata *ResourceMeta `json:"roleBindingMetadata,omitempty"` } +func (s *AutoscalingListenerSpec) Hash() string { + return hash.ComputeTemplateHash(s) +} + // AutoscalingListenerStatus defines the observed state of AutoscalingListener type AutoscalingListenerStatus struct{} diff --git a/apis/actions.github.com/v1alpha1/ephemeralrunner_types.go b/apis/actions.github.com/v1alpha1/ephemeralrunner_types.go index f9b4df528c..6c2c03150e 100644 --- a/apis/actions.github.com/v1alpha1/ephemeralrunner_types.go +++ b/apis/actions.github.com/v1alpha1/ephemeralrunner_types.go @@ -17,6 +17,7 @@ limitations under the License. package v1alpha1 import ( + "github.com/actions/actions-runner-controller/hash" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -77,7 +78,7 @@ func (er *EphemeralRunner) GitHubConfigSecret() string { } func (er *EphemeralRunner) GitHubConfigUrl() string { - return er.Spec.GitHubConfigUrl + return er.Spec.GitHubConfigURL } func (er *EphemeralRunner) GitHubProxy() *ProxyConfig { @@ -102,7 +103,7 @@ func (er *EphemeralRunner) VaultProxy() *ProxyConfig { // EphemeralRunnerSpec defines the desired state of EphemeralRunner type EphemeralRunnerSpec struct { // +required - GitHubConfigUrl string `json:"githubConfigUrl,omitempty"` + GitHubConfigURL string `json:"githubConfigUrl,omitempty"` // +required GitHubConfigSecret string `json:"githubConfigSecret,omitempty"` @@ -128,6 +129,10 @@ type EphemeralRunnerSpec struct { corev1.PodTemplateSpec `json:",inline"` } +func (s *EphemeralRunnerSpec) Hash() string { + return hash.ComputeTemplateHash(s) +} + // EphemeralRunnerStatus defines the observed state of EphemeralRunner type EphemeralRunnerStatus struct { // Turns true only if the runner is online. diff --git a/apis/actions.github.com/v1alpha1/ephemeralrunnerset_types.go b/apis/actions.github.com/v1alpha1/ephemeralrunnerset_types.go index 229bb252fc..bfcd424d84 100644 --- a/apis/actions.github.com/v1alpha1/ephemeralrunnerset_types.go +++ b/apis/actions.github.com/v1alpha1/ephemeralrunnerset_types.go @@ -28,6 +28,9 @@ type EphemeralRunnerSetSpec struct { PatchID int `json:"patchID"` // EphemeralRunnerSpec is the spec of the ephemeral runner EphemeralRunnerSpec EphemeralRunnerSpec `json:"ephemeralRunnerSpec,omitempty"` + // EphemeralRunnerMetadata is the metadata to be applied to all ephemeral runners created by this set. + // If the EphemeralRunnerMetadata is updated, the update applies to new ephemeral runners created after the update, + // but does not apply to existing ephemeral runners. // +optional EphemeralRunnerMetadata *ResourceMeta `json:"ephemeralRunnerMetadata,omitempty"` } @@ -74,12 +77,17 @@ type EphemeralRunnerSet struct { Status EphemeralRunnerSetStatus `json:"status,omitempty"` } +// EphemeralRunnerSpecHash computes the hash value of the EphemeralRunnerSpec and returns it as a string. +func (ers *EphemeralRunnerSet) EphemeralRunnerSpecHash() string { + return ers.Spec.EphemeralRunnerSpec.Hash() +} + func (ers *EphemeralRunnerSet) GitHubConfigSecret() string { return ers.Spec.EphemeralRunnerSpec.GitHubConfigSecret } func (ers *EphemeralRunnerSet) GitHubConfigUrl() string { - return ers.Spec.EphemeralRunnerSpec.GitHubConfigUrl + return ers.Spec.EphemeralRunnerSpec.GitHubConfigURL } func (ers *EphemeralRunnerSet) GitHubProxy() *ProxyConfig { diff --git a/charts/gha-runner-scale-set-controller-experimental/crds/actions.github.com_ephemeralrunnersets.yaml b/charts/gha-runner-scale-set-controller-experimental/crds/actions.github.com_ephemeralrunnersets.yaml index a2c8f787be..52ba40ed72 100644 --- a/charts/gha-runner-scale-set-controller-experimental/crds/actions.github.com_ephemeralrunnersets.yaml +++ b/charts/gha-runner-scale-set-controller-experimental/crds/actions.github.com_ephemeralrunnersets.yaml @@ -59,7 +59,10 @@ spec: description: EphemeralRunnerSetSpec defines the desired state of EphemeralRunnerSet properties: ephemeralRunnerMetadata: - description: ResourceMeta carries metadata common to all internal resources + description: |- + EphemeralRunnerMetadata is the metadata to be applied to all ephemeral runners created by this set. + If the EphemeralRunnerMetadata is updated, the update applies to new ephemeral runners created after the update, + but does not apply to existing ephemeral runners. properties: annotations: additionalProperties: diff --git a/charts/gha-runner-scale-set-controller-experimental/templates/_controller_template.tpl b/charts/gha-runner-scale-set-controller-experimental/templates/_controller_template.tpl index 18cd6ccfc6..d1391d13ea 100644 --- a/charts/gha-runner-scale-set-controller-experimental/templates/_controller_template.tpl +++ b/charts/gha-runner-scale-set-controller-experimental/templates/_controller_template.tpl @@ -50,9 +50,6 @@ args: {{- with .Values.controller.manager.config.runnerMaxConcurrentReconciles }} - "--runner-max-concurrent-reconciles={{ . }}" {{- end }} -{{- with .Values.controller.manager.config.updateStrategy }} - - "--update-strategy={{ . }}" -{{- end }} {{- if .Values.controller.metrics }} {{- with .Values.controller.metrics }} - "--listener-metrics-addr={{ .listenerAddr }}" diff --git a/charts/gha-runner-scale-set-controller-experimental/values.yaml b/charts/gha-runner-scale-set-controller-experimental/values.yaml index a06d86c695..cdc405b5d8 100644 --- a/charts/gha-runner-scale-set-controller-experimental/values.yaml +++ b/charts/gha-runner-scale-set-controller-experimental/values.yaml @@ -31,9 +31,6 @@ controller: # The maximum number of concurrent reconciles which can be run by the EphemeralRunner controller. runnerMaxConcurrentReconciles: 2 - # How the controller handles upgrades with running jobs: "immediate" or "eventual". - updateStrategy: "immediate" - # List of label prefixes that should NOT be propagated to internal resources. excludeLabelPropagationPrefixes: [] # Example: diff --git a/charts/gha-runner-scale-set-controller/crds/actions.github.com_ephemeralrunnersets.yaml b/charts/gha-runner-scale-set-controller/crds/actions.github.com_ephemeralrunnersets.yaml index a2c8f787be..52ba40ed72 100644 --- a/charts/gha-runner-scale-set-controller/crds/actions.github.com_ephemeralrunnersets.yaml +++ b/charts/gha-runner-scale-set-controller/crds/actions.github.com_ephemeralrunnersets.yaml @@ -59,7 +59,10 @@ spec: description: EphemeralRunnerSetSpec defines the desired state of EphemeralRunnerSet properties: ephemeralRunnerMetadata: - description: ResourceMeta carries metadata common to all internal resources + description: |- + EphemeralRunnerMetadata is the metadata to be applied to all ephemeral runners created by this set. + If the EphemeralRunnerMetadata is updated, the update applies to new ephemeral runners created after the update, + but does not apply to existing ephemeral runners. properties: annotations: additionalProperties: diff --git a/charts/gha-runner-scale-set-controller/templates/deployment.yaml b/charts/gha-runner-scale-set-controller/templates/deployment.yaml index 628336f491..57ad21a3f3 100644 --- a/charts/gha-runner-scale-set-controller/templates/deployment.yaml +++ b/charts/gha-runner-scale-set-controller/templates/deployment.yaml @@ -70,9 +70,6 @@ spec: {{- with .Values.flags.runnerMaxConcurrentReconciles }} - "--runner-max-concurrent-reconciles={{ . }}" {{- end }} - {{- with .Values.flags.updateStrategy }} - - "--update-strategy={{ . }}" - {{- end }} {{- if .Values.metrics }} {{- with .Values.metrics }} - "--listener-metrics-addr={{ .listenerAddr }}" diff --git a/charts/gha-runner-scale-set-controller/tests/template_test.go b/charts/gha-runner-scale-set-controller/tests/template_test.go index c39cceeec4..c84e27bee3 100644 --- a/charts/gha-runner-scale-set-controller/tests/template_test.go +++ b/charts/gha-runner-scale-set-controller/tests/template_test.go @@ -363,7 +363,6 @@ func TestTemplate_ControllerDeployment_Defaults(t *testing.T) { "--auto-scaling-runner-set-only", "--log-level=debug", "--log-format=text", - "--update-strategy=immediate", "--metrics-addr=0", "--listener-metrics-addr=0", "--listener-metrics-endpoint=", @@ -431,7 +430,6 @@ func TestTemplate_ControllerDeployment_Customize(t *testing.T) { "topologySpreadConstraints[0].maxSkew": "1", "topologySpreadConstraints[0].topologyKey": "foo", "priorityClassName": "test-priority-class", - "flags.updateStrategy": "eventual", "flags.logLevel": "info", "flags.logFormat": "json", "volumes[0].name": "customMount", @@ -516,7 +514,6 @@ func TestTemplate_ControllerDeployment_Customize(t *testing.T) { "--auto-scaler-image-pull-secrets=dockerhub", "--log-level=info", "--log-format=json", - "--update-strategy=eventual", "--listener-metrics-addr=0", "--listener-metrics-endpoint=", "--metrics-addr=0", @@ -645,7 +642,6 @@ func TestTemplate_EnableLeaderElection(t *testing.T) { "--leader-election-id=test-arc-gha-rs-controller", "--log-level=debug", "--log-format=text", - "--update-strategy=immediate", "--listener-metrics-addr=0", "--listener-metrics-endpoint=", "--metrics-addr=0", @@ -687,7 +683,6 @@ func TestTemplate_ControllerDeployment_ForwardImagePullSecrets(t *testing.T) { "--auto-scaler-image-pull-secrets=ghcr", "--log-level=debug", "--log-format=text", - "--update-strategy=immediate", "--listener-metrics-addr=0", "--listener-metrics-endpoint=", "--metrics-addr=0", @@ -778,7 +773,6 @@ func TestTemplate_ControllerDeployment_WatchSingleNamespace(t *testing.T) { "--log-level=debug", "--log-format=text", "--watch-single-namespace=demo", - "--update-strategy=immediate", "--listener-metrics-addr=0", "--listener-metrics-endpoint=", "--metrics-addr=0", diff --git a/charts/gha-runner-scale-set-controller/values.yaml b/charts/gha-runner-scale-set-controller/values.yaml index 5e76406e51..b83d505385 100644 --- a/charts/gha-runner-scale-set-controller/values.yaml +++ b/charts/gha-runner-scale-set-controller/values.yaml @@ -115,22 +115,6 @@ flags: # It may also increase the load on the API server and the external service (e.g. GitHub API). runnerMaxConcurrentReconciles: 2 - ## Defines how the controller should handle upgrades while having running jobs. - ## - ## The strategies available are: - ## - "immediate": (default) The controller will immediately apply the change causing the - ## recreation of the listener and ephemeral runner set. This can lead to an - ## overprovisioning of runners, if there are pending / running jobs. This should not - ## be a problem at a small scale, but it could lead to a significant increase of - ## resources if you have a lot of jobs running concurrently. - ## - ## - "eventual": The controller will remove the listener and ephemeral runner set - ## immediately, but will not recreate them (to apply changes) until all - ## pending / running jobs have completed. - ## This can lead to a longer time to apply the change but it will ensure - ## that you don't have any overprovisioning of runners. - updateStrategy: "immediate" - ## Defines a list of prefixes that should not be propagated to internal resources. ## This is useful when you have labels that are used for internal purposes and should not be propagated to internal resources. ## See https://github.com/actions/actions-runner-controller/issues/3533 for more information. diff --git a/config/crd/bases/actions.github.com_ephemeralrunnersets.yaml b/config/crd/bases/actions.github.com_ephemeralrunnersets.yaml index a2c8f787be..52ba40ed72 100644 --- a/config/crd/bases/actions.github.com_ephemeralrunnersets.yaml +++ b/config/crd/bases/actions.github.com_ephemeralrunnersets.yaml @@ -59,7 +59,10 @@ spec: description: EphemeralRunnerSetSpec defines the desired state of EphemeralRunnerSet properties: ephemeralRunnerMetadata: - description: ResourceMeta carries metadata common to all internal resources + description: |- + EphemeralRunnerMetadata is the metadata to be applied to all ephemeral runners created by this set. + If the EphemeralRunnerMetadata is updated, the update applies to new ephemeral runners created after the update, + but does not apply to existing ephemeral runners. properties: annotations: additionalProperties: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 2d41d8542b..dc0becfdf7 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -46,22 +46,13 @@ rules: - "" resources: - secrets - verbs: - - create - - delete - - get - - list - - update - - watch -- apiGroups: - - "" - resources: - serviceaccounts verbs: - create - delete - get - list + - update - watch - apiGroups: - actions.github.com @@ -170,15 +161,6 @@ rules: - rbac.authorization.k8s.io resources: - rolebindings - verbs: - - create - - delete - - get - - list - - watch -- apiGroups: - - rbac.authorization.k8s.io - resources: - roles verbs: - create diff --git a/controllers/actions.github.com/autoscalinglistener_controller.go b/controllers/actions.github.com/autoscalinglistener_controller.go index b45792d150..0146450f32 100644 --- a/controllers/actions.github.com/autoscalinglistener_controller.go +++ b/controllers/actions.github.com/autoscalinglistener_controller.go @@ -17,10 +17,10 @@ limitations under the License. package actionsgithubcom import ( - "bytes" "context" "fmt" "maps" + "reflect" "time" "github.com/go-logr/logr" @@ -38,10 +38,8 @@ import ( "github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1/appconfig" "github.com/actions/actions-runner-controller/controllers/actions.github.com/metrics" "github.com/actions/actions-runner-controller/github/actions" - hash "github.com/actions/actions-runner-controller/hash" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( @@ -65,9 +63,9 @@ type AutoscalingListenerReconciler struct { // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=core,resources=pods/status,verbs=get // +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update -// +kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=get;list;watch;create +// +kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=get;list;watch;create;update // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles,verbs=create;delete;get;list;watch;update -// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=rolebindings,verbs=create;delete;get;list;watch +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=rolebindings,verbs=create;delete;get;list;watch;update // +kubebuilder:rbac:groups=actions.github.com,resources=autoscalinglisteners,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=actions.github.com,resources=autoscalinglisteners/status,verbs=get;update;patch // +kubebuilder:rbac:groups=actions.github.com,resources=autoscalinglisteners/finalizers,verbs=update @@ -76,18 +74,19 @@ type AutoscalingListenerReconciler struct { func (r *AutoscalingListenerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.Log.WithValues("autoscalinglistener", req.NamespacedName) - autoscalingListener := new(v1alpha1.AutoscalingListener) - if err := r.Get(ctx, req.NamespacedName, autoscalingListener); err != nil { + var autoscalingListener v1alpha1.AutoscalingListener + if err := r.Get(ctx, req.NamespacedName, &autoscalingListener); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } + original := autoscalingListener.DeepCopy() if !autoscalingListener.DeletionTimestamp.IsZero() { - if !controllerutil.ContainsFinalizer(autoscalingListener, autoscalingListenerFinalizerName) { + if !controllerutil.ContainsFinalizer(&autoscalingListener, autoscalingListenerFinalizerName) { return ctrl.Result{}, nil } log.Info("Deleting resources") - requeue, err := r.cleanupResources(ctx, autoscalingListener, log) + requeue, err := r.cleanupResources(ctx, &autoscalingListener, log) if err != nil { log.Error(err, "Failed to cleanup resources after deletion") return ctrl.Result{}, err @@ -98,23 +97,19 @@ func (r *AutoscalingListenerReconciler) Reconcile(ctx context.Context, req ctrl. } log.Info("Removing finalizer") - err = patch(ctx, r.Client, autoscalingListener, func(obj *v1alpha1.AutoscalingListener) { - controllerutil.RemoveFinalizer(obj, autoscalingListenerFinalizerName) - }) - if err != nil && !kerrors.IsNotFound(err) { - log.Error(err, "Failed to remove finalizer") - return ctrl.Result{}, err + if controllerutil.RemoveFinalizer(&autoscalingListener, autoscalingListenerFinalizerName) { + if err := r.Patch(ctx, &autoscalingListener, client.MergeFrom(original)); err != nil && !kerrors.IsNotFound(err) { + log.Error(err, "Failed to remove finalizer") + return ctrl.Result{}, err + } } log.Info("Successfully removed finalizer after cleanup") return ctrl.Result{}, nil } - if !controllerutil.ContainsFinalizer(autoscalingListener, autoscalingListenerFinalizerName) { - log.Info("Adding finalizer") - if err := patch(ctx, r.Client, autoscalingListener, func(obj *v1alpha1.AutoscalingListener) { - controllerutil.AddFinalizer(obj, autoscalingListenerFinalizerName) - }); err != nil { + if controllerutil.AddFinalizer(&autoscalingListener, autoscalingListenerFinalizerName) { + if err := r.Patch(ctx, &autoscalingListener, client.MergeFrom(original)); err != nil { log.Error(err, "Failed to add finalizer") return ctrl.Result{}, err } @@ -125,101 +120,243 @@ func (r *AutoscalingListenerReconciler) Reconcile(ctx context.Context, req ctrl. // Check if the AutoscalingRunnerSet exists var autoscalingRunnerSet v1alpha1.AutoscalingRunnerSet - if err := r.Get(ctx, types.NamespacedName{Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace, Name: autoscalingListener.Spec.AutoscalingRunnerSetName}, &autoscalingRunnerSet); err != nil { - log.Error(err, "Failed to find AutoscalingRunnerSet.", - "namespace", autoscalingListener.Spec.AutoscalingRunnerSetNamespace, - "name", autoscalingListener.Spec.AutoscalingRunnerSetName) - return ctrl.Result{}, err - } + if err := r.Get( + ctx, + types.NamespacedName{ + Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace, + Name: autoscalingListener.Spec.AutoscalingRunnerSetName, + }, + &autoscalingRunnerSet, + ); err != nil { + if kerrors.IsNotFound(err) { + log.Info("AutoscalingRunnerSet is not found, deleting autoscaling listener", "namespace", autoscalingListener.Spec.AutoscalingRunnerSetNamespace, "name", autoscalingListener.Spec.AutoscalingRunnerSetName) + if err := r.Delete(ctx, &autoscalingListener); err != nil { + log.Error(err, "failed to delete autoscaling listener", "namespace", autoscalingListener.Namespace, "name", autoscalingListener.Name) + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } - appConfig, err := r.GetAppConfig(ctx, &autoscalingRunnerSet) - if err != nil { log.Error( - err, - "Failed to get app config for AutoscalingRunnerSet.", - "namespace", - autoscalingRunnerSet.Namespace, - "name", - autoscalingRunnerSet.GitHubConfigSecret, + err, "Failed to find AutoscalingRunnerSet.", + "namespace", autoscalingListener.Spec.AutoscalingRunnerSetNamespace, + "name", autoscalingListener.Spec.AutoscalingRunnerSetName, ) return ctrl.Result{}, err } // Make sure the runner scale set listener service account is created for the listener pod in the controller namespace - serviceAccount := new(corev1.ServiceAccount) - if err := r.Get(ctx, types.NamespacedName{Namespace: autoscalingListener.Namespace, Name: autoscalingListener.Name}, serviceAccount); err != nil { - if !kerrors.IsNotFound(err) { - log.Error(err, "Unable to get listener service accounts", "namespace", autoscalingListener.Namespace, "name", autoscalingListener.Name) + var serviceAccount corev1.ServiceAccount + err := r.Get( + ctx, + types.NamespacedName{ + Namespace: autoscalingListener.Namespace, + Name: autoscalingListener.Name, + }, + &serviceAccount, + ) + switch { + case err == nil: + desiredServiceAccount, err := r.newScaleSetListenerServiceAccount(&autoscalingListener) + if err != nil { + log.Error(err, "Failed to build desired listener service account") return ctrl.Result{}, err } + updatedServiceAccount := serviceAccount.DeepCopy() + var shouldUpdate bool + desiredLabels := r.filterAndMergeLabels(serviceAccount.Labels, desiredServiceAccount.Labels) + if !maps.Equal(serviceAccount.Labels, desiredLabels) { + updatedServiceAccount.Labels = desiredLabels + shouldUpdate = true + } + desiredAnnotations := r.mergeAnnotations(serviceAccount.Annotations, desiredServiceAccount.Annotations) + if !maps.Equal(serviceAccount.Annotations, desiredAnnotations) { + updatedServiceAccount.Annotations = desiredAnnotations + shouldUpdate = true + } + if shouldUpdate { + log.Info("Updating listener service account") + + if err := r.Update(ctx, updatedServiceAccount); err != nil { + log.Error(err, "Failed to update listener service account") + return ctrl.Result{}, err + } + + return ctrl.Result{Requeue: true}, nil + } + case kerrors.IsNotFound(err): // Create a service account for the listener pod in the controller namespace log.Info("Creating a service account for the listener pod") - return r.createServiceAccountForListener(ctx, autoscalingListener, log) + return r.createServiceAccountForListener(ctx, &autoscalingListener, log) + default: + log.Error(err, "Unable to get listener service accounts", "namespace", autoscalingListener.Namespace, "name", autoscalingListener.Name) + return ctrl.Result{}, err } - // TODO: make sure the service account is up to date - // Make sure the runner scale set listener role is created in the AutoscalingRunnerSet namespace - listenerRole := new(rbacv1.Role) - if err := r.Get(ctx, types.NamespacedName{Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace, Name: autoscalingListener.Name}, listenerRole); err != nil { - if !kerrors.IsNotFound(err) { - log.Error(err, "Unable to get listener role", "namespace", autoscalingListener.Spec.AutoscalingRunnerSetNamespace, "name", autoscalingListener.Name) - return ctrl.Result{}, err + var listenerRole rbacv1.Role + err = r.Get( + ctx, + types.NamespacedName{ + Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace, + Name: autoscalingListener.Name, + }, + &listenerRole, + ) + switch { + case err == nil: + newRole := r.newScaleSetListenerRole(&autoscalingListener) + updatedRole := listenerRole.DeepCopy() + var shouldUpdate bool + desiredLabels := r.filterAndMergeLabels(listenerRole.Labels, newRole.Labels) + if !maps.Equal(listenerRole.Labels, desiredLabels) { + updatedRole.Labels = desiredLabels + shouldUpdate = true + } + desiredAnnotations := r.mergeAnnotations(listenerRole.Annotations, newRole.Annotations) + if !maps.Equal(listenerRole.Annotations, desiredAnnotations) { + updatedRole.Annotations = desiredAnnotations + shouldUpdate = true + } + if !reflect.DeepEqual(listenerRole.Rules, newRole.Rules) { + updatedRole.Rules = newRole.Rules + shouldUpdate = true + } + if shouldUpdate { + log.Info("Updating listener role") + if err := r.Update(ctx, updatedRole); err != nil { + log.Error(err, "Failed to update listener role") + return ctrl.Result{}, err + } + return ctrl.Result{Requeue: true}, nil } - + case kerrors.IsNotFound(err): // Create a role for the listener pod in the AutoScalingRunnerSet namespace log.Info("Creating a role for the listener pod") - return r.createRoleForListener(ctx, autoscalingListener, log) - } - - // Make sure the listener role has the up-to-date rules - existingRuleHash := listenerRole.Labels["role-policy-rules-hash"] - desiredRules := rulesForListenerRole([]string{autoscalingListener.Spec.EphemeralRunnerSetName}) - desiredRulesHash := hash.ComputeTemplateHash(&desiredRules) - if existingRuleHash != desiredRulesHash { - log.Info("Updating the listener role with the up-to-date rules") - return r.updateRoleForListener(ctx, listenerRole, desiredRules, desiredRulesHash, log) + return r.createRoleForListener(ctx, &autoscalingListener, log) + default: // error + log.Error(err, "Unable to get listener role", "namespace", autoscalingListener.Spec.AutoscalingRunnerSetNamespace, "name", autoscalingListener.Name) + return ctrl.Result{}, err } // Make sure the runner scale set listener role binding is created - listenerRoleBinding := new(rbacv1.RoleBinding) - if err := r.Get(ctx, types.NamespacedName{Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace, Name: autoscalingListener.Name}, listenerRoleBinding); err != nil { - if !kerrors.IsNotFound(err) { - log.Error(err, "Unable to get listener role binding", "namespace", autoscalingListener.Spec.AutoscalingRunnerSetNamespace, "name", autoscalingListener.Name) - return ctrl.Result{}, err + var listenerRoleBinding rbacv1.RoleBinding + err = r.Get(ctx, types.NamespacedName{Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace, Name: autoscalingListener.Name}, &listenerRoleBinding) + switch { + case err == nil: + desiredRoleBinding := r.newScaleSetListenerRoleBinding( + &autoscalingListener, + &listenerRole, + &serviceAccount, + ) + updatedRoleBinding := listenerRoleBinding.DeepCopy() + var shouldUpdate bool + desiredLabels := r.filterAndMergeLabels(listenerRoleBinding.Labels, desiredRoleBinding.Labels) + if !maps.Equal(listenerRoleBinding.Labels, desiredLabels) { + updatedRoleBinding.Labels = desiredLabels + shouldUpdate = true + } + desiredAnnotations := r.mergeAnnotations(listenerRoleBinding.Annotations, desiredRoleBinding.Annotations) + if !maps.Equal(listenerRoleBinding.Annotations, desiredAnnotations) { + updatedRoleBinding.Annotations = desiredAnnotations + shouldUpdate = true + } + if shouldUpdate { + log.Info("Updating listener role binding") + if err := r.Update(ctx, updatedRoleBinding); err != nil { + log.Error(err, "Failed to update listener role binding") + return ctrl.Result{}, err + } + + log.Info("Updated listener role binding") + return ctrl.Result{Requeue: true}, nil } + case kerrors.IsNotFound(err): // Create a role binding for the listener pod in the AutoScalingRunnerSet namespace log.Info("Creating a role binding for the service account and role") - return r.createRoleBindingForListener(ctx, autoscalingListener, listenerRole, serviceAccount, log) + return r.createRoleBindingForListener( + ctx, + &autoscalingListener, + &listenerRole, + &serviceAccount, + log, + ) + default: // error + log.Error(err, "Unable to get listener role binding", "namespace", autoscalingListener.Spec.AutoscalingRunnerSetNamespace, "name", autoscalingListener.Name) + return ctrl.Result{}, err } // Create a secret containing proxy config if specified if autoscalingListener.Spec.Proxy != nil { - proxySecret := new(corev1.Secret) - if err := r.Get(ctx, types.NamespacedName{Namespace: autoscalingListener.Namespace, Name: proxyListenerSecretName(autoscalingListener)}, proxySecret); err != nil { - if !kerrors.IsNotFound(err) { - log.Error(err, "Unable to get listener proxy secret", "namespace", autoscalingListener.Namespace, "name", proxyListenerSecretName(autoscalingListener)) + var proxySecret corev1.Secret + err := r.Get( + ctx, + types.NamespacedName{ + Namespace: autoscalingListener.Namespace, + Name: proxyListenerSecretName(&autoscalingListener), + }, + &proxySecret, + ) + switch { + case err == nil: + desiredListenerProxy, err := r.newAutoscalingListenerProxySecret(&autoscalingListener, proxySecret.Data) + if err != nil { + log.Error(err, "Failed to build desired listener proxy secret") return ctrl.Result{}, err } - + updatedProxySecret := proxySecret.DeepCopy() + var shouldUpdate bool + desiredLabels := r.filterAndMergeLabels(proxySecret.Labels, desiredListenerProxy.Labels) + if !maps.Equal(proxySecret.Labels, desiredLabels) { + updatedProxySecret.Labels = desiredLabels + shouldUpdate = true + } + desiredAnnotations := r.mergeAnnotations(proxySecret.Annotations, desiredListenerProxy.Annotations) + if !maps.Equal(proxySecret.Annotations, desiredAnnotations) { + updatedProxySecret.Annotations = desiredAnnotations + shouldUpdate = true + } + if shouldUpdate { + log.Info("Updating listener proxy secret") + if err := r.Update(ctx, updatedProxySecret); err != nil { + log.Error(err, "Failed to update listener proxy secret") + return ctrl.Result{}, err + } + return ctrl.Result{Requeue: true}, nil + } + case kerrors.IsNotFound(err): // Create a mirror secret for the listener pod in the Controller namespace for listener pod to use log.Info("Creating a listener proxy secret for the listener pod") - return r.createProxySecret(ctx, autoscalingListener, log) + return r.createProxySecret(ctx, &autoscalingListener, log) + default: // error + log.Error(err, "Unable to get listener proxy secret", "namespace", autoscalingListener.Namespace, "name", proxyListenerSecretName(&autoscalingListener)) + return ctrl.Result{}, err } } - // TODO: make sure the role binding has the up-to-date role and service account + var appConfig *appconfig.AppConfig + getAppConfig := func() (*appconfig.AppConfig, error) { + if appConfig != nil { + return appConfig, nil + } - // Reconcile listener config secret and detect drift - cert := "" - if autoscalingListener.Spec.GitHubServerTLS != nil { - var err error - cert, err = r.certificate(ctx, &autoscalingRunnerSet, autoscalingListener) + cfg, err := r.GetAppConfig(ctx, &autoscalingRunnerSet) if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to build GitHub server TLS certificate value for listener config: %w", err) + log.Error( + err, + "Failed to get app config for AutoscalingRunnerSet.", + "namespace", + autoscalingRunnerSet.Namespace, + "name", + autoscalingRunnerSet.GitHubConfigSecret, + ) + return nil, err } + + appConfig = cfg + return appConfig, nil } var metricsConfig *listenerMetricsServerConfig @@ -230,43 +367,169 @@ func (r *AutoscalingListenerReconciler) Reconcile(ctx context.Context, req ctrl. } } - secretChanged, err := r.reconcileListenerConfigSecret(ctx, autoscalingListener, appConfig, metricsConfig, cert, log) - if err != nil { - log.Error(err, "Failed to reconcile listener config secret") + var listenerConfigSecret corev1.Secret + err = r.Get( + ctx, + types.NamespacedName{ + Namespace: autoscalingListener.Namespace, + Name: scaleSetListenerConfigName(&autoscalingListener), + }, + &listenerConfigSecret, + ) + switch { + case err == nil: + cfg, err := r.GetAppConfig(ctx, &autoscalingRunnerSet) + if err != nil { + return ctrl.Result{}, err + } + cert := "" + if autoscalingListener.Spec.GitHubServerTLS != nil { + cert, err = r.certificate(ctx, &autoscalingRunnerSet, &autoscalingListener) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to build GitHub server TLS certificate value for listener config: %w", err) + } + } + desiredSecret, err := r.newScaleSetListenerConfig(&autoscalingListener, cfg, metricsConfig, cert) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to build listener config secret: %w", err) + } + updatedSecret := listenerConfigSecret.DeepCopy() + var shouldUpdate bool + desiredLabels := r.filterAndMergeLabels(listenerConfigSecret.Labels, desiredSecret.Labels) + if !maps.Equal(listenerConfigSecret.Labels, desiredLabels) { + updatedSecret.Labels = desiredLabels + shouldUpdate = true + } + desiredAnnotations := r.mergeAnnotations(listenerConfigSecret.Annotations, desiredSecret.Annotations) + if !maps.Equal(listenerConfigSecret.Annotations, desiredAnnotations) { + updatedSecret.Annotations = desiredAnnotations + shouldUpdate = true + } + + if shouldUpdate { + log.Info("Updating listener config secret", "namespace", updatedSecret.Namespace, "name", updatedSecret.Name) + if err := r.Update(ctx, updatedSecret); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to update listener config secret: %w", err) + } + return ctrl.Result{Requeue: true}, nil + } + case kerrors.IsNotFound(err): + cfg, err := getAppConfig() + if err != nil { + return ctrl.Result{}, err + } + + cert := "" + if autoscalingListener.Spec.GitHubServerTLS != nil { + cert, err = r.certificate(ctx, &autoscalingRunnerSet, &autoscalingListener) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to build GitHub server TLS certificate value for listener config: %w", err) + } + } + desiredSecret, err := r.newScaleSetListenerConfig(&autoscalingListener, cfg, metricsConfig, cert) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to build listener config secret: %w", err) + } + + log.Info("Creating listener config secret", "namespace", desiredSecret.Namespace, "name", desiredSecret.Name) + if err := r.Create(ctx, desiredSecret); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to create listener config secret: %w", err) + } + + // Requeue to create listener pod with the config secret + return ctrl.Result{Requeue: true}, nil + default: + log.Error(err, "Unable to get listener config secret", "namespace", autoscalingListener.Namespace, "name", scaleSetListenerConfigName(&autoscalingListener)) return ctrl.Result{}, err } - listenerPod := new(corev1.Pod) - if err := r.Get( + var listenerPod corev1.Pod + err = r.Get( ctx, client.ObjectKey{ Namespace: autoscalingListener.Namespace, Name: autoscalingListener.Name, }, - listenerPod, - ); err != nil { - if !kerrors.IsNotFound(err) { - log.Error(err, "Unable to get listener pod", "namespace", autoscalingListener.Namespace, "name", autoscalingListener.Name) + &listenerPod, + ) + switch { + case err == nil: + desiredPod, err := r.newScaleSetListenerPod( + &autoscalingListener, + &listenerConfigSecret, + &serviceAccount, + &listenerRole, + &listenerRoleBinding, + metricsConfig, + ) + if err != nil { + log.Error(err, "Failed to build listener pod") return ctrl.Result{}, err } - if err := r.publishRunningListener(autoscalingListener, false); err != nil { + shouldReCreate := desiredPod.Annotations[annotationKeyIntegrityHash] != listenerPod.Annotations[annotationKeyIntegrityHash] + if shouldReCreate { + log.Info("Listener pod dependency changed, recreating listener pod") + if err := r.deleteListenerPod(ctx, &autoscalingListener, &listenerPod, log); err != nil { + return ctrl.Result{}, err + } + + log.Info("Listener pod is deleted, will recreate with new dependencies") + return ctrl.Result{}, nil + } + + updatedPod := listenerPod.DeepCopy() + var shouldUpdate bool + desiredLabels := r.filterAndMergeLabels(listenerPod.Labels, desiredPod.Labels) + if !maps.Equal(listenerPod.Labels, desiredLabels) { + updatedPod.Labels = desiredLabels + shouldUpdate = true + } + desiredAnnotations := r.mergeAnnotations(listenerPod.Annotations, desiredPod.Annotations) + if !maps.Equal(listenerPod.Annotations, desiredAnnotations) { + updatedPod.Annotations = desiredAnnotations + shouldUpdate = true + } + + if shouldUpdate { + log.Info("Updating listener pod", "namespace", updatedPod.Namespace, "name", updatedPod.Name) + if err := r.Update(ctx, updatedPod); err != nil { + log.Error(err, "Unable to update listener pod", "namespace", updatedPod.Namespace, "name", updatedPod.Name) + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } + + case kerrors.IsNotFound(err): + if err := r.publishRunningListener(&autoscalingListener, false); err != nil { // If publish fails, URL is incorrect which means the listener pod would never be able to start return ctrl.Result{}, nil } - // Create a listener pod in the controller namespace - log.Info("Creating a listener pod") - return r.createListenerPod(ctx, &autoscalingRunnerSet, autoscalingListener, serviceAccount, appConfig, log) - } + desiredPod, err := r.newScaleSetListenerPod( + &autoscalingListener, + &listenerConfigSecret, + &serviceAccount, + &listenerRole, + &listenerRoleBinding, + metricsConfig, + ) + if err != nil { + log.Error(err, "Failed to build listener pod") + return ctrl.Result{}, err + } - // If listener config secret changed and pod exists, restart the pod - if secretChanged { - log.Info("Listener config secret changed, restarting listener pod") - return ctrl.Result{}, r.deleteListenerPod(ctx, autoscalingListener, listenerPod, log) + log.Info("Creating listener pod", "namespace", desiredPod.Namespace, "name", desiredPod.Name) + if err := r.Create(ctx, desiredPod); err != nil { + log.Error(err, "Unable to create listener pod", "namespace", desiredPod.Namespace, "name", desiredPod.Name) + return ctrl.Result{}, err + } + default: // error + log.Error(err, "Unable to get listener pod", "namespace", autoscalingListener.Namespace, "name", autoscalingListener.Name) + return ctrl.Result{}, err } - cs := listenerContainerStatus(listenerPod) + cs := listenerContainerStatus(&listenerPod) switch { case listenerPod.Status.Reason == "Evicted": log.Info( @@ -276,7 +539,7 @@ func (r *AutoscalingListenerReconciler) Reconcile(ctx context.Context, req ctrl. "message", listenerPod.Status.Message, ) - return ctrl.Result{}, r.deleteListenerPod(ctx, autoscalingListener, listenerPod, log) + return ctrl.Result{}, r.deleteListenerPod(ctx, &autoscalingListener, &listenerPod, log) case cs == nil: log.Info("Listener pod is not ready", "namespace", listenerPod.Namespace, "name", listenerPod.Name) @@ -290,10 +553,10 @@ func (r *AutoscalingListenerReconciler) Reconcile(ctx context.Context, req ctrl. "message", cs.State.Terminated.Message, ) - return ctrl.Result{}, r.deleteListenerPod(ctx, autoscalingListener, listenerPod, log) + return ctrl.Result{}, r.deleteListenerPod(ctx, &autoscalingListener, &listenerPod, log) case cs.State.Running != nil: - if err := r.publishRunningListener(autoscalingListener, true); err != nil { + if err := r.publishRunningListener(&autoscalingListener, true); err != nil { log.Error(err, "Unable to publish running listener", "namespace", listenerPod.Namespace, "name", listenerPod.Name) // stop reconciling. We should never get to this point but if we do, // listener won't be able to start up, and the crash from the pod should @@ -428,9 +691,8 @@ func (r *AutoscalingListenerReconciler) cleanupResources(ctx context.Context, au } func (r *AutoscalingListenerReconciler) createServiceAccountForListener(ctx context.Context, autoscalingListener *v1alpha1.AutoscalingListener, logger logr.Logger) (ctrl.Result, error) { - newServiceAccount := r.newScaleSetListenerServiceAccount(autoscalingListener) - - if err := ctrl.SetControllerReference(autoscalingListener, newServiceAccount, r.Scheme); err != nil { + newServiceAccount, err := r.newScaleSetListenerServiceAccount(autoscalingListener) + if err != nil { return ctrl.Result{}, err } @@ -444,156 +706,6 @@ func (r *AutoscalingListenerReconciler) createServiceAccountForListener(ctx cont return ctrl.Result{}, nil } -func (r *AutoscalingListenerReconciler) reconcileListenerConfigSecret(ctx context.Context, autoscalingListener *v1alpha1.AutoscalingListener, appConfig *appconfig.AppConfig, metricsConfig *listenerMetricsServerConfig, cert string, logger logr.Logger) (changed bool, err error) { - desiredSecret, err := r.newScaleSetListenerConfig(autoscalingListener, appConfig, metricsConfig, cert) - if err != nil { - return false, fmt.Errorf("failed to build listener config secret: %w", err) - } - - existingSecret := &corev1.Secret{} - err = r.Get(ctx, types.NamespacedName{Name: desiredSecret.Name, Namespace: desiredSecret.Namespace}, existingSecret) - if err != nil { - if kerrors.IsNotFound(err) { - if err := ctrl.SetControllerReference(autoscalingListener, desiredSecret, r.Scheme); err != nil { - return false, fmt.Errorf("failed to set controller reference: %w", err) - } - - logger.Info("Creating listener config secret", "namespace", desiredSecret.Namespace, "name", desiredSecret.Name) - if err := r.Create(ctx, desiredSecret); err != nil { - return false, fmt.Errorf("failed to create listener config secret: %w", err) - } - - return true, nil - } - return false, fmt.Errorf("failed to get listener config secret: %w", err) - } - - if listenerConfigSecretDrifted(existingSecret, desiredSecret) { - updatedSecret := existingSecret.DeepCopy() - updatedSecret.Data = desiredSecret.Data - updatedSecret.Labels = desiredSecret.Labels - updatedSecret.Annotations = desiredSecret.Annotations - - logger.Info("Updating listener config secret", "namespace", updatedSecret.Namespace, "name", updatedSecret.Name) - if err := r.Update(ctx, updatedSecret); err != nil { - return false, fmt.Errorf("failed to update listener config secret: %w", err) - } - - return true, nil - } - - return false, nil -} - -func (r *AutoscalingListenerReconciler) createListenerPod(ctx context.Context, autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet, autoscalingListener *v1alpha1.AutoscalingListener, serviceAccount *corev1.ServiceAccount, appConfig *appconfig.AppConfig, logger logr.Logger) (ctrl.Result, error) { - var envs []corev1.EnvVar - if autoscalingListener.Spec.Proxy != nil { - httpURL := corev1.EnvVar{ - Name: "http_proxy", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: proxyListenerSecretName(autoscalingListener)}, - Key: "http_proxy", - }, - }, - } - if autoscalingListener.Spec.Proxy.HTTP != nil { - envs = append(envs, httpURL) - } - - httpsURL := corev1.EnvVar{ - Name: "https_proxy", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: proxyListenerSecretName(autoscalingListener)}, - Key: "https_proxy", - }, - }, - } - if autoscalingListener.Spec.Proxy.HTTPS != nil { - envs = append(envs, httpsURL) - } - - noProxy := corev1.EnvVar{ - Name: "no_proxy", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: proxyListenerSecretName(autoscalingListener)}, - Key: "no_proxy", - }, - }, - } - if len(autoscalingListener.Spec.Proxy.NoProxy) > 0 { - envs = append(envs, noProxy) - } - } - - cert := "" - if autoscalingListener.Spec.GitHubServerTLS != nil { - var err error - cert, err = r.certificate(ctx, autoscalingRunnerSet, autoscalingListener) - if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to create certificate env var for listener: %w", err) - } - } - - var metricsConfig *listenerMetricsServerConfig - if r.ListenerMetricsAddr != "0" { - metricsConfig = &listenerMetricsServerConfig{ - addr: r.ListenerMetricsAddr, - endpoint: r.ListenerMetricsEndpoint, - } - } - - var podConfig corev1.Secret - if err := r.Get(ctx, types.NamespacedName{Namespace: autoscalingListener.Namespace, Name: scaleSetListenerConfigName(autoscalingListener)}, &podConfig); err != nil { - if !kerrors.IsNotFound(err) { - logger.Error(err, "Unable to get listener config secret", "namespace", autoscalingListener.Namespace, "name", scaleSetListenerConfigName(autoscalingListener)) - return ctrl.Result{Requeue: true}, err - } - - logger.Info("Creating listener config secret") - - podConfig, err := r.newScaleSetListenerConfig(autoscalingListener, appConfig, metricsConfig, cert) - if err != nil { - logger.Error(err, "Failed to build listener config secret") - return ctrl.Result{}, err - } - - if err := ctrl.SetControllerReference(autoscalingListener, podConfig, r.Scheme); err != nil { - logger.Error(err, "Failed to set controller reference") - return ctrl.Result{}, err - } - - if err := r.Create(ctx, podConfig); err != nil { - logger.Error(err, "Unable to create listener config secret", "namespace", podConfig.Namespace, "name", podConfig.Name) - return ctrl.Result{}, err - } - - return ctrl.Result{Requeue: true}, nil - } - - newPod, err := r.newScaleSetListenerPod(autoscalingListener, &podConfig, serviceAccount, metricsConfig, envs...) - if err != nil { - logger.Error(err, "Failed to build listener pod") - return ctrl.Result{}, err - } - - if err := ctrl.SetControllerReference(autoscalingListener, newPod, r.Scheme); err != nil { - logger.Error(err, "Failed to set controller reference") - return ctrl.Result{}, err - } - - logger.Info("Creating listener pod", "namespace", newPod.Namespace, "name", newPod.Name) - if err := r.Create(ctx, newPod); err != nil { - logger.Error(err, "Unable to create listener pod", "namespace", newPod.Namespace, "name", newPod.Name) - return ctrl.Result{}, err - } - - logger.Info("Created listener pod", "namespace", newPod.Namespace, "name", newPod.Name) - return ctrl.Result{}, nil -} - func (r *AutoscalingListenerReconciler) certificate(ctx context.Context, autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet, autoscalingListener *v1alpha1.AutoscalingListener) (string, error) { if autoscalingListener.Spec.GitHubServerTLS.CertificateFrom == nil { return "", fmt.Errorf("githubServerTLS.certificateFrom is not specified") @@ -645,19 +757,9 @@ func (r *AutoscalingListenerReconciler) createProxySecret(ctx context.Context, a return ctrl.Result{}, fmt.Errorf("failed to convert proxy config to secret data: %w", err) } - newProxySecret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: proxyListenerSecretName(autoscalingListener), - Namespace: autoscalingListener.Namespace, - Labels: map[string]string{ - LabelKeyGitHubScaleSetNamespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace, - LabelKeyGitHubScaleSetName: autoscalingListener.Spec.AutoscalingRunnerSetName, - }, - }, - Data: data, - } - if err := ctrl.SetControllerReference(autoscalingListener, newProxySecret, r.Scheme); err != nil { - return ctrl.Result{}, fmt.Errorf("failed to create listener proxy secret: %w", err) + newProxySecret, err := r.newAutoscalingListenerProxySecret(autoscalingListener, data) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to build listener proxy secret: %w", err) } logger.Info("Creating listener proxy secret", "namespace", newProxySecret.Namespace, "name", newProxySecret.Name) @@ -684,21 +786,6 @@ func (r *AutoscalingListenerReconciler) createRoleForListener(ctx context.Contex return ctrl.Result{Requeue: true}, nil } -func (r *AutoscalingListenerReconciler) updateRoleForListener(ctx context.Context, listenerRole *rbacv1.Role, desiredRules []rbacv1.PolicyRule, desiredRulesHash string, logger logr.Logger) (ctrl.Result, error) { - updatedPatchRole := listenerRole.DeepCopy() - updatedPatchRole.Labels["role-policy-rules-hash"] = desiredRulesHash - updatedPatchRole.Rules = desiredRules - - logger.Info("Updating listener role in namespace to have the right permission", "namespace", updatedPatchRole.Namespace, "name", updatedPatchRole.Name, "oldRules", listenerRole.Rules, "newRules", updatedPatchRole.Rules) - if err := r.Update(ctx, updatedPatchRole); err != nil { - logger.Error(err, "Unable to update listener role", "namespace", updatedPatchRole.Namespace, "name", updatedPatchRole.Name, "rules", updatedPatchRole.Rules) - return ctrl.Result{}, err - } - - logger.Info("Updated listener role in namespace to have the right permission", "namespace", updatedPatchRole.Namespace, "name", updatedPatchRole.Name, "rules", updatedPatchRole.Rules) - return ctrl.Result{Requeue: true}, nil -} - func (r *AutoscalingListenerReconciler) createRoleBindingForListener(ctx context.Context, autoscalingListener *v1alpha1.AutoscalingListener, listenerRole *rbacv1.Role, serviceAccount *corev1.ServiceAccount, logger logr.Logger) (ctrl.Result, error) { newRoleBinding := r.newScaleSetListenerRoleBinding(autoscalingListener, listenerRole, serviceAccount) @@ -728,7 +815,7 @@ func (r *AutoscalingListenerReconciler) createRoleBindingForListener(ctx context } func (r *AutoscalingListenerReconciler) publishRunningListener(autoscalingListener *v1alpha1.AutoscalingListener, isUp bool) error { - githubConfigURL := autoscalingListener.Spec.GitHubConfigUrl + githubConfigURL := autoscalingListener.Spec.GitHubConfigURL parsedURL, err := actions.ParseGitHubConfigFromURL(githubConfigURL) if err != nil { return err @@ -751,29 +838,10 @@ func (r *AutoscalingListenerReconciler) publishRunningListener(autoscalingListen return nil } -// listenerConfigSecretDrifted detects if the listener config secret has drifted from the desired state. -// It compares the config.json data, labels, and annotations deterministically. -func listenerConfigSecretDrifted(existing *corev1.Secret, desired *corev1.Secret) bool { - // Compare config.json data - if !bytes.Equal(existing.Data["config.json"], desired.Data["config.json"]) { - return true - } - - // Compare labels - if !maps.Equal(existing.Labels, desired.Labels) { - return true - } - - // Compare annotations - if !maps.Equal(existing.Annotations, desired.Annotations) { - return true - } - - return false -} - // SetupWithManager sets up the controller with the Manager. func (r *AutoscalingListenerReconciler) SetupWithManager(mgr ctrl.Manager, opts ...Option) error { + r.setSchemeIfUnset(r.Scheme) + labelBasedWatchFunc := func(_ context.Context, obj client.Object) []reconcile.Request { var requests []reconcile.Request labels := obj.GetLabels() @@ -786,7 +854,8 @@ func (r *AutoscalingListenerReconciler) SetupWithManager(mgr ctrl.Manager, opts if !ok { return nil } - requests = append(requests, + requests = append( + requests, reconcile.Request{ NamespacedName: types.NamespacedName{ Name: name, diff --git a/controllers/actions.github.com/autoscalinglistener_controller_test.go b/controllers/actions.github.com/autoscalinglistener_controller_test.go index b9a6c2b06f..d48d613e6c 100644 --- a/controllers/actions.github.com/autoscalinglistener_controller_test.go +++ b/controllers/actions.github.com/autoscalinglistener_controller_test.go @@ -94,17 +94,43 @@ var _ = Describe("Test AutoScalingListener controller", func() { ObjectMeta: metav1.ObjectMeta{ Name: "test-asl", Namespace: autoscalingNS.Name, + Labels: map[string]string{ + "arc.test/listener-label": "initial", + }, }, Spec: v1alpha1.AutoscalingListenerSpec{ - GitHubConfigUrl: "https://github.com/owner/repo", + GitHubConfigURL: "https://github.com/owner/repo", GitHubConfigSecret: configSecret.Name, - RunnerScaleSetId: 1, + RunnerScaleSetID: 1, AutoscalingRunnerSetNamespace: autoscalingRunnerSet.Namespace, AutoscalingRunnerSetName: autoscalingRunnerSet.Name, EphemeralRunnerSetName: "test-ers", MaxRunners: 10, MinRunners: 1, Image: "ghcr.io/owner/repo", + ServiceAccountMetadata: &v1alpha1.ResourceMeta{ + Annotations: map[string]string{ + "arc.test/service-account-annotation": "initial", + }, + }, + RoleMetadata: &v1alpha1.ResourceMeta{ + Annotations: map[string]string{ + "arc.test/role-annotation": "initial", + }, + }, + RoleBindingMetadata: &v1alpha1.ResourceMeta{ + Annotations: map[string]string{ + "arc.test/role-binding-annotation": "initial", + }, + }, + ConfigSecretMetadata: &v1alpha1.ResourceMeta{ + Labels: map[string]string{ + "arc.test/config-secret-label": "initial", + }, + Annotations: map[string]string{ + "arc.test/config-secret-annotation": "initial", + }, + }, }, } @@ -143,7 +169,8 @@ var _ = Describe("Test AutoScalingListener controller", func() { return created.Finalizers[0], nil }, autoscalingListenerTestTimeout, - autoscalingListenerTestInterval).Should(BeEquivalentTo(autoscalingListenerFinalizerName), "AutoScalingListener should have a finalizer") + autoscalingListenerTestInterval, + ).Should(BeEquivalentTo(autoscalingListenerFinalizerName), "AutoScalingListener should have a finalizer") // Check if service account is created serviceAccount := new(corev1.ServiceAccount) @@ -171,7 +198,8 @@ var _ = Describe("Test AutoScalingListener controller", func() { return role.Rules, nil }, autoscalingListenerTestTimeout, - autoscalingListenerTestInterval).Should(BeEquivalentTo(rulesForListenerRole([]string{autoscalingListener.Spec.EphemeralRunnerSetName})), "Role should be created") + autoscalingListenerTestInterval, + ).Should(BeEquivalentTo(rulesForListenerRole([]string{autoscalingListener.Spec.EphemeralRunnerSetName})), "Role should be created") // Check if rolebinding is created roleBinding := new(rbacv1.RoleBinding) @@ -185,7 +213,8 @@ var _ = Describe("Test AutoScalingListener controller", func() { return roleBinding.RoleRef.Name, nil }, autoscalingListenerTestTimeout, - autoscalingListenerTestInterval).Should(BeEquivalentTo(autoscalingListener.Name), "Rolebinding should be created") + autoscalingListenerTestInterval, + ).Should(BeEquivalentTo(autoscalingListener.Name), "Rolebinding should be created") // Check if pod is created pod := new(corev1.Pod) @@ -199,7 +228,8 @@ var _ = Describe("Test AutoScalingListener controller", func() { return pod.Name, nil }, autoscalingListenerTestTimeout, - autoscalingListenerTestInterval).Should(BeEquivalentTo(autoscalingListener.Name), "Pod should be created") + autoscalingListenerTestInterval, + ).Should(BeEquivalentTo(autoscalingListener.Name), "Pod should be created") }) }) @@ -292,7 +322,8 @@ var _ = Describe("Test AutoScalingListener controller", func() { return nil }, autoscalingListenerTestTimeout, - autoscalingListenerTestInterval).ShouldNot(Succeed(), "failed to delete service account") + autoscalingListenerTestInterval, + ).ShouldNot(Succeed(), "failed to delete service account") // The AutoScalingListener should be deleted Eventually( @@ -309,7 +340,8 @@ var _ = Describe("Test AutoScalingListener controller", func() { return nil }, autoscalingListenerTestTimeout, - autoscalingListenerTestInterval).ShouldNot(Succeed(), "failed to delete AutoScalingListener") + autoscalingListenerTestInterval, + ).ShouldNot(Succeed(), "failed to delete AutoScalingListener") }) }) @@ -327,7 +359,8 @@ var _ = Describe("Test AutoScalingListener controller", func() { return pod.Name, nil }, autoscalingListenerTestTimeout, - autoscalingListenerTestInterval).Should(BeEquivalentTo(autoscalingListener.Name), "Pod should be created") + autoscalingListenerTestInterval, + ).Should(BeEquivalentTo(autoscalingListener.Name), "Pod should be created") // Update the AutoScalingListener updated := autoscalingListener.DeepCopy() @@ -347,7 +380,101 @@ var _ = Describe("Test AutoScalingListener controller", func() { return role.Rules, nil }, autoscalingListenerTestTimeout, - autoscalingListenerTestInterval).Should(BeEquivalentTo(rulesForListenerRole([]string{updated.Spec.EphemeralRunnerSetName})), "Role should be updated") + autoscalingListenerTestInterval, + ).Should(BeEquivalentTo(rulesForListenerRole([]string{updated.Spec.EphemeralRunnerSetName})), "Role should be updated") + }) + + It("propagates updated listener metadata to owned resources", func() { + assertPropagatedMetadata := func(expected string) { + Eventually( + func(g Gomega) { + serviceAccount := new(corev1.ServiceAccount) + err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingListener.Name, Namespace: autoscalingListener.Namespace}, serviceAccount) + g.Expect(err).NotTo(HaveOccurred(), "failed to get ServiceAccount") + g.Expect(serviceAccount.Labels["arc.test/listener-label"]).To(Equal(expected)) + g.Expect(serviceAccount.Annotations["arc.test/service-account-annotation"]).To(Equal(expected)) + if expected == "updated" { + g.Expect(serviceAccount.Annotations["arc.test/new-service-account-annotation"]).To(Equal("added")) + } + + role := new(rbacv1.Role) + err = k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingListener.Name, Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace}, role) + g.Expect(err).NotTo(HaveOccurred(), "failed to get Role") + g.Expect(role.Labels["arc.test/listener-label"]).To(Equal(expected)) + g.Expect(role.Annotations["arc.test/role-annotation"]).To(Equal(expected)) + if expected == "updated" { + g.Expect(role.Annotations["arc.test/new-role-annotation"]).To(Equal("added")) + } + + roleBinding := new(rbacv1.RoleBinding) + err = k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingListener.Name, Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace}, roleBinding) + g.Expect(err).NotTo(HaveOccurred(), "failed to get RoleBinding") + g.Expect(roleBinding.Labels["arc.test/listener-label"]).To(Equal(expected)) + g.Expect(roleBinding.Annotations["arc.test/role-binding-annotation"]).To(Equal(expected)) + if expected == "updated" { + g.Expect(roleBinding.Annotations["arc.test/new-role-binding-annotation"]).To(Equal("added")) + } + + secret := new(corev1.Secret) + err = k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerConfigName(autoscalingListener), Namespace: autoscalingListener.Namespace}, secret) + g.Expect(err).NotTo(HaveOccurred(), "failed to get config Secret") + g.Expect(secret.Labels["arc.test/config-secret-label"]).To(Equal(expected)) + g.Expect(secret.Annotations["arc.test/config-secret-annotation"]).To(Equal(expected)) + if expected == "updated" { + g.Expect(secret.Annotations["arc.test/new-config-secret-annotation"]).To(Equal("added")) + } + + pod := new(corev1.Pod) + err = k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingListener.Name, Namespace: autoscalingListener.Namespace}, pod) + g.Expect(err).NotTo(HaveOccurred(), "failed to get Pod") + g.Expect(pod.Labels["arc.test/listener-label"]).To(Equal(expected)) + }, + autoscalingListenerTestTimeout, + autoscalingListenerTestInterval, + ).Should(Succeed()) + } + + assertPropagatedMetadata("initial") + + current := new(v1alpha1.AutoscalingListener) + err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingListener.Name, Namespace: autoscalingListener.Namespace}, current) + Expect(err).NotTo(HaveOccurred(), "failed to get AutoScalingListener") + + updated := current.DeepCopy() + updated.Labels = map[string]string{ + "arc.test/listener-label": "updated", + } + updated.Spec.ServiceAccountMetadata = &v1alpha1.ResourceMeta{ + Annotations: map[string]string{ + "arc.test/service-account-annotation": "updated", + "arc.test/new-service-account-annotation": "added", + }, + } + updated.Spec.RoleMetadata = &v1alpha1.ResourceMeta{ + Annotations: map[string]string{ + "arc.test/role-annotation": "updated", + "arc.test/new-role-annotation": "added", + }, + } + updated.Spec.RoleBindingMetadata = &v1alpha1.ResourceMeta{ + Annotations: map[string]string{ + "arc.test/role-binding-annotation": "updated", + "arc.test/new-role-binding-annotation": "added", + }, + } + updated.Spec.ConfigSecretMetadata = &v1alpha1.ResourceMeta{ + Labels: map[string]string{ + "arc.test/config-secret-label": "updated", + }, + Annotations: map[string]string{ + "arc.test/config-secret-annotation": "updated", + "arc.test/new-config-secret-annotation": "added", + }, + } + err = k8sClient.Patch(ctx, updated, client.MergeFrom(current)) + Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingListener metadata") + + assertPropagatedMetadata("updated") }) It("It should re-create pod but persist config secret whenever listener container is terminated", func() { @@ -512,9 +639,9 @@ var _ = Describe("Test AutoScalingListener customization", func() { Namespace: autoscalingNS.Name, }, Spec: v1alpha1.AutoscalingListenerSpec{ - GitHubConfigUrl: "https://github.com/owner/repo", + GitHubConfigURL: "https://github.com/owner/repo", GitHubConfigSecret: configSecret.Name, - RunnerScaleSetId: 1, + RunnerScaleSetID: 1, AutoscalingRunnerSetNamespace: autoscalingRunnerSet.Namespace, AutoscalingRunnerSetName: autoscalingRunnerSet.Name, EphemeralRunnerSetName: "test-ers", @@ -771,9 +898,9 @@ var _ = Describe("Test AutoScalingListener controller with proxy", func() { Namespace: autoscalingNS.Name, }, Spec: v1alpha1.AutoscalingListenerSpec{ - GitHubConfigUrl: "https://github.com/owner/repo", + GitHubConfigURL: "https://github.com/owner/repo", GitHubConfigSecret: configSecret.Name, - RunnerScaleSetId: 1, + RunnerScaleSetID: 1, AutoscalingRunnerSetNamespace: autoscalingRunnerSet.Namespace, AutoscalingRunnerSetName: autoscalingRunnerSet.Name, EphemeralRunnerSetName: "test-ers", @@ -908,7 +1035,8 @@ var _ = Describe("Test AutoScalingListener controller with proxy", func() { }), "no_proxy environment variable not found") }, autoscalingListenerTestTimeout, - autoscalingListenerTestInterval).Should(Succeed(), "failed to create listener pod with proxy details") + autoscalingListenerTestInterval, + ).Should(Succeed(), "failed to create listener pod with proxy details") // Delete the AutoScalingListener err = k8sClient.Delete(ctx, autoscalingListener) @@ -925,7 +1053,8 @@ var _ = Describe("Test AutoScalingListener controller with proxy", func() { g.Expect(kerrors.IsNotFound(err)).To(BeTrue()) }, autoscalingListenerTestTimeout, - autoscalingListenerTestInterval).Should(Succeed(), "failed to delete secret with proxy details") + autoscalingListenerTestInterval, + ).Should(Succeed(), "failed to delete secret with proxy details") }) }) @@ -973,9 +1102,9 @@ var _ = Describe("Test AutoScalingListener controller with template modification Namespace: autoscalingNS.Name, }, Spec: v1alpha1.AutoscalingListenerSpec{ - GitHubConfigUrl: "https://github.com/owner/repo", + GitHubConfigURL: "https://github.com/owner/repo", GitHubConfigSecret: configSecret.Name, - RunnerScaleSetId: 1, + RunnerScaleSetID: 1, AutoscalingRunnerSetNamespace: autoscalingRunnerSet.Namespace, AutoscalingRunnerSetName: autoscalingRunnerSet.Name, EphemeralRunnerSetName: "test-ers", @@ -1063,7 +1192,8 @@ var _ = Describe("Test AutoScalingListener controller with template modification g.Expect(pod.ObjectMeta.Labels).To(HaveKeyWithValue("test-label-key", "test-label-value"), "pod labels should be copied from runner set template") }, autoscalingListenerTestTimeout, - autoscalingListenerTestInterval).Should(Succeed(), "failed to create listener pod with proxy details") + autoscalingListenerTestInterval, + ).Should(Succeed(), "failed to create listener pod with proxy details") // Delete the AutoScalingListener err := k8sClient.Delete(ctx, autoscalingListener) @@ -1080,7 +1210,8 @@ var _ = Describe("Test AutoScalingListener controller with template modification g.Expect(kerrors.IsNotFound(err)).To(BeTrue()) }, autoscalingListenerTestTimeout, - autoscalingListenerTestInterval).Should(Succeed(), "failed to delete secret with proxy details") + autoscalingListenerTestInterval, + ).Should(Succeed(), "failed to delete secret with proxy details") }) }) @@ -1177,7 +1308,7 @@ var _ = Describe("Test GitHub Server TLS configuration", func() { Namespace: autoscalingNS.Name, }, Spec: v1alpha1.AutoscalingListenerSpec{ - GitHubConfigUrl: "https://github.com/owner/repo", + GitHubConfigURL: "https://github.com/owner/repo", GitHubConfigSecret: configSecret.Name, GitHubServerTLS: &v1alpha1.TLSConfig{ CertificateFrom: &v1alpha1.TLSCertificateSource{ @@ -1189,7 +1320,7 @@ var _ = Describe("Test GitHub Server TLS configuration", func() { }, }, }, - RunnerScaleSetId: 1, + RunnerScaleSetID: 1, AutoscalingRunnerSetNamespace: autoscalingRunnerSet.Namespace, AutoscalingRunnerSetName: autoscalingRunnerSet.Name, EphemeralRunnerSetName: "test-ers", @@ -1240,7 +1371,8 @@ var _ = Describe("Test GitHub Server TLS configuration", func() { BeEquivalentTo(string(cert)), "GITHUB_SERVER_ROOT_CA should be the rootCA.crt", ) - }). + }, + ). WithTimeout(autoscalingRunnerSetTestTimeout). WithPolling(autoscalingListenerTestInterval). Should(Succeed(), "failed to create pod with volume and env variable") diff --git a/controllers/actions.github.com/autoscalingrunnerset_controller.go b/controllers/actions.github.com/autoscalingrunnerset_controller.go index 228465ecf6..706cec1caa 100644 --- a/controllers/actions.github.com/autoscalingrunnerset_controller.go +++ b/controllers/actions.github.com/autoscalingrunnerset_controller.go @@ -19,7 +19,7 @@ package actionsgithubcom import ( "context" "fmt" - "sort" + "maps" "strconv" "strings" "time" @@ -28,6 +28,7 @@ import ( "github.com/actions/actions-runner-controller/build" "github.com/actions/scaleset" "github.com/go-logr/logr" + "github.com/google/go-cmp/cmp" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" @@ -42,35 +43,10 @@ import ( ) const ( - annotationKeyRunnerSpecHash = "actions.github.com/runner-spec-hash" - // annotationKeyValuesHash is hash of the entire values json. - // This is used to determine if the values have changed, so we can - // re-create listener. - annotationKeyValuesHash = "actions.github.com/values-hash" - annotationKeyChangeHash = "actions.github.com/change-hash" - autoscalingRunnerSetFinalizerName = "autoscalingrunnerset.actions.github.com/finalizer" runnerScaleSetIDAnnotationKey = "runner-scale-set-id" ) -type UpdateStrategy string - -// Defines how the controller should handle upgrades while having running jobs. -const ( - // "immediate": (default) The controller will immediately apply the change causing the - // recreation of the listener and ephemeral runner set. This can lead to an - // overprovisioning of runners, if there are pending / running jobs. This should not - // be a problem at a small scale, but it could lead to a significant increase of - // resources if you have a lot of jobs running concurrently. - UpdateStrategyImmediate = UpdateStrategy("immediate") - // "eventual": The controller will remove the listener and ephemeral runner set - // immediately, but will not recreate them (to apply changes) until all - // pending / running jobs have completed. - // This can lead to a longer time to apply the change but it will ensure - // that you don't have any overprovisioning of runners. - UpdateStrategyEventual = UpdateStrategy("eventual") -) - // AutoscalingRunnerSetReconciler reconciles a AutoscalingRunnerSet object type AutoscalingRunnerSetReconciler struct { client.Client @@ -79,7 +55,6 @@ type AutoscalingRunnerSetReconciler struct { ControllerNamespace string DefaultRunnerScaleSetListenerImage string DefaultRunnerScaleSetListenerImagePullSecrets []string - UpdateStrategy UpdateStrategy ResourceBuilder } @@ -95,18 +70,19 @@ type AutoscalingRunnerSetReconciler struct { func (r *AutoscalingRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.Log.WithValues("autoscalingrunnerset", req.NamespacedName) - autoscalingRunnerSet := new(v1alpha1.AutoscalingRunnerSet) - if err := r.Get(ctx, req.NamespacedName, autoscalingRunnerSet); err != nil { + var autoscalingRunnerSet v1alpha1.AutoscalingRunnerSet + if err := r.Get(ctx, req.NamespacedName, &autoscalingRunnerSet); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } + original := autoscalingRunnerSet.DeepCopy() if !autoscalingRunnerSet.DeletionTimestamp.IsZero() { - if !controllerutil.ContainsFinalizer(autoscalingRunnerSet, autoscalingRunnerSetFinalizerName) { + if !controllerutil.ContainsFinalizer(&autoscalingRunnerSet, autoscalingRunnerSetFinalizerName) { return ctrl.Result{}, nil } log.Info("Deleting resources") - done, err := r.cleanUpResources(ctx, autoscalingRunnerSet, nil, log) + done, err := r.cleanUpResources(ctx, &autoscalingRunnerSet, log) if err != nil { log.Error(err, "Failed to clean up resources during deletion") return ctrl.Result{}, err @@ -118,18 +94,17 @@ func (r *AutoscalingRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl }, nil } - if err := r.removeFinalizersFromDependentResources(ctx, autoscalingRunnerSet, log); err != nil { + if err := r.removeFinalizersFromDependentResources(ctx, &autoscalingRunnerSet, log); err != nil { log.Error(err, "Failed to remove finalizers on dependent resources") return ctrl.Result{}, err } - log.Info("Removing finalizer") - err = patch(ctx, r.Client, autoscalingRunnerSet, func(obj *v1alpha1.AutoscalingRunnerSet) { - controllerutil.RemoveFinalizer(obj, autoscalingRunnerSetFinalizerName) - }) - if err != nil && !kerrors.IsNotFound(err) { - log.Error(err, "Failed to update autoscaling runner set without finalizer") - return ctrl.Result{}, err + if controllerutil.RemoveFinalizer(&autoscalingRunnerSet, autoscalingRunnerSetFinalizerName) { + log.Info("Removing finalizer") + if err := r.Patch(ctx, &autoscalingRunnerSet, client.MergeFrom(original)); err != nil && !kerrors.IsNotFound(err) { + log.Error(err, "Failed to update autoscaling runner set without finalizer") + return ctrl.Result{}, err + } } log.Info("Successfully removed finalizer after cleanup") @@ -137,7 +112,7 @@ func (r *AutoscalingRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl } if !v1alpha1.IsVersionAllowed(autoscalingRunnerSet.Labels[LabelKeyKubernetesVersion], build.Version) { - if err := r.Delete(ctx, autoscalingRunnerSet); err != nil { + if err := r.Delete(ctx, &autoscalingRunnerSet); err != nil { log.Error( err, "Failed to delete autoscaling runner set on version mismatch", "buildVersion", build.Version, @@ -154,178 +129,265 @@ func (r *AutoscalingRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl return ctrl.Result{}, nil } - if !controllerutil.ContainsFinalizer(autoscalingRunnerSet, autoscalingRunnerSetFinalizerName) { + if controllerutil.AddFinalizer(&autoscalingRunnerSet, autoscalingRunnerSetFinalizerName) { log.Info("Adding finalizer") - if err := patch(ctx, r.Client, autoscalingRunnerSet, func(obj *v1alpha1.AutoscalingRunnerSet) { - controllerutil.AddFinalizer(obj, autoscalingRunnerSetFinalizerName) - }); err != nil { - log.Error(err, "Failed to update autoscaling runner set with finalizer added") + + if err := r.Patch(ctx, &autoscalingRunnerSet, client.MergeFrom(original)); err != nil { + log.Error(err, "Failed to update autoscaling runner set with finalizer") return ctrl.Result{}, err } log.Info("Successfully added finalizer") + return ctrl.Result{}, nil } - if targetHash := autoscalingRunnerSet.Hash(); autoscalingRunnerSet.Annotations[annotationKeyChangeHash] != targetHash { - if err := patch(ctx, r.Client, autoscalingRunnerSet, func(obj *v1alpha1.AutoscalingRunnerSet) { - if obj.Annotations == nil { - obj.Annotations = map[string]string{} - } - obj.Annotations[annotationKeyChangeHash] = targetHash - }); err != nil { - log.Error(err, "Failed to update autoscaling runner set with change hash annotation") + // Something has changed, we need to re-apply the pending phase and change hash annotation to trigger the update of runner scale set and listener. + if targetHash := autoscalingRunnerSet.Hash(); autoscalingRunnerSet.Annotations[annotationKeyIntegrityHash] != targetHash { + // TODO: apply the version label + original := autoscalingRunnerSet.DeepCopy() + if autoscalingRunnerSet.Annotations == nil { + autoscalingRunnerSet.Annotations = map[string]string{} + } + autoscalingRunnerSet.Annotations[annotationKeyIntegrityHash] = targetHash + autoscalingRunnerSet.Status.Phase = v1alpha1.AutoscalingRunnerSetPhasePending + if err := r.Patch(ctx, &autoscalingRunnerSet, client.MergeFrom(original)); err != nil { + log.Error(err, "Failed to update autoscaling runner set with new change hash and pending phase") return ctrl.Result{}, err } - - if err := r.updateStatus(ctx, autoscalingRunnerSet, nil, v1alpha1.AutoscalingRunnerSetPhasePending, log); err != nil { - log.Error(err, "Failed to update autoscaling runner set status to pending") + if err := r.Status().Patch(ctx, &autoscalingRunnerSet, client.MergeFrom(original)); err != nil { + log.Error(err, "Failed to update autoscaling runner set status with pending phase") return ctrl.Result{}, err } + + return ctrl.Result{}, nil } outdated := autoscalingRunnerSet.Status.Phase == v1alpha1.AutoscalingRunnerSetPhaseOutdated - - if !outdated { - scaleSetIDRaw, ok := autoscalingRunnerSet.Annotations[runnerScaleSetIDAnnotationKey] - if !ok { - // Need to create a new runner scale set on Actions service - log.Info("Runner scale set id annotation does not exist. Creating a new runner scale set.") - return r.createRunnerScaleSet(ctx, autoscalingRunnerSet, log) + if outdated { + log.Info("Autoscaling runner set is in outdated phase, removing the listener") + done, err := r.cleanupListener(ctx, &autoscalingRunnerSet, log) + if err != nil { + log.Error(err, "Failed to clean up listener") + return ctrl.Result{}, err } - - if id, err := strconv.Atoi(scaleSetIDRaw); err != nil || id <= 0 { - log.Info("Runner scale set id annotation is not an id, or is <= 0. Creating a new runner scale set.") - // something modified the scaleSetId. Try to create one - return r.createRunnerScaleSet(ctx, autoscalingRunnerSet, log) + if !done { + log.Info("Waiting for listener to be cleaned up for the outdated runner set") + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil } - // Make sure the runner group of the scale set is up to date - currentRunnerGroupName, ok := autoscalingRunnerSet.Annotations[AnnotationKeyGitHubRunnerGroupName] - if !ok || (len(autoscalingRunnerSet.Spec.RunnerGroup) > 0 && !strings.EqualFold(currentRunnerGroupName, autoscalingRunnerSet.Spec.RunnerGroup)) { - log.Info("AutoScalingRunnerSet runner group changed. Updating the runner scale set.") - return r.updateRunnerScaleSetRunnerGroup(ctx, autoscalingRunnerSet, log) - } + var ephemeralRunnerSet v1alpha1.EphemeralRunnerSet + err = r.Get( + ctx, + types.NamespacedName{ + Namespace: autoscalingRunnerSet.Namespace, + Name: autoscalingRunnerSet.Name, + }, + &ephemeralRunnerSet, + ) + switch { + case kerrors.IsNotFound(err): + // If the ephemeral runner set is not found, something removed the ephemeral runner set. The ephemeral runner set should + // not be removed by the controller once it is outdated. However, if the ephemeral runner set is removed, it means no ephemeral + // runners should be running (or at least no ephemeral runners associated with the ephemeral runner set). + // Therefore, this state is acceptable, because the update to the autoscaling runner set will trigger the loop + // that will eventually create a new ephemeral runner set. + log.Info("Ephemeral runner set is not found. Ignoring the state until the autoscaling runner set is updated") + return ctrl.Result{}, nil + case err != nil: + log.Error(err, "Failed to get ephemeral runner set for the outdated runner set") + return ctrl.Result{}, err + default: + if !ephemeralRunnerSet.DeletionTimestamp.IsZero() { + // Same as NotFound case, ignore. + return ctrl.Result{}, nil + } - // Make sure the runner scale set name is up to date - currentRunnerScaleSetName, ok := autoscalingRunnerSet.Annotations[AnnotationKeyGitHubRunnerScaleSetName] - if !ok || (len(autoscalingRunnerSet.Spec.RunnerScaleSetName) > 0 && !strings.EqualFold(currentRunnerScaleSetName, autoscalingRunnerSet.Spec.RunnerScaleSetName)) { - log.Info("AutoScalingRunnerSet runner scale set name changed. Updating the runner scale set.") - return r.updateRunnerScaleSetName(ctx, autoscalingRunnerSet, log) + original := ephemeralRunnerSet.DeepCopy() + ephemeralRunnerSet.Spec.Replicas = 0 + ephemeralRunnerSet.Spec.PatchID = 0 + if err := r.Patch(ctx, &ephemeralRunnerSet, client.MergeFrom(original)); err != nil { + log.Error(err, "Failed to patch ephemeral runner set with 0 replicas and reset patch ID for the outdated runner set") + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil } } - existingRunnerSets, err := r.listEphemeralRunnerSets(ctx, autoscalingRunnerSet) - if err != nil { - log.Error(err, "Failed to list existing ephemeral runner sets") - return ctrl.Result{}, err + if shouldCreateScaleSet(&autoscalingRunnerSet) { + log.Info("Creating runner scale set") + return r.createRunnerScaleSet(ctx, &autoscalingRunnerSet, log) } - latestRunnerSet := existingRunnerSets.latest() - if latestRunnerSet == nil && !outdated { - log.Info("Latest runner set does not exist. Creating a new runner set.") - return r.createEphemeralRunnerSet(ctx, autoscalingRunnerSet, log) + // Make sure the runner group of the scale set is up to date + currentRunnerGroupName, ok := autoscalingRunnerSet.Annotations[AnnotationKeyGitHubRunnerGroupName] + if !ok || (len(autoscalingRunnerSet.Spec.RunnerGroup) > 0 && !strings.EqualFold(currentRunnerGroupName, autoscalingRunnerSet.Spec.RunnerGroup)) { + log.Info("AutoScalingRunnerSet runner group changed. Updating the runner scale set.") + return r.updateRunnerScaleSetRunnerGroup(ctx, &autoscalingRunnerSet, log) } - for _, runnerSet := range existingRunnerSets.all() { - log.Info("Find existing ephemeral runner set", "name", runnerSet.Name, "specHash", runnerSet.Annotations[annotationKeyRunnerSpecHash]) + // Make sure the runner scale set name is up to date + currentRunnerScaleSetName, ok := autoscalingRunnerSet.Annotations[AnnotationKeyGitHubRunnerScaleSetName] + if !ok || (len(autoscalingRunnerSet.Spec.RunnerScaleSetName) > 0 && !strings.EqualFold(currentRunnerScaleSetName, autoscalingRunnerSet.Spec.RunnerScaleSetName)) { + log.Info("AutoScalingRunnerSet runner scale set name changed. Updating the runner scale set.") + return r.updateRunnerScaleSetName(ctx, &autoscalingRunnerSet, log) } - outdated = outdated || (latestRunnerSet != nil && latestRunnerSet.Status.Phase == v1alpha1.EphemeralRunnerSetPhaseOutdated) - - // Make sure the AutoscalingListener is up and running in the controller namespace - listener := new(v1alpha1.AutoscalingListener) - listenerFound := true - if err := r.Get(ctx, client.ObjectKey{Namespace: r.ControllerNamespace, Name: scaleSetListenerName(autoscalingRunnerSet)}, listener); err != nil { - if !kerrors.IsNotFound(err) { - log.Error(err, "Failed to get AutoscalingListener resource") + var ephemeralRunnerSet v1alpha1.EphemeralRunnerSet + err := r.Get( + ctx, + types.NamespacedName{ + Namespace: autoscalingRunnerSet.Namespace, + Name: autoscalingRunnerSet.Name, + }, + &ephemeralRunnerSet, + ) + switch { + case kerrors.IsNotFound(err): + log.Info("Creating ephemeral runner set") + return r.createEphemeralRunnerSet(ctx, &autoscalingRunnerSet, log) + case err != nil: + log.Error(err, "Failed to get ephemeral runner") + return ctrl.Result{}, err + case ephemeralRunnerSet.Status.Phase == v1alpha1.EphemeralRunnerSetPhaseOutdated && autoscalingRunnerSet.Status.Phase == v1alpha1.AutoscalingRunnerSetPhaseRunning: + // Runners are outdated. We need to stop the listener so it stops getting new jobs. + log.Info("Ephemeral runner set is outdated. Cleaning up resources for the outdated runner set") + done, err := r.cleanupListener(ctx, &autoscalingRunnerSet, log) + if err != nil { + log.Error(err, "Failed to clean up listener for outdated ephemeral runner set") return ctrl.Result{}, err } - - listenerFound = false - log.Info("AutoscalingListener does not exist.") - } - - if outdated { - log.Info("Ephemeral runner set is outdated") - if autoscalingRunnerSet.Status.Phase != v1alpha1.AutoscalingRunnerSetPhaseOutdated { - if err := r.updateStatus(ctx, autoscalingRunnerSet, latestRunnerSet, v1alpha1.AutoscalingRunnerSetPhaseOutdated, log); err != nil { - log.Error(err, "Failed to update autoscaling runner set status to outdated") - return ctrl.Result{}, err - } + if !done { + log.Info("Waiting for listener to be cleaned up for the outdated ephemeral runner set") + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil } - done, err := r.cleanUpResources(ctx, autoscalingRunnerSet, latestRunnerSet, log) + // Then, we need to remove the ephemeral runner set to force scale-down. The ephemeral runner set + // will eventually remove all runners as soon as possible. + // + // The scale set should not be removed yet, since user did not explicitly remove the scale set (or the autoscaling runner set) + // Therefore, the autoscaling runner set should stay in outdated state until the spec is updated, + // or until the autoscaling runner set is removed. + done, err = r.cleanupEphemeralRunnerSet(ctx, &autoscalingRunnerSet, log) if err != nil { - log.Error(err, "Failed to clean up resources for outdated runner set") + log.Error(err, "Failed to clean up ephemeral runner set for outdated runner set") return ctrl.Result{}, err } - if done { - return ctrl.Result{}, nil + if !done { + log.Info("Waiting for ephemeral runner set to be cleaned up for the outdated runner set") + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil } - log.Info("Waiting for resources to be cleaned up for outdated runner set") - return ctrl.Result{ - RequeueAfter: 5 * time.Second, - }, nil - } - - // Our listener pod is out of date, so we need to delete it to get a new recreate. - listenerValuesHashChanged := listener.Annotations[annotationKeyValuesHash] != autoscalingRunnerSet.Annotations[annotationKeyValuesHash] - listenerSpecHashChanged := listener.Annotations[annotationKeyRunnerSpecHash] != autoscalingRunnerSet.ListenerSpecHash() - if listenerFound && (listenerValuesHashChanged || - listenerSpecHashChanged || - latestRunnerSet == nil || - listener.Spec.EphemeralRunnerSetName != latestRunnerSet.Name) { - log.Info("RunnerScaleSetListener is out of date. Deleting it so that it is recreated", "name", listener.Name) - if err := r.Delete(ctx, listener); err != nil { - log.Error(err, "Failed to delete AutoscalingListener resource") - return ctrl.Result{}, err - } + log.Info("Successfully cleaned up resources for the outdated runner set") - log.Info("Deleted RunnerScaleSetListener since existing one is out of date") return ctrl.Result{}, nil - } - if latestRunnerSet.Annotations[annotationKeyRunnerSpecHash] != autoscalingRunnerSet.RunnerSetSpecHash() { - if r.drainingJobs(&latestRunnerSet.Status) { - log.Info("Latest runner set spec hash does not match the current autoscaling runner set. Waiting for the running and pending runners to finish:", "running", latestRunnerSet.Status.RunningEphemeralRunners, "pending", latestRunnerSet.Status.PendingEphemeralRunners) - log.Info("Scaling down the number of desired replicas to 0") - // We are in the process of draining the jobs. The listener has been deleted and the ephemeral runner set replicas - // need to scale down to 0 - err := patch(ctx, r.Client, latestRunnerSet, func(obj *v1alpha1.EphemeralRunnerSet) { - obj.Spec.Replicas = 0 - obj.Spec.PatchID = 0 - }) - if err != nil { - log.Error(err, "Failed to patch runner set to set desired count to 0") + default: + desired, err := r.newEphemeralRunnerSet(&autoscalingRunnerSet) + if err != nil { + log.Error(err, "Failed to generate ephemeral runner set spec") + return ctrl.Result{}, nil + } + + if ephemeralRunnerSet.Annotations[annotationKeyIntegrityHash] != desired.Annotations[annotationKeyIntegrityHash] { + // When runners are actively processing jobs, defer the spec update: + // delete the listener to stop accepting new jobs, but leave the ERS + // (and its running pods) untouched until all jobs have drained. + if ephemeralRunnerSet.Status.RunningEphemeralRunners+ephemeralRunnerSet.Status.PendingEphemeralRunners > 0 { + log.Info("Ephemeral runner set spec changed but runners are still active; deleting listener to stop new jobs") + if _, err := r.cleanupListener(ctx, &autoscalingRunnerSet, log); err != nil { + log.Error(err, "Failed to clean up listener while waiting for runners to drain") + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 1 * time.Second}, nil + } + + original := ephemeralRunnerSet.DeepCopy() + ephemeralRunnerSet.Spec.EphemeralRunnerMetadata = desired.Spec.EphemeralRunnerMetadata + ephemeralRunnerSet.Spec.EphemeralRunnerSpec = desired.Spec.EphemeralRunnerSpec + ephemeralRunnerSet.Labels = r.filterAndMergeLabels(ephemeralRunnerSet.Labels, desired.Labels) + ephemeralRunnerSet.Annotations = r.mergeAnnotations(ephemeralRunnerSet.Annotations, desired.Annotations) + + log.Info("Updating ephemeral runner set spec to match the desired spec") + if err := r.Patch(ctx, &ephemeralRunnerSet, client.MergeFrom(original)); err != nil { + log.Error(err, "Failed to patch ephemeral runner set to match the desired spec") + return ctrl.Result{}, err } - return ctrl.Result{}, err + + log.Info("Successfully patched ephemeral runner set spec") + return ctrl.Result{}, nil + } + + ephemeralRunnerMetadataModified := !cmp.Equal(ephemeralRunnerSet.Spec.EphemeralRunnerMetadata, desired.Spec.EphemeralRunnerMetadata) + ephemeralRunnerLabelsModified := !maps.Equal(ephemeralRunnerSet.Labels, desired.Labels) + ephemeralRunnerAnnotationsModified := !maps.Equal(ephemeralRunnerSet.Annotations, desired.Annotations) + + if ephemeralRunnerLabelsModified || ephemeralRunnerAnnotationsModified || ephemeralRunnerMetadataModified { + original := ephemeralRunnerSet.DeepCopy() + ephemeralRunnerSet.Labels = r.filterAndMergeLabels(ephemeralRunnerSet.Labels, desired.Labels) + ephemeralRunnerSet.Annotations = r.mergeAnnotations(ephemeralRunnerSet.Annotations, desired.Annotations) + ephemeralRunnerSet.Spec.EphemeralRunnerMetadata = desired.Spec.EphemeralRunnerMetadata + log.Info("Updating ephemeral runner set metadata to match desired labels and annotations") + if err := r.Patch(ctx, &ephemeralRunnerSet, client.MergeFrom(original)); err != nil { + log.Error(err, "Failed to patch ephemeral runner set metadata to match desired labels and annotations") + return ctrl.Result{}, err + } + + log.Info("Successfully patched ephemeral runner set metadata") + return ctrl.Result{}, nil } - log.Info("Latest runner set spec hash does not match the current autoscaling runner set. Creating a new runner set") - return r.createEphemeralRunnerSet(ctx, autoscalingRunnerSet, log) } - oldRunnerSets := existingRunnerSets.old() - if len(oldRunnerSets) > 0 { - log.Info("Cleanup old ephemeral runner sets", "count", len(oldRunnerSets)) - err := r.deleteEphemeralRunnerSets(ctx, oldRunnerSets, log) + var listener v1alpha1.AutoscalingListener + err = r.Get( + ctx, + types.NamespacedName{ + Namespace: r.ControllerNamespace, + Name: scaleSetListenerName(&autoscalingRunnerSet), + }, + &listener, + ) + switch { + case kerrors.IsNotFound(err): + log.Info("AutoscalingListener does not exist, creating autoscaling listener") + return r.createAutoScalingListenerForRunnerSet(ctx, &autoscalingRunnerSet, &ephemeralRunnerSet, log) + case err != nil: + log.Error(err, "Failed to get AutoscalingListener resource") + return ctrl.Result{}, err + default: + desired, err := r.newAutoscalingListener( + &autoscalingRunnerSet, + &ephemeralRunnerSet, + r.ControllerNamespace, + r.DefaultRunnerScaleSetListenerImage, + nil, // TODO: remove + ) if err != nil { - log.Error(err, "Failed to clean up old runner sets") - return ctrl.Result{}, err + log.Error(err, "Failed to generate AutoscalingListener spec") + return ctrl.Result{}, nil } - } - // Make sure the AutoscalingListener is up and running in the controller namespace - if !listenerFound { - if r.drainingJobs(&latestRunnerSet.Status) { - log.Info("Creating a new AutoscalingListener is waiting for the running and pending runners to finish. Waiting for the running and pending runners to finish:", "running", latestRunnerSet.Status.RunningEphemeralRunners, "pending", latestRunnerSet.Status.PendingEphemeralRunners) + if !cmp.Equal(listener.Spec, desired.Spec) || + !cmp.Equal(listener.Labels, desired.Labels) || + !cmp.Equal(listener.Annotations, desired.Annotations) { + log.Info("Deleting AutoscalingListener to re-create with updated spec") + if err := r.Delete(ctx, &listener); err != nil { + log.Error(err, "Failed to delete AutoscalingListener for re-creation") + return ctrl.Result{}, err + } + log.Info("Deleted AutoscalingListener, will re-create on next reconcile") return ctrl.Result{}, nil } - log.Info("Creating a new AutoscalingListener for the runner set", "ephemeralRunnerSetName", latestRunnerSet.Name) - return r.createAutoScalingListenerForRunnerSet(ctx, autoscalingRunnerSet, latestRunnerSet, log) } - if err := r.updateStatus(ctx, autoscalingRunnerSet, latestRunnerSet, v1alpha1.AutoscalingRunnerSetPhaseRunning, log); err != nil { + log.Info("Autoscaling runner set is up to date and ready") + if err := r.updateStatus( + ctx, + &autoscalingRunnerSet, + &ephemeralRunnerSet, + v1alpha1.AutoscalingRunnerSetPhaseRunning, + log, + ); err != nil { log.Error(err, "Failed to update autoscaling runner set status to running") return ctrl.Result{}, err } @@ -333,7 +395,7 @@ func (r *AutoscalingRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl return ctrl.Result{}, nil } -func (r *AutoscalingRunnerSetReconciler) cleanUpResources(ctx context.Context, autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet, latestRunnerSet *v1alpha1.EphemeralRunnerSet, log logr.Logger) (bool, error) { +func (r *AutoscalingRunnerSetReconciler) cleanUpResources(ctx context.Context, autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet, log logr.Logger) (bool, error) { log.Info("Deleting the listener") done, err := r.cleanupListener(ctx, autoscalingRunnerSet, log) if err != nil { @@ -347,7 +409,7 @@ func (r *AutoscalingRunnerSetReconciler) cleanUpResources(ctx context.Context, a } log.Info("deleting ephemeral runner sets") - done, err = r.cleanupEphemeralRunnerSets(ctx, autoscalingRunnerSet, log) + done, err = r.cleanupEphemeralRunnerSet(ctx, autoscalingRunnerSet, log) if err != nil { log.Error(err, "Failed to clean up ephemeral runner sets") return false, err @@ -371,39 +433,41 @@ func (r *AutoscalingRunnerSetReconciler) cleanUpResources(ctx context.Context, a func (r *AutoscalingRunnerSetReconciler) updateStatus(ctx context.Context, autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet, ephemeralRunnerSet *v1alpha1.EphemeralRunnerSet, phase v1alpha1.AutoscalingRunnerSetPhase, log logr.Logger) error { countDiff := ephemeralRunnerSet != nil && ephemeralRunnerSet.Status.CurrentReplicas != autoscalingRunnerSet.Status.CurrentRunners phaseDiff := phase != autoscalingRunnerSet.Status.Phase - if countDiff || phaseDiff { - if err := patchSubResource(ctx, r.Status(), autoscalingRunnerSet, func(obj *v1alpha1.AutoscalingRunnerSet) { - obj.Status.Phase = phase - var ephemeralRunnerSetStatus v1alpha1.EphemeralRunnerSetStatus - if ephemeralRunnerSet != nil { - ephemeralRunnerSetStatus = ephemeralRunnerSet.Status - } - obj.Status.CurrentRunners = ephemeralRunnerSetStatus.CurrentReplicas - obj.Status.PendingEphemeralRunners = ephemeralRunnerSetStatus.PendingEphemeralRunners - obj.Status.RunningEphemeralRunners = ephemeralRunnerSetStatus.RunningEphemeralRunners - obj.Status.FailedEphemeralRunners = ephemeralRunnerSetStatus.FailedEphemeralRunners - }); err != nil { - log.Error(err, "Failed to update autoscaling runner set status with current runner count") - return err - } + if !countDiff && !phaseDiff { + return nil } - return nil -} -// Prevents overprovisioning of runners. -// We reach this code path when runner scale set has been patched with a new runner spec but there are still running ephemeral runners. -// The safest approach is to wait for the running ephemeral runners to finish before creating a new runner set. -func (r *AutoscalingRunnerSetReconciler) drainingJobs(latestRunnerSetStatus *v1alpha1.EphemeralRunnerSetStatus) bool { - if r.UpdateStrategy == UpdateStrategyEventual && ((latestRunnerSetStatus.RunningEphemeralRunners + latestRunnerSetStatus.PendingEphemeralRunners) > 0) { - return true + original := autoscalingRunnerSet.DeepCopy() + if phaseDiff { + autoscalingRunnerSet.Status.Phase = phase + } + + if countDiff && ephemeralRunnerSet != nil { + autoscalingRunnerSet.Status.CurrentRunners = ephemeralRunnerSet.Status.CurrentReplicas + autoscalingRunnerSet.Status.PendingEphemeralRunners = ephemeralRunnerSet.Status.PendingEphemeralRunners + autoscalingRunnerSet.Status.RunningEphemeralRunners = ephemeralRunnerSet.Status.RunningEphemeralRunners + autoscalingRunnerSet.Status.FailedEphemeralRunners = ephemeralRunnerSet.Status.FailedEphemeralRunners + } + + if err := r.Status().Patch(ctx, autoscalingRunnerSet, client.MergeFrom(original)); err != nil { + log.Error(err, "Failed to patch autoscaling runner set status") + return err } - return false + + return nil } func (r *AutoscalingRunnerSetReconciler) cleanupListener(ctx context.Context, autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet, logger logr.Logger) (done bool, err error) { logger.Info("Cleaning up the listener") var listener v1alpha1.AutoscalingListener - err = r.Get(ctx, client.ObjectKey{Namespace: r.ControllerNamespace, Name: scaleSetListenerName(autoscalingRunnerSet)}, &listener) + err = r.Get( + ctx, + client.ObjectKey{ + Namespace: r.ControllerNamespace, + Name: scaleSetListenerName(autoscalingRunnerSet), + }, + &listener, + ) switch { case err == nil: if listener.DeletionTimestamp.IsZero() { @@ -421,39 +485,32 @@ func (r *AutoscalingRunnerSetReconciler) cleanupListener(ctx context.Context, au return true, nil } -func (r *AutoscalingRunnerSetReconciler) cleanupEphemeralRunnerSets(ctx context.Context, autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet, logger logr.Logger) (done bool, err error) { - logger.Info("Cleaning up ephemeral runner sets") - runnerSets, err := r.listEphemeralRunnerSets(ctx, autoscalingRunnerSet) - if err != nil { - return false, fmt.Errorf("failed to list ephemeral runner sets: %w", err) - } - if runnerSets.empty() { - logger.Info("All ephemeral runner sets are deleted") - return true, nil - } - - logger.Info("Deleting all ephemeral runner sets", "count", runnerSets.count()) - if err := r.deleteEphemeralRunnerSets(ctx, runnerSets.all(), logger); err != nil { - return false, fmt.Errorf("failed to delete ephemeral runner sets: %w", err) - } - return false, nil -} - -func (r *AutoscalingRunnerSetReconciler) deleteEphemeralRunnerSets(ctx context.Context, oldRunnerSets []v1alpha1.EphemeralRunnerSet, logger logr.Logger) error { - for i := range oldRunnerSets { - rs := &oldRunnerSets[i] - // already deleted but contains finalizer so it still exists - if !rs.DeletionTimestamp.IsZero() { - logger.Info("Skip ephemeral runner set since it is already marked for deletion", "name", rs.Name) - continue - } - logger.Info("Deleting ephemeral runner set", "name", rs.Name) - if err := r.Delete(ctx, rs); err != nil { - return fmt.Errorf("failed to delete EphemeralRunnerSet resource: %w", err) +func (r *AutoscalingRunnerSetReconciler) cleanupEphemeralRunnerSet(ctx context.Context, autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet, logger logr.Logger) (done bool, err error) { + logger.Info("Cleaning up ephemeral runner set") + var ers v1alpha1.EphemeralRunnerSet + err = r.Get( + ctx, + client.ObjectKey{ + Namespace: autoscalingRunnerSet.Namespace, + Name: autoscalingRunnerSet.Name, + }, + &ers, + ) + switch { + case err == nil: + if ers.DeletionTimestamp.IsZero() { + logger.Info("Deleting the ephemeral runner set") + if err := r.Delete(ctx, &ers); err != nil { + return false, fmt.Errorf("failed to delete ephemeral runner set: %w", err) + } } - logger.Info("Deleted ephemeral runner set", "name", rs.Name) + return false, nil + case !kerrors.IsNotFound(err): + return false, fmt.Errorf("failed to get ephemeral runner set: %w", err) } - return nil + + logger.Info("Ephemeral runner set is deleted") + return true, nil } func (r *AutoscalingRunnerSetReconciler) removeFinalizersFromDependentResources(ctx context.Context, autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet, logger logr.Logger) error { @@ -612,11 +669,11 @@ func (r *AutoscalingRunnerSetReconciler) updateRunnerScaleSetRunnerGroup(ctx con } logger.Info("Updating runner scale set name and runner group name as annotations") - if err := patch(ctx, r.Client, autoscalingRunnerSet, func(obj *v1alpha1.AutoscalingRunnerSet) { - obj.Annotations[AnnotationKeyGitHubRunnerGroupName] = updatedRunnerScaleSet.RunnerGroupName - obj.Annotations[AnnotationKeyGitHubRunnerScaleSetName] = updatedRunnerScaleSet.Name - }); err != nil { - logger.Error(err, "Failed to update runner group name annotation") + original := autoscalingRunnerSet.DeepCopy() + autoscalingRunnerSet.Annotations[AnnotationKeyGitHubRunnerGroupName] = updatedRunnerScaleSet.RunnerGroupName + autoscalingRunnerSet.Annotations[AnnotationKeyGitHubRunnerScaleSetName] = updatedRunnerScaleSet.Name + if err := r.Patch(ctx, autoscalingRunnerSet, client.MergeFrom(original)); err != nil { + logger.Error(err, "Failed to update runner group name and runner scale set name annotation") return ctrl.Result{}, err } @@ -649,9 +706,9 @@ func (r *AutoscalingRunnerSetReconciler) updateRunnerScaleSetName(ctx context.Co } logger.Info("Updating runner scale set name as an annotation") - if err := patch(ctx, r.Client, autoscalingRunnerSet, func(obj *v1alpha1.AutoscalingRunnerSet) { - obj.Annotations[AnnotationKeyGitHubRunnerScaleSetName] = updatedRunnerScaleSet.Name - }); err != nil { + original := autoscalingRunnerSet.DeepCopy() + autoscalingRunnerSet.Annotations[AnnotationKeyGitHubRunnerScaleSetName] = updatedRunnerScaleSet.Name + if err := r.Patch(ctx, autoscalingRunnerSet, client.MergeFrom(original)); err != nil { logger.Error(err, "Failed to update runner scale set name annotation") return ctrl.Result{}, err } @@ -669,7 +726,7 @@ func (r *AutoscalingRunnerSetReconciler) deleteRunnerScaleSet(ctx context.Contex // // 2. The scale set has been deleted by the controller. // In that case, the controller will clean up annotation because the scale set does not exist anymore. - // Removal of the scale set id is also useful because permission cleanup will eventually lose permission + // Removal of the scale set id is also useful because permission cleanup will later lose permission // assigned to it on a GitHub secret, causing actions client from secret to result in permission denied // // 3. Annotation is removed manually. @@ -699,11 +756,11 @@ func (r *AutoscalingRunnerSetReconciler) deleteRunnerScaleSet(ctx context.Contex return err } - err = patch(ctx, r.Client, autoscalingRunnerSet, func(obj *v1alpha1.AutoscalingRunnerSet) { - delete(obj.Annotations, runnerScaleSetIDAnnotationKey) - }) - if err != nil { - logger.Error(err, "Failed to patch autoscaling runner set with annotation removed", "annotation", runnerScaleSetIDAnnotationKey) + original := autoscalingRunnerSet.DeepCopy() + delete(autoscalingRunnerSet.Annotations, runnerScaleSetIDAnnotationKey) + + if err := r.Patch(ctx, autoscalingRunnerSet, client.MergeFrom(original)); err != nil { + logger.Error(err, "Failed to remove runner scale set ID annotation after deleting the runner scale set", "runnerScaleSetId", runnerScaleSetID) return err } @@ -718,11 +775,6 @@ func (r *AutoscalingRunnerSetReconciler) createEphemeralRunnerSet(ctx context.Co return ctrl.Result{}, err } - if err := ctrl.SetControllerReference(autoscalingRunnerSet, desiredRunnerSet, r.Scheme); err != nil { - log.Error(err, "Failed to set controller reference to a new EphemeralRunnerSet") - return ctrl.Result{}, err - } - log.Info("Creating a new EphemeralRunnerSet resource") if err := r.Create(ctx, desiredRunnerSet); err != nil { log.Error(err, "Failed to create EphemeralRunnerSet resource") @@ -741,7 +793,13 @@ func (r *AutoscalingRunnerSetReconciler) createAutoScalingListenerForRunnerSet(c }) } - autoscalingListener, err := r.newAutoScalingListener(autoscalingRunnerSet, ephemeralRunnerSet, r.ControllerNamespace, r.DefaultRunnerScaleSetListenerImage, imagePullSecrets) + autoscalingListener, err := r.newAutoscalingListener( + autoscalingRunnerSet, + ephemeralRunnerSet, + r.ControllerNamespace, + r.DefaultRunnerScaleSetListenerImage, + imagePullSecrets, + ) if err != nil { log.Error(err, "Could not create AutoscalingListener spec") return ctrl.Result{}, err @@ -757,17 +815,20 @@ func (r *AutoscalingRunnerSetReconciler) createAutoScalingListenerForRunnerSet(c return ctrl.Result{}, nil } -func (r *AutoscalingRunnerSetReconciler) listEphemeralRunnerSets(ctx context.Context, autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet) (*EphemeralRunnerSets, error) { - list := new(v1alpha1.EphemeralRunnerSetList) - if err := r.List(ctx, list, client.InNamespace(autoscalingRunnerSet.Namespace), client.MatchingFields{resourceOwnerKey: autoscalingRunnerSet.Name}); err != nil { - return nil, fmt.Errorf("failed to list ephemeral runner sets: %w", err) +// TODO: change that +func shouldCreateScaleSet(autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet) bool { + scaleSetIDRaw, ok := autoscalingRunnerSet.Annotations[runnerScaleSetIDAnnotationKey] + if !ok { + return true } - - return &EphemeralRunnerSets{list: list}, nil + id, err := strconv.Atoi(scaleSetIDRaw) + return err != nil || id <= 0 } // SetupWithManager sets up the controller with the Manager. func (r *AutoscalingRunnerSetReconciler) SetupWithManager(mgr ctrl.Manager, opts ...Option) error { + r.ResourceBuilder.setSchemeIfUnset(r.Scheme) + return builderWithOptions( ctrl.NewControllerManagedBy(mgr). For(&v1alpha1.AutoscalingRunnerSet{}). @@ -829,12 +890,13 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeKubernetesModeRol c.logger.Info("Kubernetes mode role binding finalizer has already been removed", "name", roleBindingName) return } - err = patch(ctx, c.client, roleBinding, func(obj *rbacv1.RoleBinding) { - controllerutil.RemoveFinalizer(obj, AutoscalingRunnerSetCleanupFinalizerName) - }) - if err != nil { - c.err = fmt.Errorf("failed to patch kubernetes mode role binding without finalizer: %w", err) - return + + original := roleBinding.DeepCopy() + if controllerutil.RemoveFinalizer(roleBinding, AutoscalingRunnerSetCleanupFinalizerName) { + if err = c.client.Patch(ctx, roleBinding, client.MergeFrom(original)); err != nil { + c.err = fmt.Errorf("failed to patch kubernetes mode role binding without finalizer: %w", err) + return + } } c.logger.Info("Removed finalizer from container mode kubernetes role binding", "name", roleBindingName) return @@ -871,12 +933,12 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeKubernetesModeRol c.logger.Info("Kubernetes mode role finalizer has already been removed", "name", roleName) return } - err = patch(ctx, c.client, role, func(obj *rbacv1.Role) { - controllerutil.RemoveFinalizer(obj, AutoscalingRunnerSetCleanupFinalizerName) - }) - if err != nil { - c.err = fmt.Errorf("failed to patch kubernetes mode role without finalizer: %w", err) - return + original := role.DeepCopy() + if controllerutil.RemoveFinalizer(role, AutoscalingRunnerSetCleanupFinalizerName) { + if err = c.client.Patch(ctx, role, client.MergeFrom(original)); err != nil { + c.err = fmt.Errorf("failed to patch kubernetes mode role without finalizer: %w", err) + return + } } c.logger.Info("Removed finalizer from container mode kubernetes role") return @@ -914,13 +976,14 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeKubernetesModeSer c.logger.Info("Kubernetes mode service account finalizer has already been removed", "name", serviceAccountName) return } - err = patch(ctx, c.client, serviceAccount, func(obj *corev1.ServiceAccount) { - controllerutil.RemoveFinalizer(obj, AutoscalingRunnerSetCleanupFinalizerName) - }) - if err != nil { - c.err = fmt.Errorf("failed to patch kubernetes mode service account without finalizer: %w", err) - return + original := serviceAccount.DeepCopy() + if controllerutil.RemoveFinalizer(serviceAccount, AutoscalingRunnerSetCleanupFinalizerName) { + if err = c.client.Patch(ctx, serviceAccount, client.MergeFrom(original)); err != nil { + c.err = fmt.Errorf("failed to patch kubernetes mode service account without finalizer: %w", err) + return + } } + c.logger.Info("Removed finalizer from container mode kubernetes service account") return case kerrors.IsNotFound(err): @@ -957,13 +1020,15 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeNoPermissionServi c.logger.Info("No permission service account finalizer has already been removed", "name", serviceAccountName) return } - err = patch(ctx, c.client, serviceAccount, func(obj *corev1.ServiceAccount) { - controllerutil.RemoveFinalizer(obj, AutoscalingRunnerSetCleanupFinalizerName) - }) - if err != nil { - c.err = fmt.Errorf("failed to patch service account without finalizer: %w", err) - return + original := serviceAccount.DeepCopy() + + if controllerutil.RemoveFinalizer(serviceAccount, AutoscalingRunnerSetCleanupFinalizerName) { + if err = c.client.Patch(ctx, serviceAccount, client.MergeFrom(original)); err != nil { + c.err = fmt.Errorf("failed to patch no permission service account without finalizer: %w", err) + return + } } + c.logger.Info("Removed finalizer from no permission service account", "name", serviceAccountName) return case kerrors.IsNotFound(err): @@ -1000,12 +1065,13 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeGitHubSecretFinal c.logger.Info("GitHub secret finalizer has already been removed", "name", githubSecretName) return } - err = patch(ctx, c.client, githubSecret, func(obj *corev1.Secret) { - controllerutil.RemoveFinalizer(obj, AutoscalingRunnerSetCleanupFinalizerName) - }) - if err != nil { - c.err = fmt.Errorf("failed to patch GitHub secret without finalizer: %w", err) - return + original := githubSecret.DeepCopy() + + if controllerutil.RemoveFinalizer(githubSecret, AutoscalingRunnerSetCleanupFinalizerName) { + if err = c.client.Patch(ctx, githubSecret, client.MergeFrom(original)); err != nil { + c.err = fmt.Errorf("failed to patch GitHub secret without finalizer: %w", err) + return + } } c.logger.Info("Removed finalizer from GitHub secret", "name", githubSecretName) return @@ -1039,17 +1105,14 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeManagerRoleBindin err := c.client.Get(ctx, types.NamespacedName{Name: managerRoleBindingName, Namespace: c.autoscalingRunnerSet.Namespace}, roleBinding) switch { case err == nil: - if !controllerutil.ContainsFinalizer(roleBinding, AutoscalingRunnerSetCleanupFinalizerName) { - c.logger.Info("Manager role binding finalizer has already been removed", "name", managerRoleBindingName) - return - } - err = patch(ctx, c.client, roleBinding, func(obj *rbacv1.RoleBinding) { - controllerutil.RemoveFinalizer(obj, AutoscalingRunnerSetCleanupFinalizerName) - }) - if err != nil { - c.err = fmt.Errorf("failed to patch manager role binding without finalizer: %w", err) - return + original := roleBinding.DeepCopy() + if controllerutil.RemoveFinalizer(roleBinding, AutoscalingRunnerSetCleanupFinalizerName) { + if err = c.client.Patch(ctx, roleBinding, client.MergeFrom(original)); err != nil { + c.err = fmt.Errorf("failed to patch manager role binding without finalizer: %w", err) + return + } } + c.logger.Info("Removed finalizer from manager role binding", "name", managerRoleBindingName) return case kerrors.IsNotFound(err): @@ -1082,16 +1145,12 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeManagerRoleFinali err := c.client.Get(ctx, types.NamespacedName{Name: managerRoleName, Namespace: c.autoscalingRunnerSet.Namespace}, role) switch { case err == nil: - if !controllerutil.ContainsFinalizer(role, AutoscalingRunnerSetCleanupFinalizerName) { - c.logger.Info("Manager role finalizer has already been removed", "name", managerRoleName) - return - } - err = patch(ctx, c.client, role, func(obj *rbacv1.Role) { - controllerutil.RemoveFinalizer(obj, AutoscalingRunnerSetCleanupFinalizerName) - }) - if err != nil { - c.err = fmt.Errorf("failed to patch manager role without finalizer: %w", err) - return + original := role.DeepCopy() + if controllerutil.RemoveFinalizer(role, AutoscalingRunnerSetCleanupFinalizerName) { + if err := c.client.Patch(ctx, role, client.MergeFrom(original)); err != nil { + c.err = fmt.Errorf("failed to patch manager role without finalizer: %w", err) + return + } } c.logger.Info("Removed finalizer from manager role", "name", managerRoleName) return @@ -1103,54 +1162,3 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeManagerRoleFinali return } } - -// NOTE: if this is logic should be used for other resources, -// consider using generics - -type EphemeralRunnerSets struct { - list *v1alpha1.EphemeralRunnerSetList - sorted bool -} - -func (rs *EphemeralRunnerSets) latest() *v1alpha1.EphemeralRunnerSet { - if rs.empty() { - return nil - } - if !rs.sorted { - rs.sort() - } - return rs.list.Items[0].DeepCopy() -} - -func (rs *EphemeralRunnerSets) old() []v1alpha1.EphemeralRunnerSet { - if rs.empty() { - return nil - } - if !rs.sorted { - rs.sort() - } - copy := rs.list.DeepCopy() - return copy.Items[1:] -} - -func (rs *EphemeralRunnerSets) all() []v1alpha1.EphemeralRunnerSet { - if rs.empty() { - return nil - } - copy := rs.list.DeepCopy() - return copy.Items -} - -func (rs *EphemeralRunnerSets) empty() bool { - return rs.list == nil || len(rs.list.Items) == 0 -} - -func (rs *EphemeralRunnerSets) sort() { - sort.Slice(rs.list.Items, func(i, j int) bool { - return rs.list.Items[i].GetCreationTimestamp().After(rs.list.Items[j].GetCreationTimestamp().Time) - }) -} - -func (rs *EphemeralRunnerSets) count() int { - return len(rs.list.Items) -} diff --git a/controllers/actions.github.com/autoscalingrunnerset_controller_test.go b/controllers/actions.github.com/autoscalingrunnerset_controller_test.go index 1592fc43e1..55ebe5c6ae 100644 --- a/controllers/actions.github.com/autoscalingrunnerset_controller_test.go +++ b/controllers/actions.github.com/autoscalingrunnerset_controller_test.go @@ -115,7 +115,9 @@ var _ = Describe("Test AutoScalingRunnerSet controller", Ordered, func() { Name: "test-asrs", Namespace: autoscalingNS.Name, Labels: map[string]string{ - LabelKeyKubernetesVersion: buildVersion, + LabelKeyKubernetesVersion: buildVersion, + "arc.test/label-drift": "initial", + "arc.test/spec-update-label": "initial", }, }, Spec: v1alpha1.AutoscalingRunnerSetSpec{ @@ -124,6 +126,19 @@ var _ = Describe("Test AutoScalingRunnerSet controller", Ordered, func() { MaxRunners: &max, MinRunners: &min, RunnerGroup: "testgroup", + EphemeralRunnerSetMetadata: &v1alpha1.ResourceMeta{ + Annotations: map[string]string{ + "arc.test/metadata-annotation": "initial", + }, + }, + EphemeralRunnerMetadata: &v1alpha1.ResourceMeta{ + Labels: map[string]string{ + "arc.test/runner-metadata-label": "initial", + }, + Annotations: map[string]string{ + "arc.test/runner-metadata-annotation": "initial", + }, + }, Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ @@ -442,8 +457,7 @@ var _ = Describe("Test AutoScalingRunnerSet controller", Ordered, func() { }) Context("When updating a new AutoScalingRunnerSet", func() { - It("It should re-create EphemeralRunnerSet and Listener as needed when updating AutoScalingRunnerSet", func() { - // Wait till the listener is created + It("updates EphemeralRunnerSet when the runner image changes without touching the Listener", func() { listener := new(v1alpha1.AutoscalingListener) Eventually( func() error { @@ -452,161 +466,315 @@ var _ = Describe("Test AutoScalingRunnerSet controller", Ordered, func() { autoscalingRunnerSetTestTimeout, autoscalingRunnerSetTestInterval, ).Should(Succeed(), "Listener should be created") + originalListenerUID := listener.UID + originalListenerResourceVersion := listener.ResourceVersion - runnerSetList := new(v1alpha1.EphemeralRunnerSetList) - err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace)) - Expect(err).NotTo(HaveOccurred(), "failed to list EphemeralRunnerSet") - Expect(len(runnerSetList.Items)).To(Equal(1), "There should be 1 EphemeralRunnerSet") - runnerSet := runnerSetList.Items[0] + runnerSet := new(v1alpha1.EphemeralRunnerSet) + Eventually( + func() error { + return k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet) + }, + autoscalingRunnerSetTestTimeout, + autoscalingRunnerSetTestInterval, + ).Should(Succeed(), "EphemeralRunnerSet should be created") + originalRunnerSetUID := runnerSet.UID + originalRunnerSetHash := runnerSet.Annotations[annotationKeyIntegrityHash] - // Update the AutoScalingRunnerSet.Spec.Template - // This should trigger re-creation of EphemeralRunnerSet and Listener patched := autoscalingRunnerSet.DeepCopy() - patched.Spec.Template.Spec.PriorityClassName = "test-priority-class" - if patched.Annotations == nil { - patched.Annotations = make(map[string]string) - } - patched.Annotations[annotationKeyValuesHash] = "test-hash" - err = k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet)) + patched.Spec.Template.Spec.Containers[0].Image = "ghcr.io/actions/runner:updated" + err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet)) Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet") - autoscalingRunnerSet = patched.DeepCopy() - // We should create a new EphemeralRunnerSet and delete the old one, eventually, we will have only one EphemeralRunnerSet Eventually( - func() (string, error) { - runnerSetList := new(v1alpha1.EphemeralRunnerSetList) - err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace)) - if err != nil { - return "", err - } + func(g Gomega) { + current := new(v1alpha1.EphemeralRunnerSet) + err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current) + g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet") + g.Expect(current.UID).To(Equal(originalRunnerSetUID), "EphemeralRunnerSet should be updated in place") + g.Expect(current.Spec.EphemeralRunnerSpec.PodTemplateSpec.Spec.Containers[0].Image).To(Equal("ghcr.io/actions/runner:updated")) + g.Expect(current.Annotations[annotationKeyIntegrityHash]).NotTo(Equal(originalRunnerSetHash), "EphemeralRunnerSet spec hash should change") + }, + autoscalingRunnerSetTestTimeout, + autoscalingRunnerSetTestInterval, + ).Should(Succeed()) - if len(runnerSetList.Items) != 1 { - return "", fmt.Errorf("We should have only 1 EphemeralRunnerSet, but got %v", len(runnerSetList.Items)) - } + Consistently( + func(g Gomega) { + current := new(v1alpha1.AutoscalingListener) + err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, current) + g.Expect(err).NotTo(HaveOccurred(), "failed to get Listener") + g.Expect(current.UID).To(Equal(originalListenerUID), "Listener should not be recreated") + g.Expect(current.ResourceVersion).To(Equal(originalListenerResourceVersion), "Listener should not be updated") + }, + time.Second*5, + autoscalingRunnerSetTestInterval, + ).Should(Succeed()) + }) + + It("recreates only the Listener when max runners changes", func() { + listener := new(v1alpha1.AutoscalingListener) + Eventually( + func() error { + return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener) + }, + autoscalingRunnerSetTestTimeout, + autoscalingRunnerSetTestInterval, + ).Should(Succeed(), "Listener should be created") + originalListenerUID := listener.UID + + runnerSet := new(v1alpha1.EphemeralRunnerSet) + Eventually( + func() error { + return k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet) + }, + autoscalingRunnerSetTestTimeout, + autoscalingRunnerSetTestInterval, + ).Should(Succeed(), "EphemeralRunnerSet should be created") + originalRunnerSetUID := runnerSet.UID + originalRunnerSetHash := runnerSet.Annotations[annotationKeyIntegrityHash] + + patched := autoscalingRunnerSet.DeepCopy() + max := 20 + patched.Spec.MaxRunners = &max + err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet)) + Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet") - return runnerSetList.Items[0].Annotations[annotationKeyRunnerSpecHash], nil + Eventually( + func(g Gomega) { + current := new(v1alpha1.AutoscalingListener) + err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, current) + g.Expect(err).NotTo(HaveOccurred(), "failed to get Listener") + g.Expect(current.UID).NotTo(Equal(originalListenerUID), "Listener should be recreated") + g.Expect(current.Spec.MaxRunners).To(Equal(max)) }, autoscalingRunnerSetTestTimeout, autoscalingRunnerSetTestInterval, - ).ShouldNot(BeEquivalentTo(runnerSet.Annotations[annotationKeyRunnerSpecHash]), "New EphemeralRunnerSet should be created") + ).Should(Succeed()) + + Consistently( + func(g Gomega) { + current := new(v1alpha1.EphemeralRunnerSet) + err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current) + g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet") + g.Expect(current.UID).To(Equal(originalRunnerSetUID), "EphemeralRunnerSet should not be recreated") + g.Expect(current.Annotations[annotationKeyIntegrityHash]).To(Equal(originalRunnerSetHash), "EphemeralRunnerSet spec should not change") + }, + time.Second*5, + autoscalingRunnerSetTestInterval, + ).Should(Succeed()) + }) - // We should create a new listener + It("updates EphemeralRunnerSet labels when only AutoScalingRunnerSet labels change", func() { + runnerSet := new(v1alpha1.EphemeralRunnerSet) Eventually( func() (string, error) { - listener := new(v1alpha1.AutoscalingListener) - err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener) + err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet) if err != nil { return "", err } - - return listener.Spec.EphemeralRunnerSetName, nil + return runnerSet.Labels["arc.test/label-drift"], nil }, autoscalingRunnerSetTestTimeout, autoscalingRunnerSetTestInterval, - ).ShouldNot(BeEquivalentTo(runnerSet.Name), "New Listener should be created") + ).Should(Equal("initial"), "EphemeralRunnerSet should start with the predefined label") - // Only update the Spec for the AutoScalingListener - // This should trigger re-creation of the Listener only - runnerSetList = new(v1alpha1.EphemeralRunnerSetList) - err = k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace)) - Expect(err).NotTo(HaveOccurred(), "failed to list EphemeralRunnerSet") - Expect(len(runnerSetList.Items)).To(Equal(1), "There should be 1 EphemeralRunnerSet") - runnerSet = runnerSetList.Items[0] - - listener = new(v1alpha1.AutoscalingListener) - err = k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener) - Expect(err).NotTo(HaveOccurred(), "failed to get Listener") - - patched = autoscalingRunnerSet.DeepCopy() - min := 10 - patched.Spec.MinRunners = &min - err = k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet)) - Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet") + patched := autoscalingRunnerSet.DeepCopy() + patched.Labels["arc.test/label-drift"] = "updated" + err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet)) + Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet labels") - // We should not re-create a new EphemeralRunnerSet - Consistently( + Eventually( func() (string, error) { - runnerSetList := new(v1alpha1.EphemeralRunnerSetList) - err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace)) + current := new(v1alpha1.EphemeralRunnerSet) + err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current) if err != nil { return "", err } - - if len(runnerSetList.Items) != 1 { - return "", fmt.Errorf("We should have only 1 EphemeralRunnerSet, but got %v", len(runnerSetList.Items)) - } - - return string(runnerSetList.Items[0].UID), nil + return current.Labels["arc.test/label-drift"], nil }, autoscalingRunnerSetTestTimeout, autoscalingRunnerSetTestInterval, - ).Should(BeEquivalentTo(string(runnerSet.UID)), "New EphemeralRunnerSet should not be created") + ).Should(Equal("updated"), "EphemeralRunnerSet should be patched with label-only drift") + }) - // We should only re-create a new listener + It("updates EphemeralRunnerSet annotations when only EphemeralRunnerSet metadata annotations change", func() { + runnerSet := new(v1alpha1.EphemeralRunnerSet) Eventually( func() (string, error) { - listener := new(v1alpha1.AutoscalingListener) - err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener) + err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet) if err != nil { return "", err } + return runnerSet.Annotations["arc.test/metadata-annotation"], nil + }, + autoscalingRunnerSetTestTimeout, + autoscalingRunnerSetTestInterval, + ).Should(Equal("initial"), "EphemeralRunnerSet should start with the predefined annotation") - return string(listener.UID), nil + patched := autoscalingRunnerSet.DeepCopy() + patched.Spec.EphemeralRunnerSetMetadata.Annotations["arc.test/metadata-annotation"] = "updated" + patched.Spec.EphemeralRunnerSetMetadata.Annotations["arc.test/new-metadata-annotation"] = "added" + err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet)) + Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet EphemeralRunnerSet metadata") + + Eventually( + func(g Gomega) { + current := new(v1alpha1.EphemeralRunnerSet) + err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current) + g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet") + g.Expect(current.Annotations["arc.test/metadata-annotation"]).To(Equal("updated")) + g.Expect(current.Annotations["arc.test/new-metadata-annotation"]).To(Equal("added")) }, autoscalingRunnerSetTestTimeout, autoscalingRunnerSetTestInterval, - ).ShouldNot(BeEquivalentTo(string(listener.UID)), "New Listener should be created") + ).Should(Succeed(), "EphemeralRunnerSet should be patched with annotation-only metadata drift") + }) - // Only update the values hash for the autoscaling runner set - // This should trigger re-creation of the Listener only - runnerSetList = new(v1alpha1.EphemeralRunnerSetList) - err = k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace)) - Expect(err).NotTo(HaveOccurred(), "failed to list EphemeralRunnerSet") - Expect(len(runnerSetList.Items)).To(Equal(1), "There should be 1 EphemeralRunnerSet") - runnerSet = runnerSetList.Items[0] + It("updates EphemeralRunnerSet runner metadata when only EphemeralRunner metadata changes", func() { + runnerSet := new(v1alpha1.EphemeralRunnerSet) + Eventually( + func(g Gomega) { + err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet) + g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet") + g.Expect(runnerSet.Spec.EphemeralRunnerMetadata).NotTo(BeNil()) + g.Expect(runnerSet.Spec.EphemeralRunnerMetadata.Labels["arc.test/runner-metadata-label"]).To(Equal("initial")) + g.Expect(runnerSet.Spec.EphemeralRunnerMetadata.Annotations["arc.test/runner-metadata-annotation"]).To(Equal("initial")) + }, + autoscalingRunnerSetTestTimeout, + autoscalingRunnerSetTestInterval, + ).Should(Succeed()) - listener = new(v1alpha1.AutoscalingListener) - err = k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener) - Expect(err).NotTo(HaveOccurred(), "failed to get Listener") + patched := autoscalingRunnerSet.DeepCopy() + patched.Spec.EphemeralRunnerMetadata.Labels["arc.test/runner-metadata-label"] = "updated" + patched.Spec.EphemeralRunnerMetadata.Annotations["arc.test/runner-metadata-annotation"] = "updated" + patched.Spec.EphemeralRunnerMetadata.Annotations["arc.test/new-runner-metadata-annotation"] = "added" + err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet)) + Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet EphemeralRunner metadata") - patched = autoscalingRunnerSet.DeepCopy() - patched.Annotations[annotationKeyValuesHash] = "hash-changes" - err = k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet)) - Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet") + Eventually( + func(g Gomega) { + current := new(v1alpha1.EphemeralRunnerSet) + err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current) + g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet") + g.Expect(current.Spec.EphemeralRunnerMetadata).NotTo(BeNil()) + g.Expect(current.Spec.EphemeralRunnerMetadata.Labels["arc.test/runner-metadata-label"]).To(Equal("updated")) + g.Expect(current.Spec.EphemeralRunnerMetadata.Annotations["arc.test/runner-metadata-annotation"]).To(Equal("updated")) + g.Expect(current.Spec.EphemeralRunnerMetadata.Annotations["arc.test/new-runner-metadata-annotation"]).To(Equal("added")) + }, + autoscalingRunnerSetTestTimeout, + autoscalingRunnerSetTestInterval, + ).Should(Succeed()) + }) - // We should not re-create a new EphemeralRunnerSet - Consistently( + It("propagates latest labels while updating EphemeralRunnerSet spec", func() { + runnerSet := new(v1alpha1.EphemeralRunnerSet) + Eventually( func() (string, error) { - runnerSetList := new(v1alpha1.EphemeralRunnerSetList) - err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace)) + err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet) if err != nil { return "", err } + return runnerSet.Labels["arc.test/spec-update-label"], nil + }, + autoscalingRunnerSetTestTimeout, + autoscalingRunnerSetTestInterval, + ).Should(Equal("initial"), "EphemeralRunnerSet should start with the predefined label") - if len(runnerSetList.Items) != 1 { - return "", fmt.Errorf("We should have only 1 EphemeralRunnerSet, but got %v", len(runnerSetList.Items)) - } + patched := autoscalingRunnerSet.DeepCopy() + patched.Labels["arc.test/spec-update-label"] = "updated" + patched.Spec.EphemeralRunnerSetMetadata.Annotations["arc.test/metadata-annotation"] = "updated" + patched.Spec.EphemeralRunnerSetMetadata.Annotations["arc.test/new-spec-update-annotation"] = "added" + patched.Spec.Template.Spec.Containers[0].Image = "ghcr.io/actions/runner:updated-with-label" + err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet)) + Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet spec and labels") + + Eventually( + func(g Gomega) { + current := new(v1alpha1.EphemeralRunnerSet) + err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current) + g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet") + g.Expect(current.Spec.EphemeralRunnerSpec.PodTemplateSpec.Spec.Containers[0].Image).To(Equal("ghcr.io/actions/runner:updated-with-label")) + g.Expect(current.Labels["arc.test/spec-update-label"]).To(Equal("updated")) + g.Expect(current.Annotations["arc.test/metadata-annotation"]).To(Equal("updated")) + g.Expect(current.Annotations["arc.test/new-spec-update-annotation"]).To(Equal("added")) + }, + autoscalingRunnerSetTestTimeout, + autoscalingRunnerSetTestInterval, + ).Should(Succeed()) + }) - return string(runnerSetList.Items[0].UID), nil + It("updates EphemeralRunnerSet and Listener when the GitHub config secret changes", func() { + updatedSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "github-config-secret-updated", + Namespace: autoscalingRunnerSet.Namespace, + }, + Data: map[string][]byte{ + "github_token": []byte(defaultGitHubToken), + }, + } + err := k8sClient.Create(ctx, updatedSecret) + Expect(err).NotTo(HaveOccurred(), "failed to create updated GitHub config secret") + + listener := new(v1alpha1.AutoscalingListener) + Eventually( + func() error { + return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener) }, autoscalingRunnerSetTestTimeout, autoscalingRunnerSetTestInterval, - ).Should(BeEquivalentTo(string(runnerSet.UID)), "New EphemeralRunnerSet should not be created") + ).Should(Succeed(), "Listener should be created") + originalListenerUID := listener.UID - // We should only re-create a new listener + runnerSet := new(v1alpha1.EphemeralRunnerSet) Eventually( - func() (string, error) { - listener := new(v1alpha1.AutoscalingListener) - err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener) - if err != nil { - return "", err - } + func() error { + return k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet) + }, + autoscalingRunnerSetTestTimeout, + autoscalingRunnerSetTestInterval, + ).Should(Succeed(), "EphemeralRunnerSet should be created") + originalRunnerSetUID := runnerSet.UID + + patched := autoscalingRunnerSet.DeepCopy() + patched.Spec.GitHubConfigSecret = updatedSecret.Name + err = k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet)) + Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet") + + Eventually( + func(g Gomega) { + current := new(v1alpha1.AutoscalingRunnerSet) + err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current) + g.Expect(err).NotTo(HaveOccurred(), "failed to get AutoScalingRunnerSet") + g.Expect(current.Spec.GitHubConfigSecret).To(Equal(updatedSecret.Name)) + }, + autoscalingRunnerSetTestTimeout, + autoscalingRunnerSetTestInterval, + ).Should(Succeed()) - return string(listener.UID), nil + Eventually( + func(g Gomega) { + current := new(v1alpha1.EphemeralRunnerSet) + err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current) + g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet") + g.Expect(current.UID).To(Equal(originalRunnerSetUID), "EphemeralRunnerSet should be updated in place") + g.Expect(current.Spec.EphemeralRunnerSpec.GitHubConfigSecret).To(Equal(updatedSecret.Name)) }, autoscalingRunnerSetTestTimeout, autoscalingRunnerSetTestInterval, - ).ShouldNot(BeEquivalentTo(string(listener.UID)), "New Listener should be created") + ).Should(Succeed()) + + Eventually( + func(g Gomega) { + current := new(v1alpha1.AutoscalingListener) + err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, current) + g.Expect(err).NotTo(HaveOccurred(), "failed to get Listener") + g.Expect(current.UID).NotTo(Equal(originalListenerUID), "Listener should be recreated") + g.Expect(current.Spec.GitHubConfigSecret).To(Equal(updatedSecret.Name)) + }, + autoscalingRunnerSetTestTimeout, + autoscalingRunnerSetTestInterval, + ).Should(Succeed()) }) It("It should update RunnerScaleSet's runner group on service when it changes", func() { @@ -670,9 +838,7 @@ var _ = Describe("Test AutoScalingRunnerSet controller", Ordered, func() { }) Context("When updating an AutoscalingRunnerSet with running or pending jobs", func() { - It("It should wait for running and pending jobs to finish before applying the update. Update Strategy is set to eventual.", func() { - // Switch update strategy to eventual (drain jobs ) - controller.UpdateStrategy = UpdateStrategyEventual + It("It should wait for running and pending jobs to finish before applying the update.", func() { // Wait till the listener is created listener := new(v1alpha1.AutoscalingListener) Eventually( @@ -740,7 +906,7 @@ var _ = Describe("Test AutoScalingRunnerSet controller", Ordered, func() { if patched.Annotations == nil { patched.Annotations = make(map[string]string) } - patched.Annotations[annotationKeyValuesHash] = "testgroup2" + patched.Annotations[annotationKeyIntegrityHash] = "testgroup2" patched.Spec.Template.Spec = corev1.PodSpec{ Containers: []corev1.Container{ { diff --git a/controllers/actions.github.com/clientutil.go b/controllers/actions.github.com/clientutil.go deleted file mode 100644 index ac255d7aa0..0000000000 --- a/controllers/actions.github.com/clientutil.go +++ /dev/null @@ -1,32 +0,0 @@ -package actionsgithubcom - -import ( - "context" - - kclient "sigs.k8s.io/controller-runtime/pkg/client" -) - -type kubernetesObject[T kclient.Object] interface { - kclient.Object - DeepCopy() T -} - -type patcher interface { - Patch(ctx context.Context, obj kclient.Object, patch kclient.Patch, opts ...kclient.PatchOption) error -} - -func patch[T kubernetesObject[T]](ctx context.Context, client patcher, obj T, update func(obj T)) error { - original := obj.DeepCopy() - update(obj) - return client.Patch(ctx, obj, kclient.MergeFrom(original)) -} - -type subResourcePatcher interface { - Patch(ctx context.Context, obj kclient.Object, patch kclient.Patch, opts ...kclient.SubResourcePatchOption) error -} - -func patchSubResource[T kubernetesObject[T]](ctx context.Context, client subResourcePatcher, obj T, update func(obj T)) error { - original := obj.DeepCopy() - update(obj) - return client.Patch(ctx, obj, kclient.MergeFrom(original)) -} diff --git a/controllers/actions.github.com/ephemeralrunner_controller.go b/controllers/actions.github.com/ephemeralrunner_controller.go index ee5e2c875a..68ca871dae 100644 --- a/controllers/actions.github.com/ephemeralrunner_controller.go +++ b/controllers/actions.github.com/ephemeralrunner_controller.go @@ -79,19 +79,20 @@ const maxFailures = 5 func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.Log.WithValues("ephemeralrunner", req.NamespacedName) - ephemeralRunner := new(v1alpha1.EphemeralRunner) - if err := r.Get(ctx, req.NamespacedName, ephemeralRunner); err != nil { + var ephemeralRunner v1alpha1.EphemeralRunner + if err := r.Get(ctx, req.NamespacedName, &ephemeralRunner); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } + original := ephemeralRunner.DeepCopy() if !ephemeralRunner.DeletionTimestamp.IsZero() { - if !controllerutil.ContainsFinalizer(ephemeralRunner, ephemeralRunnerFinalizerName) { + if !controllerutil.ContainsFinalizer(&ephemeralRunner, ephemeralRunnerFinalizerName) { return ctrl.Result{}, nil } - if controllerutil.ContainsFinalizer(ephemeralRunner, ephemeralRunnerActionsFinalizerName) { + if controllerutil.ContainsFinalizer(&ephemeralRunner, ephemeralRunnerActionsFinalizerName) { log.Info("Trying to clean up runner from the service") - ok, err := r.cleanupRunnerFromService(ctx, ephemeralRunner, log) + ok, err := r.cleanupRunnerFromService(ctx, &ephemeralRunner, log) if err != nil { log.Error(err, "Failed to clean up runner from service") return ctrl.Result{}, err @@ -102,16 +103,18 @@ func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Requ } log.Info("Runner is cleaned up from the service, removing finalizer") - if err := patch(ctx, r.Client, ephemeralRunner, func(obj *v1alpha1.EphemeralRunner) { - controllerutil.RemoveFinalizer(obj, ephemeralRunnerActionsFinalizerName) - }); err != nil { - return ctrl.Result{}, err + if controllerutil.RemoveFinalizer(&ephemeralRunner, ephemeralRunnerActionsFinalizerName) { + log.Info("Removed finalizer from ephemeral runner") + if err := r.Patch(ctx, &ephemeralRunner, client.MergeFrom(original)); err != nil { + log.Error(err, "Failed to update ephemeral runner after removing finalizer") + return ctrl.Result{}, err + } } log.Info("Removed finalizer from ephemeral runner") } log.Info("Finalizing ephemeral runner") - err := r.cleanupResources(ctx, ephemeralRunner, log) + err := r.cleanupResources(ctx, &ephemeralRunner, log) if err != nil { log.Error(err, "Failed to clean up ephemeral runner owned resources") return ctrl.Result{}, err @@ -119,7 +122,7 @@ func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Requ if ephemeralRunner.HasContainerHookConfigured() { log.Info("Runner has container hook configured, cleaning up container hook resources") - err = r.cleanupContainerHooksResources(ctx, ephemeralRunner, log) + err = r.cleanupContainerHooksResources(ctx, &ephemeralRunner, log) if err != nil { log.Error(err, "Failed to clean up container hooks resources") return ctrl.Result{}, err @@ -127,12 +130,12 @@ func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Requ } log.Info("Removing finalizer") - err = patch(ctx, r.Client, ephemeralRunner, func(obj *v1alpha1.EphemeralRunner) { - controllerutil.RemoveFinalizer(obj, ephemeralRunnerFinalizerName) - }) - if err != nil && !kerrors.IsNotFound(err) { - log.Error(err, "Failed to update ephemeral runner without the finalizer") - return ctrl.Result{}, err + if controllerutil.RemoveFinalizer(&ephemeralRunner, ephemeralRunnerFinalizerName) { + log.Info("Removed finalizer from ephemeral runner") + if err := r.Patch(ctx, &ephemeralRunner, client.MergeFrom(original)); client.IgnoreNotFound(err) != nil { + log.Error(err, "Failed to update ephemeral runner after removing finalizer") + return ctrl.Result{}, err + } } log.Info("Successfully removed finalizer after cleanup") @@ -141,7 +144,7 @@ func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Requ if ephemeralRunner.IsDone() { log.Info("Cleaning up resources after after ephemeral runner termination", "phase", ephemeralRunner.Status.Phase) - err := r.cleanupResources(ctx, ephemeralRunner, log) + err := r.cleanupResources(ctx, &ephemeralRunner, log) if err != nil { log.Error(err, "Failed to clean up ephemeral runner owned resources") return ctrl.Result{}, err @@ -153,15 +156,17 @@ func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, nil } - addFinalizers := !controllerutil.ContainsFinalizer(ephemeralRunner, ephemeralRunnerFinalizerName) || !controllerutil.ContainsFinalizer(ephemeralRunner, ephemeralRunnerActionsFinalizerName) + addFinalizers := !controllerutil.ContainsFinalizer(&ephemeralRunner, ephemeralRunnerFinalizerName) || !controllerutil.ContainsFinalizer(&ephemeralRunner, ephemeralRunnerActionsFinalizerName) if addFinalizers { log.Info("Adding finalizers") - if err := patch(ctx, r.Client, ephemeralRunner, func(obj *v1alpha1.EphemeralRunner) { - controllerutil.AddFinalizer(obj, ephemeralRunnerFinalizerName) - controllerutil.AddFinalizer(obj, ephemeralRunnerActionsFinalizerName) - }); err != nil { - log.Error(err, "Failed to update with finalizer set") - return ctrl.Result{}, err + var addedFinalizers bool + addedFinalizers = addedFinalizers || controllerutil.AddFinalizer(&ephemeralRunner, ephemeralRunnerFinalizerName) + addedFinalizers = addedFinalizers || controllerutil.AddFinalizer(&ephemeralRunner, ephemeralRunnerActionsFinalizerName) + if addedFinalizers { + if err := r.Patch(ctx, &ephemeralRunner, client.MergeFrom(original)); err != nil { + log.Error(err, "Failed to update with finalizer set") + return ctrl.Result{}, err + } } log.Info("Successfully added finalizers") } @@ -173,12 +178,12 @@ func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, err } - jitConfig, err := r.createRunnerJitConfig(ctx, ephemeralRunner, log) + jitConfig, err := r.createRunnerJitConfig(ctx, &ephemeralRunner, log) switch { case err == nil: // create secret if not created log.Info("Creating new ephemeral runner secret for jitconfig.") - jitSecret, err := r.createSecret(ctx, ephemeralRunner, jitConfig, log) + jitSecret, err := r.createSecret(ctx, &ephemeralRunner, jitConfig, log) if err != nil { return ctrl.Result{}, fmt.Errorf("failed to create secret: %w", err) } @@ -190,7 +195,7 @@ func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{Requeue: true}, nil case errors.Is(err, fatalError): log.Info("JIT config cannot be created for this ephemeral runner, issuing delete", "error", err.Error()) - if err := r.Delete(ctx, ephemeralRunner); err != nil { + if err := r.Delete(ctx, &ephemeralRunner); err != nil { return ctrl.Result{}, fmt.Errorf("failed to delete the ephemeral runner: %w", err) } log.Info("Request to delete ephemeral runner has been issued") @@ -215,20 +220,19 @@ func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Requ } runnerName := string(secret.Data["runnerName"]) - if err := patchSubResource(ctx, r.Status(), ephemeralRunner, func(obj *v1alpha1.EphemeralRunner) { - obj.Status.RunnerID = runnerID - obj.Status.RunnerName = runnerName - }); err != nil { - return ctrl.Result{}, fmt.Errorf("failed to update runner status for RunnerId/RunnerName/RunnerJITConfig: %w", err) - } + ephemeralRunner.Status.RunnerID = runnerID ephemeralRunner.Status.RunnerName = runnerName + + if err := r.Status().Patch(ctx, &ephemeralRunner, client.MergeFrom(original)); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to update runner status for RunnerId/RunnerName: %w", err) + } log.Info("Updated ephemeral runner status with runnerId and runnerName") } if len(ephemeralRunner.Status.Failures) > maxFailures { log.Info(fmt.Sprintf("EphemeralRunner has failed more than %d times. Deleting ephemeral runner so it can be re-created", maxFailures)) - if err := r.Delete(ctx, ephemeralRunner); err != nil { + if err := r.Delete(ctx, &ephemeralRunner); err != nil { log.Error(fmt.Errorf("failed to delete ephemeral runner after %d failures: %w", maxFailures, err), "Failed to delete ephemeral runner") return ctrl.Result{}, err } @@ -242,7 +246,8 @@ func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Requ nextReconciliation := lastFailure.Add(backoffDuration) if !lastFailure.IsZero() && now.Before(&metav1.Time{Time: nextReconciliation}) { requeueAfter := nextReconciliation.Sub(now.Time) - log.Info("Backing off the next reconciliation due to failure", + log.Info( + "Backing off the next reconciliation due to failure", "lastFailure", lastFailure, "nextReconciliation", nextReconciliation, "requeueAfter", requeueAfter, @@ -261,7 +266,7 @@ func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Requ } log.Info("Ephemeral runner pod does not exist. Creating new ephemeral runner") - result, err := r.createPod(ctx, ephemeralRunner, secret, log) + result, err := r.createPod(ctx, &ephemeralRunner, secret, log) switch { case err == nil: return result, nil @@ -271,7 +276,7 @@ func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Requ case kerrors.IsInvalid(err): log.Error(err, "Failed to create a pod due to unrecoverable failure") errMessage := fmt.Sprintf("Failed to create the pod: %v", err) - if err := r.markAsFailed(ctx, ephemeralRunner, errMessage, ReasonInvalidPodFailure, log); err != nil { + if err := r.markAsFailed(ctx, &ephemeralRunner, errMessage, ReasonInvalidPodFailure, log); err != nil { log.Error(err, "Failed to set ephemeral runner to phase Failed") return ctrl.Result{}, err } @@ -283,7 +288,7 @@ func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Requ switch { case isResourceQuotaExceeded && isAboutToExpire: log.Error(err, "Failed to create a pod due to resource quota exceeded and the ephemeral runner is about to expire; re-creating the ephemeral runner") - if err := r.Delete(ctx, ephemeralRunner); err != nil { + if err := r.Delete(ctx, &ephemeralRunner); err != nil { log.Error(err, "Failed to delete the ephemeral runner") return ctrl.Result{}, err } @@ -298,7 +303,7 @@ func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Requ } log.Error(err, "Failed to create a pod due to unrecoverable failure") errMessage := fmt.Sprintf("Failed to create the pod: %v", err) - if err := r.markAsFailed(ctx, ephemeralRunner, errMessage, ReasonInvalidPodFailure, log); err != nil { + if err := r.markAsFailed(ctx, &ephemeralRunner, errMessage, ReasonInvalidPodFailure, log); err != nil { log.Error(err, "Failed to set ephemeral runner to phase Failed") return ctrl.Result{}, err } @@ -312,7 +317,8 @@ func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Requ cs := runnerContainerStatus(pod) switch { case pod.Status.Phase == corev1.PodFailed: // All containers are stopped - log.Info("Pod is in failed phase, inspecting runner container status", + log.Info( + "Pod is in failed phase, inspecting runner container status", "podReason", pod.Status.Reason, "podMessage", pod.Status.Message, "podConditions", pod.Status.Conditions, @@ -321,7 +327,7 @@ func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Requ // Therefore, we should try to restart it. if cs == nil || cs.State.Terminated == nil { log.Info("Runner container does not have state set, deleting pod as failed so it can be restarted") - return ctrl.Result{}, r.deleteEphemeralRunnerOrPod(ctx, ephemeralRunner, pod, log) + return ctrl.Result{}, r.deleteEphemeralRunnerOrPod(ctx, &ephemeralRunner, pod, log) } switch cs.State.Terminated.ExitCode { @@ -331,13 +337,13 @@ func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Requ // If the runner container exits with 0, we assume that the runner has finished successfully. // If side-car container exits with non-zero, it shouldn't affect the runner. Runner exit code // drives the controller's inference of whether the job has succeeded or failed. - if err := r.Delete(ctx, ephemeralRunner); err != nil { + if err := r.Delete(ctx, &ephemeralRunner); err != nil { log.Error(err, "Failed to delete ephemeral runner after successful completion") return ctrl.Result{}, err } return ctrl.Result{}, nil case 7: - if err := r.markAsOutdated(ctx, ephemeralRunner, log); err != nil { + if err := r.markAsOutdated(ctx, &ephemeralRunner, log); err != nil { log.Error(err, "Failed to set ephemeral runner to phase Outdated") return ctrl.Result{}, err } @@ -349,13 +355,14 @@ func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Requ "Ephemeral runner container has failed, and runner container termination exit code is non-zero", "containerTerminatedState", cs.State.Terminated, ) - return ctrl.Result{}, r.deleteEphemeralRunnerOrPod(ctx, ephemeralRunner, pod, log) + return ctrl.Result{}, r.deleteEphemeralRunnerOrPod(ctx, &ephemeralRunner, pod, log) case initContainerFailed(pod): - log.Info("Pod has a failed init container, deleting pod as failed so it can be restarted", + log.Info( + "Pod has a failed init container, deleting pod as failed so it can be restarted", "initContainerStatuses", pod.Status.InitContainerStatuses, ) - return ctrl.Result{}, r.deleteEphemeralRunnerOrPod(ctx, ephemeralRunner, pod, log) + return ctrl.Result{}, r.deleteEphemeralRunnerOrPod(ctx, &ephemeralRunner, pod, log) case cs == nil: // starting, no container state yet @@ -364,14 +371,14 @@ func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Requ case cs.State.Terminated == nil: // container is not terminated and pod phase is not failed, so runner is still running log.Info("Runner container is still running; updating ephemeral runner status") - if err := r.updateRunStatusFromPod(ctx, ephemeralRunner, pod, log); err != nil { + if err := r.updateRunStatusFromPod(ctx, &ephemeralRunner, pod, log); err != nil { log.Info("Failed to update ephemeral runner status. Requeue to not miss this event") return ctrl.Result{}, err } return ctrl.Result{}, nil case cs.State.Terminated.ExitCode == 7: // outdated - if err := r.markAsOutdated(ctx, ephemeralRunner, log); err != nil { + if err := r.markAsOutdated(ctx, &ephemeralRunner, log); err != nil { log.Error(err, "Failed to set ephemeral runner to phase Outdated") return ctrl.Result{}, err } @@ -379,11 +386,11 @@ func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Requ case cs.State.Terminated.ExitCode != 0: // failed log.Info("Ephemeral runner container failed", "exitCode", cs.State.Terminated.ExitCode) - return ctrl.Result{}, r.deleteEphemeralRunnerOrPod(ctx, ephemeralRunner, pod, log) + return ctrl.Result{}, r.deleteEphemeralRunnerOrPod(ctx, &ephemeralRunner, pod, log) default: // succeeded log.Info("Ephemeral runner has finished successfully, deleting ephemeral runner", "exitCode", cs.State.Terminated.ExitCode) - if err := r.Delete(ctx, ephemeralRunner); err != nil { + if err := r.Delete(ctx, &ephemeralRunner); err != nil { log.Error(err, "Failed to delete ephemeral runner after successful completion") return ctrl.Result{}, err } @@ -568,11 +575,12 @@ func (r *EphemeralRunnerReconciler) cleanupRunnerLinkedSecrets(ctx context.Conte func (r *EphemeralRunnerReconciler) markAsFailed(ctx context.Context, ephemeralRunner *v1alpha1.EphemeralRunner, errMessage string, reason string, log logr.Logger) error { log.Info("Updating ephemeral runner status to Failed") - if err := patchSubResource(ctx, r.Status(), ephemeralRunner, func(obj *v1alpha1.EphemeralRunner) { - obj.Status.Phase = v1alpha1.EphemeralRunnerPhaseFailed - obj.Status.Reason = reason - obj.Status.Message = errMessage - }); err != nil { + + original := ephemeralRunner.DeepCopy() + ephemeralRunner.Status.Phase = v1alpha1.EphemeralRunnerPhaseFailed + ephemeralRunner.Status.Reason = reason + ephemeralRunner.Status.Message = errMessage + if err := r.Status().Patch(ctx, ephemeralRunner, client.MergeFrom(original)); err != nil { return fmt.Errorf("failed to update ephemeral runner status Phase/Message: %w", err) } @@ -588,11 +596,12 @@ func (r *EphemeralRunnerReconciler) markAsFailed(ctx context.Context, ephemeralR func (r *EphemeralRunnerReconciler) markAsOutdated(ctx context.Context, ephemeralRunner *v1alpha1.EphemeralRunner, log logr.Logger) error { log.Info("Updating ephemeral runner status to Outdated") - if err := patchSubResource(ctx, r.Status(), ephemeralRunner, func(obj *v1alpha1.EphemeralRunner) { - obj.Status.Phase = v1alpha1.EphemeralRunnerPhaseOutdated - obj.Status.Reason = "Outdated" - obj.Status.Message = "Runner is deprecated" - }); err != nil { + original := ephemeralRunner.DeepCopy() + ephemeralRunner.Status.Phase = v1alpha1.EphemeralRunnerPhaseOutdated + ephemeralRunner.Status.Reason = "Outdated" + ephemeralRunner.Status.Message = "Runner is deprecated" + + if err := r.Status().Patch(ctx, ephemeralRunner, client.MergeFrom(original)); err != nil { return fmt.Errorf("failed to update ephemeral runner status Phase/Message: %w", err) } @@ -616,16 +625,17 @@ func (r *EphemeralRunnerReconciler) deletePodAsFailed(ctx context.Context, ephem } log.Info("Updating ephemeral runner status to track the failure count") - if err := patchSubResource(ctx, r.Status(), ephemeralRunner, func(obj *v1alpha1.EphemeralRunner) { - if obj.Status.Failures == nil { - obj.Status.Failures = make(map[string]metav1.Time) - } - obj.Status.Failures[string(pod.UID)] = metav1.Now() - obj.Status.Ready = false - obj.Status.Reason = pod.Status.Reason - obj.Status.Message = pod.Status.Message - }); err != nil { - return fmt.Errorf("failed to update ephemeral runner status: failed attempts: %w", err) + original := ephemeralRunner.DeepCopy() + if ephemeralRunner.Status.Failures == nil { + ephemeralRunner.Status.Failures = make(map[string]metav1.Time) + } + ephemeralRunner.Status.Failures[string(pod.UID)] = metav1.Now() + ephemeralRunner.Status.Ready = false + ephemeralRunner.Status.Reason = pod.Status.Reason + ephemeralRunner.Status.Message = pod.Status.Message + + if err := r.Status().Patch(ctx, ephemeralRunner, client.MergeFrom(original)); err != nil { + return fmt.Errorf("failed to update ephemeral runner status with failure count: %w", err) } log.Info("EphemeralRunner pod is deleted and status is updated with failure count") @@ -744,10 +754,9 @@ func (r *EphemeralRunnerReconciler) createPod(ctx context.Context, runner *v1alp } log.Info("Creating new pod for ephemeral runner") - newPod := r.newEphemeralRunnerPod(runner, secret, envs...) - - if err := ctrl.SetControllerReference(runner, newPod, r.Scheme); err != nil { - log.Error(err, "Failed to set controller reference to a new pod") + newPod, err := r.newEphemeralRunnerPod(runner, secret, envs...) + if err != nil { + log.Error(err, "Failed to build new pod") return ctrl.Result{}, err } @@ -761,7 +770,7 @@ func (r *EphemeralRunnerReconciler) createPod(ctx context.Context, runner *v1alp "runnerScaleSetId", runner.Spec.RunnerScaleSetID, "runnerName", runner.Status.RunnerName, "runnerId", runner.Status.RunnerID, - "configUrl", runner.Spec.GitHubConfigUrl, + "configUrl", runner.Spec.GitHubConfigURL, "podName", newPod.Name) return ctrl.Result{}, nil @@ -769,10 +778,9 @@ func (r *EphemeralRunnerReconciler) createPod(ctx context.Context, runner *v1alp func (r *EphemeralRunnerReconciler) createSecret(ctx context.Context, runner *v1alpha1.EphemeralRunner, jitConfig *scaleset.RunnerScaleSetJitRunnerConfig, log logr.Logger) (*corev1.Secret, error) { log.Info("Creating new secret for ephemeral runner") - jitSecret := r.newEphemeralRunnerJitSecret(runner, jitConfig) - - if err := ctrl.SetControllerReference(runner, jitSecret, r.Scheme); err != nil { - return nil, fmt.Errorf("failed to set controller reference: %w", err) + jitSecret, err := r.newEphemeralRunnerJitSecret(runner, jitConfig) + if err != nil { + return nil, fmt.Errorf("failed to build jit secret: %w", err) } log.Info("Created new secret spec for ephemeral runner") @@ -818,13 +826,13 @@ func (r *EphemeralRunnerReconciler) updateRunStatusFromPod(ctx context.Context, "statusMessage", pod.Status.Message, "ready", ready, ) - err := patchSubResource(ctx, r.Status(), ephemeralRunner, func(obj *v1alpha1.EphemeralRunner) { - obj.Status.Phase = phase - obj.Status.Ready = ready - obj.Status.Reason = pod.Status.Reason - obj.Status.Message = pod.Status.Message - }) - if err != nil { + original := ephemeralRunner.DeepCopy() + ephemeralRunner.Status.Phase = phase + ephemeralRunner.Status.Ready = ready + ephemeralRunner.Status.Reason = pod.Status.Reason + ephemeralRunner.Status.Message = pod.Status.Message + + if err := r.Status().Patch(ctx, ephemeralRunner, client.MergeFrom(original)); err != nil { return fmt.Errorf("failed to update runner status for Phase/Reason/Message/Ready: %w", err) } @@ -850,6 +858,8 @@ func (r *EphemeralRunnerReconciler) deleteRunnerFromService(ctx context.Context, // SetupWithManager sets up the controller with the Manager. func (r *EphemeralRunnerReconciler) SetupWithManager(mgr ctrl.Manager, opts ...Option) error { + r.ResourceBuilder.setSchemeIfUnset(r.Scheme) + return builderWithOptions( ctrl.NewControllerManagedBy(mgr). For(&v1alpha1.EphemeralRunner{}). diff --git a/controllers/actions.github.com/ephemeralrunner_controller_test.go b/controllers/actions.github.com/ephemeralrunner_controller_test.go index 6b92a0ee3a..80c27134a7 100644 --- a/controllers/actions.github.com/ephemeralrunner_controller_test.go +++ b/controllers/actions.github.com/ephemeralrunner_controller_test.go @@ -41,7 +41,7 @@ func newExampleRunner(name, namespace, configSecretName string) *v1alpha1.Epheme Namespace: namespace, }, Spec: v1alpha1.EphemeralRunnerSpec{ - GitHubConfigUrl: "https://github.com/owner/repo", + GitHubConfigURL: "https://github.com/owner/repo", GitHubConfigSecret: configSecretName, RunnerScaleSetID: 1, PodTemplateSpec: corev1.PodTemplateSpec{ @@ -198,12 +198,13 @@ var _ = Describe("EphemeralRunner", func() { Expect(err).To(BeNil(), "failed to delete pod") pod = new(corev1.Pod) - Eventually(func() (bool, error) { - if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod); err != nil { - return false, err - } - return true, nil - }, + Eventually( + func() (bool, error) { + if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod); err != nil { + return false, err + } + return true, nil + }, ephemeralRunnerTimeout, ephemeralRunnerInterval, ).Should(BeEquivalentTo(true)) @@ -547,9 +548,10 @@ var _ = Describe("EphemeralRunner", func() { It("It should mark as failed when job is not assigned and pod is failed", func() { er := new(v1alpha1.EphemeralRunner) - Eventually(func() error { - return k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, er) - }, + Eventually( + func() error { + return k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, er) + }, ephemeralRunnerTimeout, ephemeralRunnerInterval, ).Should(Succeed(), "failed to get ephemeral runner") @@ -1362,7 +1364,7 @@ var _ = Describe("EphemeralRunner", func() { Expect(err).NotTo(HaveOccurred(), "failed to create secret credentials") ephemeralRunner := newExampleRunner("test-runner", autoScalingNS.Name, configSecret.Name) - ephemeralRunner.Spec.GitHubConfigUrl = "http://example.com/org/repo" + ephemeralRunner.Spec.GitHubConfigURL = "http://example.com/org/repo" ephemeralRunner.Spec.Proxy = &v1alpha1.ProxyConfig{ HTTP: &v1alpha1.ProxyServerConfig{ Url: proxy.URL, @@ -1524,7 +1526,7 @@ var _ = Describe("EphemeralRunner", func() { } ephemeralRunner := newExampleRunner("test-runner", autoScalingNS.Name, configSecret.Name) - ephemeralRunner.Spec.GitHubConfigUrl = server.URL + "/my-org" + ephemeralRunner.Spec.GitHubConfigURL = server.URL + "/my-org" ephemeralRunner.Spec.GitHubServerTLS = &v1alpha1.TLSConfig{ CertificateFrom: &v1alpha1.TLSCertificateSource{ ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ diff --git a/controllers/actions.github.com/ephemeralrunnerset_controller.go b/controllers/actions.github.com/ephemeralrunnerset_controller.go index acfcba5765..5ed40400d3 100644 --- a/controllers/actions.github.com/ephemeralrunnerset_controller.go +++ b/controllers/actions.github.com/ephemeralrunnerset_controller.go @@ -20,8 +20,10 @@ import ( "context" "errors" "fmt" + "maps" "sort" "strconv" + "time" "github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1" "github.com/actions/actions-runner-controller/controllers/actions.github.com/metrics" @@ -32,7 +34,6 @@ import ( "go.uber.org/multierr" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" @@ -42,17 +43,16 @@ import ( ) const ( - ephemeralRunnerSetFinalizerName = "ephemeralrunner.actions.github.com/finalizer" + // EphemeralRunnerSetFinalizerName is the finalizer name used in EphemeralRunnerSet resource to protect the cleanup process of the child ephemeral runners and proxy secret. + EphemeralRunnerSetFinalizerName = "ephemeralrunnerset.actions.github.com/finalizer" ) // EphemeralRunnerSetReconciler reconciles a EphemeralRunnerSet object type EphemeralRunnerSetReconciler struct { client.Client - Log logr.Logger - Scheme *runtime.Scheme - + Log logr.Logger + Scheme *runtime.Scheme PublishMetrics bool - ResourceBuilder } @@ -77,19 +77,20 @@ type EphemeralRunnerSetReconciler struct { func (r *EphemeralRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.Log.WithValues("ephemeralrunnerset", req.NamespacedName) - ephemeralRunnerSet := new(v1alpha1.EphemeralRunnerSet) - if err := r.Get(ctx, req.NamespacedName, ephemeralRunnerSet); err != nil { + var ephemeralRunnerSet v1alpha1.EphemeralRunnerSet + if err := r.Get(ctx, req.NamespacedName, &ephemeralRunnerSet); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } + original := ephemeralRunnerSet.DeepCopy() // Requested deletion does not need reconciled. if !ephemeralRunnerSet.DeletionTimestamp.IsZero() { - if !controllerutil.ContainsFinalizer(ephemeralRunnerSet, ephemeralRunnerSetFinalizerName) { + if !controllerutil.ContainsFinalizer(&ephemeralRunnerSet, EphemeralRunnerSetFinalizerName) { return ctrl.Result{}, nil } log.Info("Deleting resources") - done, err := r.cleanUpEphemeralRunners(ctx, ephemeralRunnerSet, log) + done, err := r.cleanUpEphemeralRunners(ctx, &ephemeralRunnerSet, log) if err != nil { log.Error(err, "Failed to clean up EphemeralRunners") return ctrl.Result{}, err @@ -99,25 +100,33 @@ func (r *EphemeralRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.R return ctrl.Result{}, nil } - log.Info("Removing finalizer") - if err := patch(ctx, r.Client, ephemeralRunnerSet, func(obj *v1alpha1.EphemeralRunnerSet) { - controllerutil.RemoveFinalizer(obj, ephemeralRunnerSetFinalizerName) - }); err != nil && !kerrors.IsNotFound(err) { - log.Error(err, "Failed to update ephemeral runner set with removed finalizer") + done, err = r.cleanUpEphemeralRunnerSetProxySecret(ctx, &ephemeralRunnerSet, log) + if err != nil { + log.Error(err, "Failed to clean up EphemeralRunnerSet proxy secret") return ctrl.Result{}, err } + if !done { + log.Info("Waiting for proxy secret to be deleted") + return ctrl.Result{RequeueAfter: 1 * time.Second}, nil + } + + log.Info("Removing finalizer") + if controllerutil.RemoveFinalizer(&ephemeralRunnerSet, EphemeralRunnerSetFinalizerName) { + if err := r.Patch(ctx, &ephemeralRunnerSet, client.MergeFrom(original)); err != nil { + log.Error(err, "Failed to update ephemeral runner set with removed finalizer") + return ctrl.Result{}, err + } + } log.Info("Successfully removed finalizer after cleanup") return ctrl.Result{}, nil } // Add finalizer if not present - if !controllerutil.ContainsFinalizer(ephemeralRunnerSet, ephemeralRunnerSetFinalizerName) { + if controllerutil.AddFinalizer(&ephemeralRunnerSet, EphemeralRunnerSetFinalizerName) { log.Info("Adding finalizer") - if err := patch(ctx, r.Client, ephemeralRunnerSet, func(obj *v1alpha1.EphemeralRunnerSet) { - controllerutil.AddFinalizer(obj, ephemeralRunnerSetFinalizerName) - }); err != nil { - log.Error(err, "Failed to update ephemeral runner set with finalizer added") + if err := r.Patch(ctx, &ephemeralRunnerSet, client.MergeFrom(original)); err != nil { + log.Error(err, "Failed to update ephemeral runner set with new finalizer") return ctrl.Result{}, err } @@ -125,34 +134,100 @@ func (r *EphemeralRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.R return ctrl.Result{}, nil } + // If hash spec has changed, delete idle ephemeral runners + // in order to apply the change to the runners that did not yet receive a job. + ephemeralRunnerIntegrityHash := ephemeralRunnerSetIntegrityHash(&ephemeralRunnerSet) + if ephemeralRunnerSet.Annotations[annotationKeyIntegrityHash] != ephemeralRunnerIntegrityHash { + log.Info("EphemeralRunnerSpec has changed, deleting idle ephemeral runners to apply the new spec") + if _, err := r.cleanUpEphemeralRunners(ctx, &ephemeralRunnerSet, log); err != nil { + log.Error(err, "Failed to clean up EphemeralRunners") + return ctrl.Result{}, err + } + + if _, err := r.cleanUpEphemeralRunnerSetProxySecret(ctx, &ephemeralRunnerSet, log); err != nil { + log.Error(err, "Failed to clean up EphemeralRunnerSet proxy secret") + return ctrl.Result{}, err + } + + log.Info("Updating EphemeralRunnerSet with new spec hash") + original := ephemeralRunnerSet.DeepCopy() + if ephemeralRunnerSet.Annotations == nil { + ephemeralRunnerSet.Annotations = make(map[string]string) + } + ephemeralRunnerSet.Annotations[annotationKeyIntegrityHash] = ephemeralRunnerIntegrityHash + if err := r.Patch(ctx, &ephemeralRunnerSet, client.MergeFrom(original)); err != nil { + log.Error(err, "Failed to update ephemeral runner set with new spec hash") + return ctrl.Result{}, err + } + + log.Info("Updated ephemeral runner set with new spec hash") + return ctrl.Result{}, nil + } + if ephemeralRunnerSet.Status.Phase == v1alpha1.EphemeralRunnerSetPhaseOutdated { - log.Info("ephemeral runner set is outdated, waiting for autoscaling runner set to remove it") + if _, err := r.cleanUpEphemeralRunners(ctx, &ephemeralRunnerSet, log); err != nil { + log.Error(err, "Failed to clean up EphemeralRunners") + return ctrl.Result{}, err + } return ctrl.Result{}, nil } // Create proxy secret if not present if ephemeralRunnerSet.Spec.EphemeralRunnerSpec.Proxy != nil { - proxySecret := new(corev1.Secret) - if err := r.Get(ctx, types.NamespacedName{Namespace: ephemeralRunnerSet.Namespace, Name: proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet)}, proxySecret); err != nil { - if !kerrors.IsNotFound(err) { - log.Error(err, "Unable to get ephemeralRunnerSet proxy secret", "namespace", ephemeralRunnerSet.Namespace, "name", proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet)) + var proxySecret corev1.Secret + err := r.Get( + ctx, + types.NamespacedName{ + Namespace: ephemeralRunnerSet.Namespace, + Name: proxyEphemeralRunnerSetSecretName(&ephemeralRunnerSet), + }, + &proxySecret, + ) + switch { + case err == nil: + desiredRunnerSetProxy, err := r.newEphemeralRunnerSetProxySecret(&ephemeralRunnerSet, proxySecret.Data) + if err != nil { + log.Error(err, "Failed to build desired ephemeralRunnerSet proxy secret") return ctrl.Result{}, err } - + updatedProxySecret := proxySecret.DeepCopy() + var shouldUpdate bool + desiredLabels := r.filterAndMergeLabels(proxySecret.Labels, desiredRunnerSetProxy.Labels) + if !maps.Equal(proxySecret.Labels, desiredLabels) { + updatedProxySecret.Labels = desiredLabels + shouldUpdate = true + } + desiredAnnotations := r.mergeAnnotations(proxySecret.Annotations, desiredRunnerSetProxy.Annotations) + if !maps.Equal(proxySecret.Annotations, desiredAnnotations) { + updatedProxySecret.Annotations = desiredAnnotations + shouldUpdate = true + } + if shouldUpdate { + log.Info("Updating ephemeralRunnerSet proxy secret") + if err := r.Update(ctx, updatedProxySecret); err != nil { + log.Error(err, "Failed to update ephemeralRunnerSet proxy secret") + return ctrl.Result{}, err + } + return ctrl.Result{Requeue: true}, nil + } + case kerrors.IsNotFound(err): // Create a compiled secret for the runner pods in the runnerset namespace log.Info("Creating a ephemeralRunnerSet proxy secret for the runner pods") - if err := r.createProxySecret(ctx, ephemeralRunnerSet, log); err != nil { + if err := r.createProxySecret(ctx, &ephemeralRunnerSet, log); err != nil { log.Error(err, "Unable to create ephemeralRunnerSet proxy secret", "namespace", ephemeralRunnerSet.Namespace, "set-name", ephemeralRunnerSet.Name) return ctrl.Result{}, err } + default: + log.Error(err, "Unable to get ephemeralRunnerSet proxy secret", "namespace", ephemeralRunnerSet.Namespace, "name", proxyEphemeralRunnerSetSecretName(&ephemeralRunnerSet)) + return ctrl.Result{}, err } } // Find all EphemeralRunner with matching namespace and own by this EphemeralRunnerSet. - ephemeralRunnerList := new(v1alpha1.EphemeralRunnerList) + var ephemeralRunnerList v1alpha1.EphemeralRunnerList if err := r.List( ctx, - ephemeralRunnerList, + &ephemeralRunnerList, client.InNamespace(req.Namespace), client.MatchingFields{resourceOwnerKey: req.Name}, ); err != nil { @@ -160,9 +235,10 @@ func (r *EphemeralRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.R return ctrl.Result{}, err } - ephemeralRunnersByState := newEphemeralRunnersByStates(ephemeralRunnerList) + ephemeralRunnersByState := newEphemeralRunnersByStates(&ephemeralRunnerList) - log.Info("Ephemeral runner counts", + log.Info( + "Ephemeral runner counts", "outdated", len(ephemeralRunnersByState.outdated), "pending", len(ephemeralRunnersByState.pending), "running", len(ephemeralRunnersByState.running), @@ -172,7 +248,7 @@ func (r *EphemeralRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.R ) if r.PublishMetrics { - githubConfigURL := ephemeralRunnerSet.Spec.EphemeralRunnerSpec.GitHubConfigUrl + githubConfigURL := ephemeralRunnerSet.Spec.EphemeralRunnerSpec.GitHubConfigURL parsedURL, err := actions.ParseGitHubConfigFromURL(githubConfigURL) if err != nil { log.Error(err, "Github Config URL is invalid", "URL", githubConfigURL) @@ -206,7 +282,7 @@ func (r *EphemeralRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.R case total < ephemeralRunnerSet.Spec.Replicas: // Handle scale up count := ephemeralRunnerSet.Spec.Replicas - total log.Info("Creating new ephemeral runners (scale up)", "count", count) - if err := r.createEphemeralRunners(ctx, ephemeralRunnerSet, count, log); err != nil { + if err := r.createEphemeralRunners(ctx, &ephemeralRunnerSet, count, log); err != nil { log.Error(err, "failed to make ephemeral runner") return ctrl.Result{}, err } @@ -221,7 +297,7 @@ func (r *EphemeralRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.R log.Info("Deleting ephemeral runners (scale down)", "count", count) if err := r.deleteIdleEphemeralRunners( ctx, - ephemeralRunnerSet, + &ephemeralRunnerSet, ephemeralRunnersByState.pending, ephemeralRunnersByState.running, count, @@ -233,10 +309,11 @@ func (r *EphemeralRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.R } } - return ctrl.Result{}, r.updateStatus(ctx, ephemeralRunnerSet, ephemeralRunnersByState, log) + return ctrl.Result{}, r.updateStatus(ctx, &ephemeralRunnerSet, ephemeralRunnersByState, log) } func (r *EphemeralRunnerSetReconciler) updateStatus(ctx context.Context, ephemeralRunnerSet *v1alpha1.EphemeralRunnerSet, state *ephemeralRunnersByState, log logr.Logger) error { + original := ephemeralRunnerSet.DeepCopy() total := state.scaleTotal() var phase v1alpha1.EphemeralRunnerSetPhase switch { @@ -257,15 +334,13 @@ func (r *EphemeralRunnerSetReconciler) updateStatus(ctx context.Context, ephemer // Update the status if needed. if ephemeralRunnerSet.Status != desiredStatus { - log.Info("Updating status with current runners count", "count", total) - ephemeralRunnerSet := ephemeralRunnerSet.DeepCopy() - ephemeralRunnerSet.Status.CurrentReplicas = -1 // ALWAYS update current replicas - if err := patchSubResource(ctx, r.Status(), ephemeralRunnerSet, func(obj *v1alpha1.EphemeralRunnerSet) { - obj.Status = desiredStatus - }); err != nil { - log.Error(err, "Failed to update status with current runners count") + ephemeralRunnerSet.Status = desiredStatus + if err := r.Status().Patch(ctx, ephemeralRunnerSet, client.MergeFrom(original)); err != nil { + log.Error(err, "Failed to update EphemeralRunnerSet status") return err } + log.Info("Updated EphemeralRunnerSet status", "status", ephemeralRunnerSet.Status) + } return nil } @@ -323,7 +398,8 @@ func (r *EphemeralRunnerSetReconciler) cleanUpEphemeralRunners(ctx context.Conte ephemeralRunnerState := newEphemeralRunnersByStates(ephemeralRunnerList) - log.Info("Clean up runner counts", + log.Info( + "Clean up runner counts", "pending", len(ephemeralRunnerState.pending), "running", len(ephemeralRunnerState.running), "finished", len(ephemeralRunnerState.finished), @@ -359,7 +435,7 @@ func (r *EphemeralRunnerSetReconciler) cleanUpEphemeralRunners(ctx context.Conte log.Info("Cleanup pending or running ephemeral runners") errs = errs[0:0] - for _, ephemeralRunner := range append(ephemeralRunnerState.pending, ephemeralRunnerState.running...) { + for _, ephemeralRunner := range ephemeralRunnerState.pending { log.Info("Removing the ephemeral runner from the service", "name", ephemeralRunner.Name) _, err := r.deleteEphemeralRunnerWithActionsClient(ctx, ephemeralRunner, actionsClient, log) if err != nil { @@ -367,6 +443,24 @@ func (r *EphemeralRunnerSetReconciler) cleanUpEphemeralRunners(ctx context.Conte } } + for _, ephemeralRunner := range ephemeralRunnerState.running { + if ephemeralRunner.HasJob() { + log.Info( + "Skipping ephemeral runner since it is running a job", + "name", ephemeralRunner.Name, + "workflowRunId", ephemeralRunner.Status.WorkflowRunID, + "jobId", ephemeralRunner.Status.JobID, + ) + continue + } + + log.Info("Removing the idle ephemeral runner from the service", "name", ephemeralRunner.Name) + _, err := r.deleteEphemeralRunnerWithActionsClient(ctx, ephemeralRunner, actionsClient, log) + if err != nil { + errs = append(errs, err) + } + } + if len(errs) > 0 { mergedErrs := multierr.Combine(errs...) log.Error(mergedErrs, "Failed to remove ephemeral runners from the service") @@ -376,22 +470,58 @@ func (r *EphemeralRunnerSetReconciler) cleanUpEphemeralRunners(ctx context.Conte return false, nil } +func (r *EphemeralRunnerSetReconciler) cleanUpEphemeralRunnerSetProxySecret(ctx context.Context, ephemeralRunnerSet *v1alpha1.EphemeralRunnerSet, log logr.Logger) (done bool, err error) { + if ephemeralRunnerSet.Spec.EphemeralRunnerSpec.Proxy == nil { + return true, nil + } + var proxySecret corev1.Secret + err = r.Get( + ctx, + types.NamespacedName{ + Namespace: ephemeralRunnerSet.Namespace, + Name: proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet), + }, + &proxySecret, + ) + switch { + case err == nil: + log.Info("Deleting proxy secret") + if err := r.Delete(ctx, &proxySecret); err != nil && !kerrors.IsNotFound(err) { + log.Error(err, "Failed to delete proxy secret") + return false, err + } + log.Info("Deleted proxy secret") + return false, nil + case kerrors.IsNotFound(err): + log.Info("Proxy secret already deleted") + return true, nil + default: + log.Error( + err, + "Unable to get ephemeralRunnerSet proxy secret", + "namespace", + ephemeralRunnerSet.Namespace, + "name", + proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet), + ) + return false, err + } +} + // createEphemeralRunners provisions `count` number of v1alpha1.EphemeralRunner resources in the cluster. func (r *EphemeralRunnerSetReconciler) createEphemeralRunners(ctx context.Context, runnerSet *v1alpha1.EphemeralRunnerSet, count int, log logr.Logger) error { // Track multiple errors at once and return the bundle. errs := make([]error, 0) for i := range count { - ephemeralRunner := r.newEphemeralRunner(runnerSet) - if runnerSet.Spec.EphemeralRunnerSpec.Proxy != nil { - ephemeralRunner.Spec.ProxySecretRef = proxyEphemeralRunnerSetSecretName(runnerSet) - } - - // Make sure that we own the resource we create. - if err := ctrl.SetControllerReference(runnerSet, ephemeralRunner, r.Scheme); err != nil { - log.Error(err, "failed to set controller reference on ephemeral runner") + ephemeralRunner, err := r.newEphemeralRunner(runnerSet) + if err != nil { + log.Error(err, "failed to build ephemeral runner") errs = append(errs, err) continue } + if runnerSet.Spec.EphemeralRunnerSpec.Proxy != nil { + ephemeralRunner.Spec.ProxySecretRef = proxyEphemeralRunnerSetSecretName(runnerSet) + } log.Info("Creating new ephemeral runner", "progress", i+1, "total", count) if err := r.Create(ctx, ephemeralRunner); err != nil { @@ -416,21 +546,9 @@ func (r *EphemeralRunnerSetReconciler) createProxySecret(ctx context.Context, ep return fmt.Errorf("failed to convert proxy config to secret data: %w", err) } - runnerPodProxySecret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet), - Namespace: ephemeralRunnerSet.Namespace, - Labels: map[string]string{ - LabelKeyGitHubScaleSetName: ephemeralRunnerSet.Labels[LabelKeyGitHubScaleSetName], - LabelKeyGitHubScaleSetNamespace: ephemeralRunnerSet.Labels[LabelKeyGitHubScaleSetNamespace], - }, - }, - Data: proxySecretData, - } - - // Make sure that we own the resource we create. - if err := ctrl.SetControllerReference(ephemeralRunnerSet, runnerPodProxySecret, r.Scheme); err != nil { - log.Error(err, "failed to set controller reference on proxy secret") + runnerPodProxySecret, err := r.newEphemeralRunnerSetProxySecret(ephemeralRunnerSet, proxySecretData) + if err != nil { + log.Error(err, "failed to build proxy secret") return err } @@ -523,6 +641,8 @@ func (r *EphemeralRunnerSetReconciler) deleteEphemeralRunnerWithActionsClient(ct // SetupWithManager sets up the controller with the Manager. func (r *EphemeralRunnerSetReconciler) SetupWithManager(mgr ctrl.Manager, opts ...Option) error { + r.setSchemeIfUnset(r.Scheme) + return builderWithOptions( ctrl.NewControllerManagedBy(mgr). For(&v1alpha1.EphemeralRunnerSet{}). diff --git a/controllers/actions.github.com/ephemeralrunnerset_controller_test.go b/controllers/actions.github.com/ephemeralrunnerset_controller_test.go index 4a0a41b378..c345965256 100644 --- a/controllers/actions.github.com/ephemeralrunnerset_controller_test.go +++ b/controllers/actions.github.com/ephemeralrunnerset_controller_test.go @@ -71,10 +71,13 @@ var _ = Describe("Test EphemeralRunnerSet controller", func() { ObjectMeta: metav1.ObjectMeta{ Name: "test-asrs", Namespace: autoscalingNS.Name, + Annotations: map[string]string{ + "arc.test/runner-set-annotation": "initial", + }, }, Spec: v1alpha1.EphemeralRunnerSetSpec{ EphemeralRunnerSpec: v1alpha1.EphemeralRunnerSpec{ - GitHubConfigUrl: "https://github.com/owner/repo", + GitHubConfigURL: "https://github.com/owner/repo", GitHubConfigSecret: configSecret.Name, RunnerScaleSetID: 100, PodTemplateSpec: corev1.PodTemplateSpec{ @@ -113,7 +116,8 @@ var _ = Describe("Test EphemeralRunnerSet controller", func() { return created.Finalizers[0], nil }, ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(ephemeralRunnerSetFinalizerName), "EphemeralRunnerSet should have a finalizer") + ephemeralRunnerSetTestInterval, + ).Should(BeEquivalentTo(EphemeralRunnerSetFinalizerName), "EphemeralRunnerSet should have a finalizer") // Check if the number of ephemeral runners are stay 0 Consistently( @@ -126,7 +130,8 @@ var _ = Describe("Test EphemeralRunnerSet controller", func() { return len(runnerList.Items), nil }, ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(0), "No EphemeralRunner should be created") + ephemeralRunnerSetTestInterval, + ).Should(BeEquivalentTo(0), "No EphemeralRunner should be created") // Check if the status stay 0 Consistently( @@ -140,7 +145,8 @@ var _ = Describe("Test EphemeralRunnerSet controller", func() { return int(runnerSet.Status.CurrentReplicas), nil }, ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(0), "EphemeralRunnerSet status should be 0") + ephemeralRunnerSetTestInterval, + ).Should(BeEquivalentTo(0), "EphemeralRunnerSet status should be 0") // Scaling up the EphemeralRunnerSet updated := created.DeepCopy() @@ -178,7 +184,8 @@ var _ = Describe("Test EphemeralRunnerSet controller", func() { return len(runnerList.Items), nil }, ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(5), "5 EphemeralRunner should be created") + ephemeralRunnerSetTestInterval, + ).Should(BeEquivalentTo(5), "5 EphemeralRunner should be created") // Check if the status is updated Eventually( @@ -192,7 +199,8 @@ var _ = Describe("Test EphemeralRunnerSet controller", func() { return int(runnerSet.Status.CurrentReplicas), nil }, ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(5), "EphemeralRunnerSet status should be 5") + ephemeralRunnerSetTestInterval, + ).Should(BeEquivalentTo(5), "EphemeralRunnerSet status should be 5") }) }) @@ -238,7 +246,8 @@ var _ = Describe("Test EphemeralRunnerSet controller", func() { return len(runnerList.Items), nil }, ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(5), "5 EphemeralRunner should be created") + ephemeralRunnerSetTestInterval, + ).Should(BeEquivalentTo(5), "5 EphemeralRunner should be created") // Delete the EphemeralRunnerSet err = k8sClient.Delete(ctx, created) @@ -255,7 +264,8 @@ var _ = Describe("Test EphemeralRunnerSet controller", func() { return len(runnerList.Items), nil }, ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(0), "All EphemeralRunner should be deleted") + ephemeralRunnerSetTestInterval, + ).Should(BeEquivalentTo(0), "All EphemeralRunner should be deleted") // Check if the EphemeralRunnerSet is deleted Eventually( @@ -273,7 +283,8 @@ var _ = Describe("Test EphemeralRunnerSet controller", func() { return fmt.Errorf("EphemeralRunnerSet is not deleted") }, ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval).Should(Succeed(), "EphemeralRunnerSet should be deleted") + ephemeralRunnerSetTestInterval, + ).Should(Succeed(), "EphemeralRunnerSet should be deleted") }) }) @@ -304,6 +315,64 @@ var _ = Describe("Test EphemeralRunnerSet controller", func() { ).Should(BeEquivalentTo(5), "5 EphemeralRunner should be created") }) + It("propagates updated EphemeralRunnerSet annotations to newly created EphemeralRunners", func() { + ers := new(v1alpha1.EphemeralRunnerSet) + err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, ers) + Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet") + + updated := ers.DeepCopy() + updated.Spec.Replicas = 1 + updated.Spec.PatchID = 0 + err = k8sClient.Patch(ctx, updated, client.MergeFrom(ers)) + Expect(err).NotTo(HaveOccurred(), "failed to scale EphemeralRunnerSet") + + runnerList := new(v1alpha1.EphemeralRunnerList) + Eventually( + func(g Gomega) { + err := listEphemeralRunnersAndRemoveFinalizers(ctx, k8sClient, runnerList, ephemeralRunnerSet.Namespace) + g.Expect(err).NotTo(HaveOccurred(), "failed to list EphemeralRunners") + g.Expect(runnerList.Items).To(HaveLen(1)) + g.Expect(runnerList.Items[0].Annotations["arc.test/runner-set-annotation"]).To(Equal("initial")) + }, + ephemeralRunnerSetTestTimeout, + ephemeralRunnerSetTestInterval, + ).Should(Succeed()) + + ers = new(v1alpha1.EphemeralRunnerSet) + err = k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, ers) + Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet") + + updated = ers.DeepCopy() + updated.Annotations["arc.test/runner-set-annotation"] = "updated" + updated.Annotations["arc.test/new-runner-set-annotation"] = "added" + updated.Spec.Replicas = 2 + updated.Spec.PatchID = 1 + err = k8sClient.Patch(ctx, updated, client.MergeFrom(ers)) + Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet annotations") + + Eventually( + func(g Gomega) { + err := listEphemeralRunnersAndRemoveFinalizers(ctx, k8sClient, runnerList, ephemeralRunnerSet.Namespace) + g.Expect(err).NotTo(HaveOccurred(), "failed to list EphemeralRunners") + g.Expect(runnerList.Items).To(HaveLen(2)) + + annotationsByValue := map[string]int{} + var updatedRunnerHasNewAnnotation bool + for _, runner := range runnerList.Items { + annotationsByValue[runner.Annotations["arc.test/runner-set-annotation"]]++ + if runner.Annotations["arc.test/runner-set-annotation"] == "updated" && runner.Annotations["arc.test/new-runner-set-annotation"] == "added" { + updatedRunnerHasNewAnnotation = true + } + } + g.Expect(annotationsByValue["initial"]).To(Equal(1)) + g.Expect(annotationsByValue["updated"]).To(Equal(1)) + g.Expect(updatedRunnerHasNewAnnotation).To(BeTrue()) + }, + ephemeralRunnerSetTestTimeout, + ephemeralRunnerSetTestInterval, + ).Should(Succeed()) + }) + It("Should scale up when patch ID changes", func() { ers := new(v1alpha1.EphemeralRunnerSet) err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, ers) @@ -910,6 +979,115 @@ var _ = Describe("Test EphemeralRunnerSet controller", func() { ).Should(BeNil(), "2 EphemeralRunner should be created and none should be in Succeeded phase") }) + It("Should delete idle runners, keep busy runners, and create new runners when the spec changes", func() { + ers := new(v1alpha1.EphemeralRunnerSet) + err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, ers) + Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet") + + updated := ers.DeepCopy() + updated.Spec.Replicas = 3 + updated.Spec.PatchID = 0 + err = k8sClient.Patch(ctx, updated, client.MergeFrom(ers)) + Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet") + + runnerList := new(v1alpha1.EphemeralRunnerList) + Eventually( + func() (int, error) { + err := listEphemeralRunnersAndRemoveFinalizers(ctx, k8sClient, runnerList, ephemeralRunnerSet.Namespace) + if err != nil { + return -1, err + } + + return len(runnerList.Items), nil + }, + ephemeralRunnerSetTestTimeout, + ephemeralRunnerSetTestInterval, + ).Should(BeEquivalentTo(3), "3 EphemeralRunner should be created") + + idleRunnerNames := map[string]struct{}{} + for i := 0; i < 2; i++ { + idleRunner := runnerList.Items[i].DeepCopy() + idleRunner.Status.Phase = v1alpha1.EphemeralRunnerPhaseRunning + idleRunner.Status.RunnerID = i + 101 + err = k8sClient.Status().Patch(ctx, idleRunner, client.MergeFrom(&runnerList.Items[i])) + Expect(err).NotTo(HaveOccurred(), "failed to update idle EphemeralRunner") + idleRunnerNames[idleRunner.Name] = struct{}{} + } + + busyRunner := runnerList.Items[2].DeepCopy() + busyRunner.Status.Phase = v1alpha1.EphemeralRunnerPhaseRunning + busyRunner.Status.RunnerID = 103 + busyRunner.Status.JobID = "job-1" + busyRunner.Status.WorkflowRunID = 9001 + err = k8sClient.Status().Patch(ctx, busyRunner, client.MergeFrom(&runnerList.Items[2])) + Expect(err).NotTo(HaveOccurred(), "failed to update busy EphemeralRunner") + + busyRunnerName := busyRunner.Name + + ers = new(v1alpha1.EphemeralRunnerSet) + err = k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, ers) + Expect(err).NotTo(HaveOccurred(), "failed to re-fetch EphemeralRunnerSet") + + updated = ers.DeepCopy() + updated.Spec.EphemeralRunnerSpec.PodTemplateSpec.Spec.Containers[0].Image = "ghcr.io/actions/runner:new" + err = k8sClient.Patch(ctx, updated, client.MergeFrom(ers)) + Expect(err).NotTo(HaveOccurred(), "failed to patch EphemeralRunnerSet with new spec") + + Eventually( + func() error { + err := listEphemeralRunnersAndRemoveFinalizers(ctx, k8sClient, runnerList, ephemeralRunnerSet.Namespace) + if err != nil { + return err + } + + if len(runnerList.Items) != 3 { + return fmt.Errorf("expected 3 runners after spec update, got %d", len(runnerList.Items)) + } + + busyRunnerFound := false + newSpecRunnerCount := 0 + for _, runner := range runnerList.Items { + if _, ok := idleRunnerNames[runner.Name]; ok { + return fmt.Errorf("expected idle runner %s to be deleted", runner.Name) + } + + if runner.Name == busyRunnerName { + busyRunnerFound = true + if !runner.HasJob() { + return fmt.Errorf("expected remaining runner to still be busy") + } + if runner.Spec.PodTemplateSpec.Spec.Containers[0].Image != "ghcr.io/actions/runner" { + return fmt.Errorf("expected busy runner to keep original image, got %s", runner.Spec.PodTemplateSpec.Spec.Containers[0].Image) + } + continue + } + + if len(runner.Spec.PodTemplateSpec.Spec.Containers) == 0 { + return fmt.Errorf("new runner has empty container spec") + } + + if runner.Spec.PodTemplateSpec.Spec.Containers[0].Image != "ghcr.io/actions/runner:new" { + return fmt.Errorf("expected new runner image to be updated, got %s", runner.Spec.PodTemplateSpec.Spec.Containers[0].Image) + } + + newSpecRunnerCount++ + } + + if !busyRunnerFound { + return fmt.Errorf("expected busy runner %s to remain", busyRunnerName) + } + + if newSpecRunnerCount != 2 { + return fmt.Errorf("expected 2 runners with updated spec, got %d", newSpecRunnerCount) + } + + return nil + }, + ephemeralRunnerSetTestTimeout, + ephemeralRunnerSetTestInterval, + ).Should(BeNil(), "busy runner should stay while idle runners are replaced with the updated spec") + }) + It("Should update status on Ephemeral Runner state changes", func() { created := new(v1alpha1.EphemeralRunnerSet) Eventually( @@ -1144,7 +1322,7 @@ var _ = Describe("Test EphemeralRunnerSet controller with proxy settings", func( Spec: v1alpha1.EphemeralRunnerSetSpec{ Replicas: 1, EphemeralRunnerSpec: v1alpha1.EphemeralRunnerSpec{ - GitHubConfigUrl: "http://example.com/owner/repo", + GitHubConfigURL: "http://example.com/owner/repo", GitHubConfigSecret: configSecret.Name, RunnerScaleSetID: 100, Proxy: &v1alpha1.ProxyConfig{ @@ -1175,29 +1353,30 @@ var _ = Describe("Test EphemeralRunnerSet controller with proxy settings", func( err = k8sClient.Create(ctx, ephemeralRunnerSet) Expect(err).NotTo(HaveOccurred(), "failed to create EphemeralRunnerSet") - Eventually(func(g Gomega) { - // Compiled / flattened proxy secret should exist at this point - actualProxySecret := &corev1.Secret{} - err = k8sClient.Get(ctx, client.ObjectKey{ - Namespace: autoscalingNS.Name, - Name: proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet), - }, actualProxySecret) - g.Expect(err).NotTo(HaveOccurred(), "failed to get compiled / flattened proxy secret") - - secretFetcher := func(name string) (*corev1.Secret, error) { - secret := &corev1.Secret{} + Eventually( + func(g Gomega) { + // Compiled / flattened proxy secret should exist at this point + actualProxySecret := &corev1.Secret{} err = k8sClient.Get(ctx, client.ObjectKey{ Namespace: autoscalingNS.Name, - Name: name, - }, secret) - return secret, err - } + Name: proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet), + }, actualProxySecret) + g.Expect(err).NotTo(HaveOccurred(), "failed to get compiled / flattened proxy secret") + + secretFetcher := func(name string) (*corev1.Secret, error) { + secret := &corev1.Secret{} + err = k8sClient.Get(ctx, client.ObjectKey{ + Namespace: autoscalingNS.Name, + Name: name, + }, secret) + return secret, err + } - // Assert that the proxy secret is created with the correct values - expectedData, err := ephemeralRunnerSet.Spec.EphemeralRunnerSpec.Proxy.ToSecretData(secretFetcher) - g.Expect(err).NotTo(HaveOccurred(), "failed to get proxy secret data") - g.Expect(actualProxySecret.Data).To(Equal(expectedData)) - }, + // Assert that the proxy secret is created with the correct values + expectedData, err := ephemeralRunnerSet.Spec.EphemeralRunnerSpec.Proxy.ToSecretData(secretFetcher) + g.Expect(err).NotTo(HaveOccurred(), "failed to get proxy secret data") + g.Expect(actualProxySecret.Data).To(Equal(expectedData)) + }, ephemeralRunnerSetTestTimeout, ephemeralRunnerSetTestInterval, ).Should(Succeed(), "compiled / flattened proxy secret should exist") @@ -1250,34 +1429,37 @@ var _ = Describe("Test EphemeralRunnerSet controller with proxy settings", func( return len(runnerList.Items), nil }, ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(1), "1 EphemeralRunner should exist") + ephemeralRunnerSetTestInterval, + ).Should(BeEquivalentTo(1), "1 EphemeralRunner should exist") // Delete the EphemeralRunnerSet err = k8sClient.Delete(ctx, ephemeralRunnerSet) Expect(err).NotTo(HaveOccurred(), "failed to delete EphemeralRunnerSet") - Eventually(func(g Gomega) (int, error) { - runnerList := new(v1alpha1.EphemeralRunnerList) - err := listEphemeralRunnersAndRemoveFinalizers(ctx, k8sClient, runnerList, ephemeralRunnerSet.Namespace) - if err != nil { - return -1, err - } - return len(runnerList.Items), nil - }, + Eventually( + func(g Gomega) (int, error) { + runnerList := new(v1alpha1.EphemeralRunnerList) + err := listEphemeralRunnersAndRemoveFinalizers(ctx, k8sClient, runnerList, ephemeralRunnerSet.Namespace) + if err != nil { + return -1, err + } + return len(runnerList.Items), nil + }, ephemeralRunnerSetTestTimeout, ephemeralRunnerSetTestInterval, ).Should(BeEquivalentTo(0), "EphemeralRunners should be deleted") // Assert that the proxy secret is deleted - Eventually(func(g Gomega) { - proxySecret := &corev1.Secret{} - err = k8sClient.Get(ctx, client.ObjectKey{ - Namespace: autoscalingNS.Name, - Name: proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet), - }, proxySecret) - g.Expect(err).To(HaveOccurred(), "proxy secret should be deleted") - g.Expect(kerrors.IsNotFound(err)).To(BeTrue(), "proxy secret should be deleted") - }, + Eventually( + func(g Gomega) { + proxySecret := &corev1.Secret{} + err = k8sClient.Get(ctx, client.ObjectKey{ + Namespace: autoscalingNS.Name, + Name: proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet), + }, proxySecret) + g.Expect(err).To(HaveOccurred(), "proxy secret should be deleted") + g.Expect(kerrors.IsNotFound(err)).To(BeTrue(), "proxy secret should be deleted") + }, ephemeralRunnerSetTestTimeout, ephemeralRunnerSetTestInterval, ).Should(Succeed(), "proxy secret should be deleted") @@ -1323,7 +1505,7 @@ var _ = Describe("Test EphemeralRunnerSet controller with proxy settings", func( Spec: v1alpha1.EphemeralRunnerSetSpec{ Replicas: 1, EphemeralRunnerSpec: v1alpha1.EphemeralRunnerSpec{ - GitHubConfigUrl: "http://example.com/owner/repo", + GitHubConfigURL: "http://example.com/owner/repo", GitHubConfigSecret: configSecret.Name, RunnerScaleSetID: 100, Proxy: &v1alpha1.ProxyConfig{ @@ -1350,14 +1532,15 @@ var _ = Describe("Test EphemeralRunnerSet controller with proxy settings", func( Expect(err).NotTo(HaveOccurred(), "failed to create EphemeralRunnerSet") runnerList := new(v1alpha1.EphemeralRunnerList) - Eventually(func() (int, error) { - err := listEphemeralRunnersAndRemoveFinalizers(ctx, k8sClient, runnerList, ephemeralRunnerSet.Namespace) - if err != nil { - return -1, err - } + Eventually( + func() (int, error) { + err := listEphemeralRunnersAndRemoveFinalizers(ctx, k8sClient, runnerList, ephemeralRunnerSet.Namespace) + if err != nil { + return -1, err + } - return len(runnerList.Items), nil - }, + return len(runnerList.Items), nil + }, ephemeralRunnerSetTestTimeout, ephemeralRunnerSetTestInterval, ).Should(BeEquivalentTo(1), "failed to create ephemeral runner") @@ -1443,7 +1626,7 @@ var _ = Describe("Test EphemeralRunnerSet controller with custom root CA", func( Spec: v1alpha1.EphemeralRunnerSetSpec{ Replicas: 1, EphemeralRunnerSpec: v1alpha1.EphemeralRunnerSpec{ - GitHubConfigUrl: "https://github.example.com/api/v3", + GitHubConfigURL: "https://github.example.com/api/v3", GitHubConfigSecret: configSecret.Name, GitHubServerTLS: &v1alpha1.TLSConfig{ CertificateFrom: &v1alpha1.TLSCertificateSource{ @@ -1474,14 +1657,15 @@ var _ = Describe("Test EphemeralRunnerSet controller with custom root CA", func( Expect(err).NotTo(HaveOccurred(), "failed to create EphemeralRunnerSet") runnerList := new(v1alpha1.EphemeralRunnerList) - Eventually(func() (int, error) { - err := listEphemeralRunnersAndRemoveFinalizers(ctx, k8sClient, runnerList, ephemeralRunnerSet.Namespace) - if err != nil { - return -1, err - } + Eventually( + func() (int, error) { + err := listEphemeralRunnersAndRemoveFinalizers(ctx, k8sClient, runnerList, ephemeralRunnerSet.Namespace) + if err != nil { + return -1, err + } - return len(runnerList.Items), nil - }, + return len(runnerList.Items), nil + }, ephemeralRunnerSetTestTimeout, ephemeralRunnerSetTestInterval, ).Should(BeEquivalentTo(1), "failed to create ephemeral runner") @@ -1504,9 +1688,9 @@ func listEphemeralRunnersAndRemoveFinalizers(ctx context.Context, k8sClient clie liveItems := make([]v1alpha1.EphemeralRunner, 0) for _, item := range list.Items { if !item.DeletionTimestamp.IsZero() { - if err := patch(ctx, k8sClient, &item, func(runner *v1alpha1.EphemeralRunner) { - runner.Finalizers = []string{} - }); err != nil { + original := item.DeepCopy() + item.Finalizers = []string{} + if err := k8sClient.Patch(ctx, &item, client.MergeFrom(original)); err != nil { return err } continue diff --git a/controllers/actions.github.com/helpers_test.go b/controllers/actions.github.com/helpers_test.go index f8f9e810ff..b798958623 100644 --- a/controllers/actions.github.com/helpers_test.go +++ b/controllers/actions.github.com/helpers_test.go @@ -14,6 +14,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/config" "sigs.k8s.io/controller-runtime/pkg/manager" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" ) const defaultGitHubToken = "gh_token" @@ -54,6 +55,9 @@ func createNamespace(t ginkgo.GinkgoTInterface, client client.Client) (*corev1.N Controller: config.Controller{ SkipNameValidation: ptr.To(true), }, + Metrics: metricsserver.Options{ + BindAddress: "0", + }, Cache: cache.Options{ DefaultNamespaces: map[string]cache.Config{ ns.Name: {}, diff --git a/controllers/actions.github.com/mocks_test.go b/controllers/actions.github.com/mocks_test.go index 0663de5c2a..7b6f8ca3ef 100644 --- a/controllers/actions.github.com/mocks_test.go +++ b/controllers/actions.github.com/mocks_test.go @@ -11,1662 +11,8 @@ import ( "github.com/actions/actions-runner-controller/controllers/actions.github.com/multiclient" "github.com/actions/actions-runner-controller/controllers/actions.github.com/object" mock "github.com/stretchr/testify/mock" - "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" ) -// newMockkubernetesObject creates a new instance of mockkubernetesObject. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func newMockkubernetesObject[T client.Object](t interface { - mock.TestingT - Cleanup(func()) -}) *mockkubernetesObject[T] { - mock := &mockkubernetesObject[T]{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} - -// mockkubernetesObject is an autogenerated mock type for the kubernetesObject type -type mockkubernetesObject[T client.Object] struct { - mock.Mock -} - -type mockkubernetesObject_Expecter[T client.Object] struct { - mock *mock.Mock -} - -func (_m *mockkubernetesObject[T]) EXPECT() *mockkubernetesObject_Expecter[T] { - return &mockkubernetesObject_Expecter[T]{mock: &_m.Mock} -} - -// DeepCopy provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) DeepCopy() T { - ret := _mock.Called() - - if len(ret) == 0 { - panic("no return value specified for DeepCopy") - } - - var r0 T - if returnFunc, ok := ret.Get(0).(func() T); ok { - r0 = returnFunc() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(T) - } - } - return r0 -} - -// mockkubernetesObject_DeepCopy_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeepCopy' -type mockkubernetesObject_DeepCopy_Call[T client.Object] struct { - *mock.Call -} - -// DeepCopy is a helper method to define mock.On call -func (_e *mockkubernetesObject_Expecter[T]) DeepCopy() *mockkubernetesObject_DeepCopy_Call[T] { - return &mockkubernetesObject_DeepCopy_Call[T]{Call: _e.mock.On("DeepCopy")} -} - -func (_c *mockkubernetesObject_DeepCopy_Call[T]) Run(run func()) *mockkubernetesObject_DeepCopy_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - run() - }) - return _c -} - -func (_c *mockkubernetesObject_DeepCopy_Call[T]) Return(v T) *mockkubernetesObject_DeepCopy_Call[T] { - _c.Call.Return(v) - return _c -} - -func (_c *mockkubernetesObject_DeepCopy_Call[T]) RunAndReturn(run func() T) *mockkubernetesObject_DeepCopy_Call[T] { - _c.Call.Return(run) - return _c -} - -// DeepCopyObject provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) DeepCopyObject() runtime.Object { - ret := _mock.Called() - - if len(ret) == 0 { - panic("no return value specified for DeepCopyObject") - } - - var r0 runtime.Object - if returnFunc, ok := ret.Get(0).(func() runtime.Object); ok { - r0 = returnFunc() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(runtime.Object) - } - } - return r0 -} - -// mockkubernetesObject_DeepCopyObject_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeepCopyObject' -type mockkubernetesObject_DeepCopyObject_Call[T client.Object] struct { - *mock.Call -} - -// DeepCopyObject is a helper method to define mock.On call -func (_e *mockkubernetesObject_Expecter[T]) DeepCopyObject() *mockkubernetesObject_DeepCopyObject_Call[T] { - return &mockkubernetesObject_DeepCopyObject_Call[T]{Call: _e.mock.On("DeepCopyObject")} -} - -func (_c *mockkubernetesObject_DeepCopyObject_Call[T]) Run(run func()) *mockkubernetesObject_DeepCopyObject_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - run() - }) - return _c -} - -func (_c *mockkubernetesObject_DeepCopyObject_Call[T]) Return(object runtime.Object) *mockkubernetesObject_DeepCopyObject_Call[T] { - _c.Call.Return(object) - return _c -} - -func (_c *mockkubernetesObject_DeepCopyObject_Call[T]) RunAndReturn(run func() runtime.Object) *mockkubernetesObject_DeepCopyObject_Call[T] { - _c.Call.Return(run) - return _c -} - -// GetAnnotations provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) GetAnnotations() map[string]string { - ret := _mock.Called() - - if len(ret) == 0 { - panic("no return value specified for GetAnnotations") - } - - var r0 map[string]string - if returnFunc, ok := ret.Get(0).(func() map[string]string); ok { - r0 = returnFunc() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(map[string]string) - } - } - return r0 -} - -// mockkubernetesObject_GetAnnotations_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetAnnotations' -type mockkubernetesObject_GetAnnotations_Call[T client.Object] struct { - *mock.Call -} - -// GetAnnotations is a helper method to define mock.On call -func (_e *mockkubernetesObject_Expecter[T]) GetAnnotations() *mockkubernetesObject_GetAnnotations_Call[T] { - return &mockkubernetesObject_GetAnnotations_Call[T]{Call: _e.mock.On("GetAnnotations")} -} - -func (_c *mockkubernetesObject_GetAnnotations_Call[T]) Run(run func()) *mockkubernetesObject_GetAnnotations_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - run() - }) - return _c -} - -func (_c *mockkubernetesObject_GetAnnotations_Call[T]) Return(stringToString map[string]string) *mockkubernetesObject_GetAnnotations_Call[T] { - _c.Call.Return(stringToString) - return _c -} - -func (_c *mockkubernetesObject_GetAnnotations_Call[T]) RunAndReturn(run func() map[string]string) *mockkubernetesObject_GetAnnotations_Call[T] { - _c.Call.Return(run) - return _c -} - -// GetCreationTimestamp provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) GetCreationTimestamp() v1.Time { - ret := _mock.Called() - - if len(ret) == 0 { - panic("no return value specified for GetCreationTimestamp") - } - - var r0 v1.Time - if returnFunc, ok := ret.Get(0).(func() v1.Time); ok { - r0 = returnFunc() - } else { - r0 = ret.Get(0).(v1.Time) - } - return r0 -} - -// mockkubernetesObject_GetCreationTimestamp_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetCreationTimestamp' -type mockkubernetesObject_GetCreationTimestamp_Call[T client.Object] struct { - *mock.Call -} - -// GetCreationTimestamp is a helper method to define mock.On call -func (_e *mockkubernetesObject_Expecter[T]) GetCreationTimestamp() *mockkubernetesObject_GetCreationTimestamp_Call[T] { - return &mockkubernetesObject_GetCreationTimestamp_Call[T]{Call: _e.mock.On("GetCreationTimestamp")} -} - -func (_c *mockkubernetesObject_GetCreationTimestamp_Call[T]) Run(run func()) *mockkubernetesObject_GetCreationTimestamp_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - run() - }) - return _c -} - -func (_c *mockkubernetesObject_GetCreationTimestamp_Call[T]) Return(time v1.Time) *mockkubernetesObject_GetCreationTimestamp_Call[T] { - _c.Call.Return(time) - return _c -} - -func (_c *mockkubernetesObject_GetCreationTimestamp_Call[T]) RunAndReturn(run func() v1.Time) *mockkubernetesObject_GetCreationTimestamp_Call[T] { - _c.Call.Return(run) - return _c -} - -// GetDeletionGracePeriodSeconds provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) GetDeletionGracePeriodSeconds() *int64 { - ret := _mock.Called() - - if len(ret) == 0 { - panic("no return value specified for GetDeletionGracePeriodSeconds") - } - - var r0 *int64 - if returnFunc, ok := ret.Get(0).(func() *int64); ok { - r0 = returnFunc() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*int64) - } - } - return r0 -} - -// mockkubernetesObject_GetDeletionGracePeriodSeconds_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetDeletionGracePeriodSeconds' -type mockkubernetesObject_GetDeletionGracePeriodSeconds_Call[T client.Object] struct { - *mock.Call -} - -// GetDeletionGracePeriodSeconds is a helper method to define mock.On call -func (_e *mockkubernetesObject_Expecter[T]) GetDeletionGracePeriodSeconds() *mockkubernetesObject_GetDeletionGracePeriodSeconds_Call[T] { - return &mockkubernetesObject_GetDeletionGracePeriodSeconds_Call[T]{Call: _e.mock.On("GetDeletionGracePeriodSeconds")} -} - -func (_c *mockkubernetesObject_GetDeletionGracePeriodSeconds_Call[T]) Run(run func()) *mockkubernetesObject_GetDeletionGracePeriodSeconds_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - run() - }) - return _c -} - -func (_c *mockkubernetesObject_GetDeletionGracePeriodSeconds_Call[T]) Return(n *int64) *mockkubernetesObject_GetDeletionGracePeriodSeconds_Call[T] { - _c.Call.Return(n) - return _c -} - -func (_c *mockkubernetesObject_GetDeletionGracePeriodSeconds_Call[T]) RunAndReturn(run func() *int64) *mockkubernetesObject_GetDeletionGracePeriodSeconds_Call[T] { - _c.Call.Return(run) - return _c -} - -// GetDeletionTimestamp provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) GetDeletionTimestamp() *v1.Time { - ret := _mock.Called() - - if len(ret) == 0 { - panic("no return value specified for GetDeletionTimestamp") - } - - var r0 *v1.Time - if returnFunc, ok := ret.Get(0).(func() *v1.Time); ok { - r0 = returnFunc() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*v1.Time) - } - } - return r0 -} - -// mockkubernetesObject_GetDeletionTimestamp_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetDeletionTimestamp' -type mockkubernetesObject_GetDeletionTimestamp_Call[T client.Object] struct { - *mock.Call -} - -// GetDeletionTimestamp is a helper method to define mock.On call -func (_e *mockkubernetesObject_Expecter[T]) GetDeletionTimestamp() *mockkubernetesObject_GetDeletionTimestamp_Call[T] { - return &mockkubernetesObject_GetDeletionTimestamp_Call[T]{Call: _e.mock.On("GetDeletionTimestamp")} -} - -func (_c *mockkubernetesObject_GetDeletionTimestamp_Call[T]) Run(run func()) *mockkubernetesObject_GetDeletionTimestamp_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - run() - }) - return _c -} - -func (_c *mockkubernetesObject_GetDeletionTimestamp_Call[T]) Return(time *v1.Time) *mockkubernetesObject_GetDeletionTimestamp_Call[T] { - _c.Call.Return(time) - return _c -} - -func (_c *mockkubernetesObject_GetDeletionTimestamp_Call[T]) RunAndReturn(run func() *v1.Time) *mockkubernetesObject_GetDeletionTimestamp_Call[T] { - _c.Call.Return(run) - return _c -} - -// GetFinalizers provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) GetFinalizers() []string { - ret := _mock.Called() - - if len(ret) == 0 { - panic("no return value specified for GetFinalizers") - } - - var r0 []string - if returnFunc, ok := ret.Get(0).(func() []string); ok { - r0 = returnFunc() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]string) - } - } - return r0 -} - -// mockkubernetesObject_GetFinalizers_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetFinalizers' -type mockkubernetesObject_GetFinalizers_Call[T client.Object] struct { - *mock.Call -} - -// GetFinalizers is a helper method to define mock.On call -func (_e *mockkubernetesObject_Expecter[T]) GetFinalizers() *mockkubernetesObject_GetFinalizers_Call[T] { - return &mockkubernetesObject_GetFinalizers_Call[T]{Call: _e.mock.On("GetFinalizers")} -} - -func (_c *mockkubernetesObject_GetFinalizers_Call[T]) Run(run func()) *mockkubernetesObject_GetFinalizers_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - run() - }) - return _c -} - -func (_c *mockkubernetesObject_GetFinalizers_Call[T]) Return(strings []string) *mockkubernetesObject_GetFinalizers_Call[T] { - _c.Call.Return(strings) - return _c -} - -func (_c *mockkubernetesObject_GetFinalizers_Call[T]) RunAndReturn(run func() []string) *mockkubernetesObject_GetFinalizers_Call[T] { - _c.Call.Return(run) - return _c -} - -// GetGenerateName provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) GetGenerateName() string { - ret := _mock.Called() - - if len(ret) == 0 { - panic("no return value specified for GetGenerateName") - } - - var r0 string - if returnFunc, ok := ret.Get(0).(func() string); ok { - r0 = returnFunc() - } else { - r0 = ret.Get(0).(string) - } - return r0 -} - -// mockkubernetesObject_GetGenerateName_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetGenerateName' -type mockkubernetesObject_GetGenerateName_Call[T client.Object] struct { - *mock.Call -} - -// GetGenerateName is a helper method to define mock.On call -func (_e *mockkubernetesObject_Expecter[T]) GetGenerateName() *mockkubernetesObject_GetGenerateName_Call[T] { - return &mockkubernetesObject_GetGenerateName_Call[T]{Call: _e.mock.On("GetGenerateName")} -} - -func (_c *mockkubernetesObject_GetGenerateName_Call[T]) Run(run func()) *mockkubernetesObject_GetGenerateName_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - run() - }) - return _c -} - -func (_c *mockkubernetesObject_GetGenerateName_Call[T]) Return(s string) *mockkubernetesObject_GetGenerateName_Call[T] { - _c.Call.Return(s) - return _c -} - -func (_c *mockkubernetesObject_GetGenerateName_Call[T]) RunAndReturn(run func() string) *mockkubernetesObject_GetGenerateName_Call[T] { - _c.Call.Return(run) - return _c -} - -// GetGeneration provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) GetGeneration() int64 { - ret := _mock.Called() - - if len(ret) == 0 { - panic("no return value specified for GetGeneration") - } - - var r0 int64 - if returnFunc, ok := ret.Get(0).(func() int64); ok { - r0 = returnFunc() - } else { - r0 = ret.Get(0).(int64) - } - return r0 -} - -// mockkubernetesObject_GetGeneration_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetGeneration' -type mockkubernetesObject_GetGeneration_Call[T client.Object] struct { - *mock.Call -} - -// GetGeneration is a helper method to define mock.On call -func (_e *mockkubernetesObject_Expecter[T]) GetGeneration() *mockkubernetesObject_GetGeneration_Call[T] { - return &mockkubernetesObject_GetGeneration_Call[T]{Call: _e.mock.On("GetGeneration")} -} - -func (_c *mockkubernetesObject_GetGeneration_Call[T]) Run(run func()) *mockkubernetesObject_GetGeneration_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - run() - }) - return _c -} - -func (_c *mockkubernetesObject_GetGeneration_Call[T]) Return(n int64) *mockkubernetesObject_GetGeneration_Call[T] { - _c.Call.Return(n) - return _c -} - -func (_c *mockkubernetesObject_GetGeneration_Call[T]) RunAndReturn(run func() int64) *mockkubernetesObject_GetGeneration_Call[T] { - _c.Call.Return(run) - return _c -} - -// GetLabels provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) GetLabels() map[string]string { - ret := _mock.Called() - - if len(ret) == 0 { - panic("no return value specified for GetLabels") - } - - var r0 map[string]string - if returnFunc, ok := ret.Get(0).(func() map[string]string); ok { - r0 = returnFunc() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(map[string]string) - } - } - return r0 -} - -// mockkubernetesObject_GetLabels_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetLabels' -type mockkubernetesObject_GetLabels_Call[T client.Object] struct { - *mock.Call -} - -// GetLabels is a helper method to define mock.On call -func (_e *mockkubernetesObject_Expecter[T]) GetLabels() *mockkubernetesObject_GetLabels_Call[T] { - return &mockkubernetesObject_GetLabels_Call[T]{Call: _e.mock.On("GetLabels")} -} - -func (_c *mockkubernetesObject_GetLabels_Call[T]) Run(run func()) *mockkubernetesObject_GetLabels_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - run() - }) - return _c -} - -func (_c *mockkubernetesObject_GetLabels_Call[T]) Return(stringToString map[string]string) *mockkubernetesObject_GetLabels_Call[T] { - _c.Call.Return(stringToString) - return _c -} - -func (_c *mockkubernetesObject_GetLabels_Call[T]) RunAndReturn(run func() map[string]string) *mockkubernetesObject_GetLabels_Call[T] { - _c.Call.Return(run) - return _c -} - -// GetManagedFields provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) GetManagedFields() []v1.ManagedFieldsEntry { - ret := _mock.Called() - - if len(ret) == 0 { - panic("no return value specified for GetManagedFields") - } - - var r0 []v1.ManagedFieldsEntry - if returnFunc, ok := ret.Get(0).(func() []v1.ManagedFieldsEntry); ok { - r0 = returnFunc() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]v1.ManagedFieldsEntry) - } - } - return r0 -} - -// mockkubernetesObject_GetManagedFields_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetManagedFields' -type mockkubernetesObject_GetManagedFields_Call[T client.Object] struct { - *mock.Call -} - -// GetManagedFields is a helper method to define mock.On call -func (_e *mockkubernetesObject_Expecter[T]) GetManagedFields() *mockkubernetesObject_GetManagedFields_Call[T] { - return &mockkubernetesObject_GetManagedFields_Call[T]{Call: _e.mock.On("GetManagedFields")} -} - -func (_c *mockkubernetesObject_GetManagedFields_Call[T]) Run(run func()) *mockkubernetesObject_GetManagedFields_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - run() - }) - return _c -} - -func (_c *mockkubernetesObject_GetManagedFields_Call[T]) Return(managedFieldsEntrys []v1.ManagedFieldsEntry) *mockkubernetesObject_GetManagedFields_Call[T] { - _c.Call.Return(managedFieldsEntrys) - return _c -} - -func (_c *mockkubernetesObject_GetManagedFields_Call[T]) RunAndReturn(run func() []v1.ManagedFieldsEntry) *mockkubernetesObject_GetManagedFields_Call[T] { - _c.Call.Return(run) - return _c -} - -// GetName provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) GetName() string { - ret := _mock.Called() - - if len(ret) == 0 { - panic("no return value specified for GetName") - } - - var r0 string - if returnFunc, ok := ret.Get(0).(func() string); ok { - r0 = returnFunc() - } else { - r0 = ret.Get(0).(string) - } - return r0 -} - -// mockkubernetesObject_GetName_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetName' -type mockkubernetesObject_GetName_Call[T client.Object] struct { - *mock.Call -} - -// GetName is a helper method to define mock.On call -func (_e *mockkubernetesObject_Expecter[T]) GetName() *mockkubernetesObject_GetName_Call[T] { - return &mockkubernetesObject_GetName_Call[T]{Call: _e.mock.On("GetName")} -} - -func (_c *mockkubernetesObject_GetName_Call[T]) Run(run func()) *mockkubernetesObject_GetName_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - run() - }) - return _c -} - -func (_c *mockkubernetesObject_GetName_Call[T]) Return(s string) *mockkubernetesObject_GetName_Call[T] { - _c.Call.Return(s) - return _c -} - -func (_c *mockkubernetesObject_GetName_Call[T]) RunAndReturn(run func() string) *mockkubernetesObject_GetName_Call[T] { - _c.Call.Return(run) - return _c -} - -// GetNamespace provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) GetNamespace() string { - ret := _mock.Called() - - if len(ret) == 0 { - panic("no return value specified for GetNamespace") - } - - var r0 string - if returnFunc, ok := ret.Get(0).(func() string); ok { - r0 = returnFunc() - } else { - r0 = ret.Get(0).(string) - } - return r0 -} - -// mockkubernetesObject_GetNamespace_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetNamespace' -type mockkubernetesObject_GetNamespace_Call[T client.Object] struct { - *mock.Call -} - -// GetNamespace is a helper method to define mock.On call -func (_e *mockkubernetesObject_Expecter[T]) GetNamespace() *mockkubernetesObject_GetNamespace_Call[T] { - return &mockkubernetesObject_GetNamespace_Call[T]{Call: _e.mock.On("GetNamespace")} -} - -func (_c *mockkubernetesObject_GetNamespace_Call[T]) Run(run func()) *mockkubernetesObject_GetNamespace_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - run() - }) - return _c -} - -func (_c *mockkubernetesObject_GetNamespace_Call[T]) Return(s string) *mockkubernetesObject_GetNamespace_Call[T] { - _c.Call.Return(s) - return _c -} - -func (_c *mockkubernetesObject_GetNamespace_Call[T]) RunAndReturn(run func() string) *mockkubernetesObject_GetNamespace_Call[T] { - _c.Call.Return(run) - return _c -} - -// GetObjectKind provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) GetObjectKind() schema.ObjectKind { - ret := _mock.Called() - - if len(ret) == 0 { - panic("no return value specified for GetObjectKind") - } - - var r0 schema.ObjectKind - if returnFunc, ok := ret.Get(0).(func() schema.ObjectKind); ok { - r0 = returnFunc() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(schema.ObjectKind) - } - } - return r0 -} - -// mockkubernetesObject_GetObjectKind_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetObjectKind' -type mockkubernetesObject_GetObjectKind_Call[T client.Object] struct { - *mock.Call -} - -// GetObjectKind is a helper method to define mock.On call -func (_e *mockkubernetesObject_Expecter[T]) GetObjectKind() *mockkubernetesObject_GetObjectKind_Call[T] { - return &mockkubernetesObject_GetObjectKind_Call[T]{Call: _e.mock.On("GetObjectKind")} -} - -func (_c *mockkubernetesObject_GetObjectKind_Call[T]) Run(run func()) *mockkubernetesObject_GetObjectKind_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - run() - }) - return _c -} - -func (_c *mockkubernetesObject_GetObjectKind_Call[T]) Return(objectKind schema.ObjectKind) *mockkubernetesObject_GetObjectKind_Call[T] { - _c.Call.Return(objectKind) - return _c -} - -func (_c *mockkubernetesObject_GetObjectKind_Call[T]) RunAndReturn(run func() schema.ObjectKind) *mockkubernetesObject_GetObjectKind_Call[T] { - _c.Call.Return(run) - return _c -} - -// GetOwnerReferences provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) GetOwnerReferences() []v1.OwnerReference { - ret := _mock.Called() - - if len(ret) == 0 { - panic("no return value specified for GetOwnerReferences") - } - - var r0 []v1.OwnerReference - if returnFunc, ok := ret.Get(0).(func() []v1.OwnerReference); ok { - r0 = returnFunc() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]v1.OwnerReference) - } - } - return r0 -} - -// mockkubernetesObject_GetOwnerReferences_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetOwnerReferences' -type mockkubernetesObject_GetOwnerReferences_Call[T client.Object] struct { - *mock.Call -} - -// GetOwnerReferences is a helper method to define mock.On call -func (_e *mockkubernetesObject_Expecter[T]) GetOwnerReferences() *mockkubernetesObject_GetOwnerReferences_Call[T] { - return &mockkubernetesObject_GetOwnerReferences_Call[T]{Call: _e.mock.On("GetOwnerReferences")} -} - -func (_c *mockkubernetesObject_GetOwnerReferences_Call[T]) Run(run func()) *mockkubernetesObject_GetOwnerReferences_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - run() - }) - return _c -} - -func (_c *mockkubernetesObject_GetOwnerReferences_Call[T]) Return(ownerReferences []v1.OwnerReference) *mockkubernetesObject_GetOwnerReferences_Call[T] { - _c.Call.Return(ownerReferences) - return _c -} - -func (_c *mockkubernetesObject_GetOwnerReferences_Call[T]) RunAndReturn(run func() []v1.OwnerReference) *mockkubernetesObject_GetOwnerReferences_Call[T] { - _c.Call.Return(run) - return _c -} - -// GetResourceVersion provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) GetResourceVersion() string { - ret := _mock.Called() - - if len(ret) == 0 { - panic("no return value specified for GetResourceVersion") - } - - var r0 string - if returnFunc, ok := ret.Get(0).(func() string); ok { - r0 = returnFunc() - } else { - r0 = ret.Get(0).(string) - } - return r0 -} - -// mockkubernetesObject_GetResourceVersion_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetResourceVersion' -type mockkubernetesObject_GetResourceVersion_Call[T client.Object] struct { - *mock.Call -} - -// GetResourceVersion is a helper method to define mock.On call -func (_e *mockkubernetesObject_Expecter[T]) GetResourceVersion() *mockkubernetesObject_GetResourceVersion_Call[T] { - return &mockkubernetesObject_GetResourceVersion_Call[T]{Call: _e.mock.On("GetResourceVersion")} -} - -func (_c *mockkubernetesObject_GetResourceVersion_Call[T]) Run(run func()) *mockkubernetesObject_GetResourceVersion_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - run() - }) - return _c -} - -func (_c *mockkubernetesObject_GetResourceVersion_Call[T]) Return(s string) *mockkubernetesObject_GetResourceVersion_Call[T] { - _c.Call.Return(s) - return _c -} - -func (_c *mockkubernetesObject_GetResourceVersion_Call[T]) RunAndReturn(run func() string) *mockkubernetesObject_GetResourceVersion_Call[T] { - _c.Call.Return(run) - return _c -} - -// GetSelfLink provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) GetSelfLink() string { - ret := _mock.Called() - - if len(ret) == 0 { - panic("no return value specified for GetSelfLink") - } - - var r0 string - if returnFunc, ok := ret.Get(0).(func() string); ok { - r0 = returnFunc() - } else { - r0 = ret.Get(0).(string) - } - return r0 -} - -// mockkubernetesObject_GetSelfLink_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetSelfLink' -type mockkubernetesObject_GetSelfLink_Call[T client.Object] struct { - *mock.Call -} - -// GetSelfLink is a helper method to define mock.On call -func (_e *mockkubernetesObject_Expecter[T]) GetSelfLink() *mockkubernetesObject_GetSelfLink_Call[T] { - return &mockkubernetesObject_GetSelfLink_Call[T]{Call: _e.mock.On("GetSelfLink")} -} - -func (_c *mockkubernetesObject_GetSelfLink_Call[T]) Run(run func()) *mockkubernetesObject_GetSelfLink_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - run() - }) - return _c -} - -func (_c *mockkubernetesObject_GetSelfLink_Call[T]) Return(s string) *mockkubernetesObject_GetSelfLink_Call[T] { - _c.Call.Return(s) - return _c -} - -func (_c *mockkubernetesObject_GetSelfLink_Call[T]) RunAndReturn(run func() string) *mockkubernetesObject_GetSelfLink_Call[T] { - _c.Call.Return(run) - return _c -} - -// GetUID provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) GetUID() types.UID { - ret := _mock.Called() - - if len(ret) == 0 { - panic("no return value specified for GetUID") - } - - var r0 types.UID - if returnFunc, ok := ret.Get(0).(func() types.UID); ok { - r0 = returnFunc() - } else { - r0 = ret.Get(0).(types.UID) - } - return r0 -} - -// mockkubernetesObject_GetUID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetUID' -type mockkubernetesObject_GetUID_Call[T client.Object] struct { - *mock.Call -} - -// GetUID is a helper method to define mock.On call -func (_e *mockkubernetesObject_Expecter[T]) GetUID() *mockkubernetesObject_GetUID_Call[T] { - return &mockkubernetesObject_GetUID_Call[T]{Call: _e.mock.On("GetUID")} -} - -func (_c *mockkubernetesObject_GetUID_Call[T]) Run(run func()) *mockkubernetesObject_GetUID_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - run() - }) - return _c -} - -func (_c *mockkubernetesObject_GetUID_Call[T]) Return(uID types.UID) *mockkubernetesObject_GetUID_Call[T] { - _c.Call.Return(uID) - return _c -} - -func (_c *mockkubernetesObject_GetUID_Call[T]) RunAndReturn(run func() types.UID) *mockkubernetesObject_GetUID_Call[T] { - _c.Call.Return(run) - return _c -} - -// SetAnnotations provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) SetAnnotations(annotations map[string]string) { - _mock.Called(annotations) - return -} - -// mockkubernetesObject_SetAnnotations_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetAnnotations' -type mockkubernetesObject_SetAnnotations_Call[T client.Object] struct { - *mock.Call -} - -// SetAnnotations is a helper method to define mock.On call -// - annotations map[string]string -func (_e *mockkubernetesObject_Expecter[T]) SetAnnotations(annotations interface{}) *mockkubernetesObject_SetAnnotations_Call[T] { - return &mockkubernetesObject_SetAnnotations_Call[T]{Call: _e.mock.On("SetAnnotations", annotations)} -} - -func (_c *mockkubernetesObject_SetAnnotations_Call[T]) Run(run func(annotations map[string]string)) *mockkubernetesObject_SetAnnotations_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - var arg0 map[string]string - if args[0] != nil { - arg0 = args[0].(map[string]string) - } - run( - arg0, - ) - }) - return _c -} - -func (_c *mockkubernetesObject_SetAnnotations_Call[T]) Return() *mockkubernetesObject_SetAnnotations_Call[T] { - _c.Call.Return() - return _c -} - -func (_c *mockkubernetesObject_SetAnnotations_Call[T]) RunAndReturn(run func(annotations map[string]string)) *mockkubernetesObject_SetAnnotations_Call[T] { - _c.Run(run) - return _c -} - -// SetCreationTimestamp provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) SetCreationTimestamp(timestamp v1.Time) { - _mock.Called(timestamp) - return -} - -// mockkubernetesObject_SetCreationTimestamp_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetCreationTimestamp' -type mockkubernetesObject_SetCreationTimestamp_Call[T client.Object] struct { - *mock.Call -} - -// SetCreationTimestamp is a helper method to define mock.On call -// - timestamp v1.Time -func (_e *mockkubernetesObject_Expecter[T]) SetCreationTimestamp(timestamp interface{}) *mockkubernetesObject_SetCreationTimestamp_Call[T] { - return &mockkubernetesObject_SetCreationTimestamp_Call[T]{Call: _e.mock.On("SetCreationTimestamp", timestamp)} -} - -func (_c *mockkubernetesObject_SetCreationTimestamp_Call[T]) Run(run func(timestamp v1.Time)) *mockkubernetesObject_SetCreationTimestamp_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - var arg0 v1.Time - if args[0] != nil { - arg0 = args[0].(v1.Time) - } - run( - arg0, - ) - }) - return _c -} - -func (_c *mockkubernetesObject_SetCreationTimestamp_Call[T]) Return() *mockkubernetesObject_SetCreationTimestamp_Call[T] { - _c.Call.Return() - return _c -} - -func (_c *mockkubernetesObject_SetCreationTimestamp_Call[T]) RunAndReturn(run func(timestamp v1.Time)) *mockkubernetesObject_SetCreationTimestamp_Call[T] { - _c.Run(run) - return _c -} - -// SetDeletionGracePeriodSeconds provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) SetDeletionGracePeriodSeconds(n *int64) { - _mock.Called(n) - return -} - -// mockkubernetesObject_SetDeletionGracePeriodSeconds_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetDeletionGracePeriodSeconds' -type mockkubernetesObject_SetDeletionGracePeriodSeconds_Call[T client.Object] struct { - *mock.Call -} - -// SetDeletionGracePeriodSeconds is a helper method to define mock.On call -// - n *int64 -func (_e *mockkubernetesObject_Expecter[T]) SetDeletionGracePeriodSeconds(n interface{}) *mockkubernetesObject_SetDeletionGracePeriodSeconds_Call[T] { - return &mockkubernetesObject_SetDeletionGracePeriodSeconds_Call[T]{Call: _e.mock.On("SetDeletionGracePeriodSeconds", n)} -} - -func (_c *mockkubernetesObject_SetDeletionGracePeriodSeconds_Call[T]) Run(run func(n *int64)) *mockkubernetesObject_SetDeletionGracePeriodSeconds_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - var arg0 *int64 - if args[0] != nil { - arg0 = args[0].(*int64) - } - run( - arg0, - ) - }) - return _c -} - -func (_c *mockkubernetesObject_SetDeletionGracePeriodSeconds_Call[T]) Return() *mockkubernetesObject_SetDeletionGracePeriodSeconds_Call[T] { - _c.Call.Return() - return _c -} - -func (_c *mockkubernetesObject_SetDeletionGracePeriodSeconds_Call[T]) RunAndReturn(run func(n *int64)) *mockkubernetesObject_SetDeletionGracePeriodSeconds_Call[T] { - _c.Run(run) - return _c -} - -// SetDeletionTimestamp provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) SetDeletionTimestamp(timestamp *v1.Time) { - _mock.Called(timestamp) - return -} - -// mockkubernetesObject_SetDeletionTimestamp_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetDeletionTimestamp' -type mockkubernetesObject_SetDeletionTimestamp_Call[T client.Object] struct { - *mock.Call -} - -// SetDeletionTimestamp is a helper method to define mock.On call -// - timestamp *v1.Time -func (_e *mockkubernetesObject_Expecter[T]) SetDeletionTimestamp(timestamp interface{}) *mockkubernetesObject_SetDeletionTimestamp_Call[T] { - return &mockkubernetesObject_SetDeletionTimestamp_Call[T]{Call: _e.mock.On("SetDeletionTimestamp", timestamp)} -} - -func (_c *mockkubernetesObject_SetDeletionTimestamp_Call[T]) Run(run func(timestamp *v1.Time)) *mockkubernetesObject_SetDeletionTimestamp_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - var arg0 *v1.Time - if args[0] != nil { - arg0 = args[0].(*v1.Time) - } - run( - arg0, - ) - }) - return _c -} - -func (_c *mockkubernetesObject_SetDeletionTimestamp_Call[T]) Return() *mockkubernetesObject_SetDeletionTimestamp_Call[T] { - _c.Call.Return() - return _c -} - -func (_c *mockkubernetesObject_SetDeletionTimestamp_Call[T]) RunAndReturn(run func(timestamp *v1.Time)) *mockkubernetesObject_SetDeletionTimestamp_Call[T] { - _c.Run(run) - return _c -} - -// SetFinalizers provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) SetFinalizers(finalizers []string) { - _mock.Called(finalizers) - return -} - -// mockkubernetesObject_SetFinalizers_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetFinalizers' -type mockkubernetesObject_SetFinalizers_Call[T client.Object] struct { - *mock.Call -} - -// SetFinalizers is a helper method to define mock.On call -// - finalizers []string -func (_e *mockkubernetesObject_Expecter[T]) SetFinalizers(finalizers interface{}) *mockkubernetesObject_SetFinalizers_Call[T] { - return &mockkubernetesObject_SetFinalizers_Call[T]{Call: _e.mock.On("SetFinalizers", finalizers)} -} - -func (_c *mockkubernetesObject_SetFinalizers_Call[T]) Run(run func(finalizers []string)) *mockkubernetesObject_SetFinalizers_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - var arg0 []string - if args[0] != nil { - arg0 = args[0].([]string) - } - run( - arg0, - ) - }) - return _c -} - -func (_c *mockkubernetesObject_SetFinalizers_Call[T]) Return() *mockkubernetesObject_SetFinalizers_Call[T] { - _c.Call.Return() - return _c -} - -func (_c *mockkubernetesObject_SetFinalizers_Call[T]) RunAndReturn(run func(finalizers []string)) *mockkubernetesObject_SetFinalizers_Call[T] { - _c.Run(run) - return _c -} - -// SetGenerateName provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) SetGenerateName(name string) { - _mock.Called(name) - return -} - -// mockkubernetesObject_SetGenerateName_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetGenerateName' -type mockkubernetesObject_SetGenerateName_Call[T client.Object] struct { - *mock.Call -} - -// SetGenerateName is a helper method to define mock.On call -// - name string -func (_e *mockkubernetesObject_Expecter[T]) SetGenerateName(name interface{}) *mockkubernetesObject_SetGenerateName_Call[T] { - return &mockkubernetesObject_SetGenerateName_Call[T]{Call: _e.mock.On("SetGenerateName", name)} -} - -func (_c *mockkubernetesObject_SetGenerateName_Call[T]) Run(run func(name string)) *mockkubernetesObject_SetGenerateName_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - var arg0 string - if args[0] != nil { - arg0 = args[0].(string) - } - run( - arg0, - ) - }) - return _c -} - -func (_c *mockkubernetesObject_SetGenerateName_Call[T]) Return() *mockkubernetesObject_SetGenerateName_Call[T] { - _c.Call.Return() - return _c -} - -func (_c *mockkubernetesObject_SetGenerateName_Call[T]) RunAndReturn(run func(name string)) *mockkubernetesObject_SetGenerateName_Call[T] { - _c.Run(run) - return _c -} - -// SetGeneration provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) SetGeneration(generation int64) { - _mock.Called(generation) - return -} - -// mockkubernetesObject_SetGeneration_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetGeneration' -type mockkubernetesObject_SetGeneration_Call[T client.Object] struct { - *mock.Call -} - -// SetGeneration is a helper method to define mock.On call -// - generation int64 -func (_e *mockkubernetesObject_Expecter[T]) SetGeneration(generation interface{}) *mockkubernetesObject_SetGeneration_Call[T] { - return &mockkubernetesObject_SetGeneration_Call[T]{Call: _e.mock.On("SetGeneration", generation)} -} - -func (_c *mockkubernetesObject_SetGeneration_Call[T]) Run(run func(generation int64)) *mockkubernetesObject_SetGeneration_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - var arg0 int64 - if args[0] != nil { - arg0 = args[0].(int64) - } - run( - arg0, - ) - }) - return _c -} - -func (_c *mockkubernetesObject_SetGeneration_Call[T]) Return() *mockkubernetesObject_SetGeneration_Call[T] { - _c.Call.Return() - return _c -} - -func (_c *mockkubernetesObject_SetGeneration_Call[T]) RunAndReturn(run func(generation int64)) *mockkubernetesObject_SetGeneration_Call[T] { - _c.Run(run) - return _c -} - -// SetLabels provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) SetLabels(labels map[string]string) { - _mock.Called(labels) - return -} - -// mockkubernetesObject_SetLabels_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetLabels' -type mockkubernetesObject_SetLabels_Call[T client.Object] struct { - *mock.Call -} - -// SetLabels is a helper method to define mock.On call -// - labels map[string]string -func (_e *mockkubernetesObject_Expecter[T]) SetLabels(labels interface{}) *mockkubernetesObject_SetLabels_Call[T] { - return &mockkubernetesObject_SetLabels_Call[T]{Call: _e.mock.On("SetLabels", labels)} -} - -func (_c *mockkubernetesObject_SetLabels_Call[T]) Run(run func(labels map[string]string)) *mockkubernetesObject_SetLabels_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - var arg0 map[string]string - if args[0] != nil { - arg0 = args[0].(map[string]string) - } - run( - arg0, - ) - }) - return _c -} - -func (_c *mockkubernetesObject_SetLabels_Call[T]) Return() *mockkubernetesObject_SetLabels_Call[T] { - _c.Call.Return() - return _c -} - -func (_c *mockkubernetesObject_SetLabels_Call[T]) RunAndReturn(run func(labels map[string]string)) *mockkubernetesObject_SetLabels_Call[T] { - _c.Run(run) - return _c -} - -// SetManagedFields provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) SetManagedFields(managedFields []v1.ManagedFieldsEntry) { - _mock.Called(managedFields) - return -} - -// mockkubernetesObject_SetManagedFields_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetManagedFields' -type mockkubernetesObject_SetManagedFields_Call[T client.Object] struct { - *mock.Call -} - -// SetManagedFields is a helper method to define mock.On call -// - managedFields []v1.ManagedFieldsEntry -func (_e *mockkubernetesObject_Expecter[T]) SetManagedFields(managedFields interface{}) *mockkubernetesObject_SetManagedFields_Call[T] { - return &mockkubernetesObject_SetManagedFields_Call[T]{Call: _e.mock.On("SetManagedFields", managedFields)} -} - -func (_c *mockkubernetesObject_SetManagedFields_Call[T]) Run(run func(managedFields []v1.ManagedFieldsEntry)) *mockkubernetesObject_SetManagedFields_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - var arg0 []v1.ManagedFieldsEntry - if args[0] != nil { - arg0 = args[0].([]v1.ManagedFieldsEntry) - } - run( - arg0, - ) - }) - return _c -} - -func (_c *mockkubernetesObject_SetManagedFields_Call[T]) Return() *mockkubernetesObject_SetManagedFields_Call[T] { - _c.Call.Return() - return _c -} - -func (_c *mockkubernetesObject_SetManagedFields_Call[T]) RunAndReturn(run func(managedFields []v1.ManagedFieldsEntry)) *mockkubernetesObject_SetManagedFields_Call[T] { - _c.Run(run) - return _c -} - -// SetName provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) SetName(name string) { - _mock.Called(name) - return -} - -// mockkubernetesObject_SetName_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetName' -type mockkubernetesObject_SetName_Call[T client.Object] struct { - *mock.Call -} - -// SetName is a helper method to define mock.On call -// - name string -func (_e *mockkubernetesObject_Expecter[T]) SetName(name interface{}) *mockkubernetesObject_SetName_Call[T] { - return &mockkubernetesObject_SetName_Call[T]{Call: _e.mock.On("SetName", name)} -} - -func (_c *mockkubernetesObject_SetName_Call[T]) Run(run func(name string)) *mockkubernetesObject_SetName_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - var arg0 string - if args[0] != nil { - arg0 = args[0].(string) - } - run( - arg0, - ) - }) - return _c -} - -func (_c *mockkubernetesObject_SetName_Call[T]) Return() *mockkubernetesObject_SetName_Call[T] { - _c.Call.Return() - return _c -} - -func (_c *mockkubernetesObject_SetName_Call[T]) RunAndReturn(run func(name string)) *mockkubernetesObject_SetName_Call[T] { - _c.Run(run) - return _c -} - -// SetNamespace provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) SetNamespace(namespace string) { - _mock.Called(namespace) - return -} - -// mockkubernetesObject_SetNamespace_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetNamespace' -type mockkubernetesObject_SetNamespace_Call[T client.Object] struct { - *mock.Call -} - -// SetNamespace is a helper method to define mock.On call -// - namespace string -func (_e *mockkubernetesObject_Expecter[T]) SetNamespace(namespace interface{}) *mockkubernetesObject_SetNamespace_Call[T] { - return &mockkubernetesObject_SetNamespace_Call[T]{Call: _e.mock.On("SetNamespace", namespace)} -} - -func (_c *mockkubernetesObject_SetNamespace_Call[T]) Run(run func(namespace string)) *mockkubernetesObject_SetNamespace_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - var arg0 string - if args[0] != nil { - arg0 = args[0].(string) - } - run( - arg0, - ) - }) - return _c -} - -func (_c *mockkubernetesObject_SetNamespace_Call[T]) Return() *mockkubernetesObject_SetNamespace_Call[T] { - _c.Call.Return() - return _c -} - -func (_c *mockkubernetesObject_SetNamespace_Call[T]) RunAndReturn(run func(namespace string)) *mockkubernetesObject_SetNamespace_Call[T] { - _c.Run(run) - return _c -} - -// SetOwnerReferences provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) SetOwnerReferences(ownerReferences []v1.OwnerReference) { - _mock.Called(ownerReferences) - return -} - -// mockkubernetesObject_SetOwnerReferences_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetOwnerReferences' -type mockkubernetesObject_SetOwnerReferences_Call[T client.Object] struct { - *mock.Call -} - -// SetOwnerReferences is a helper method to define mock.On call -// - ownerReferences []v1.OwnerReference -func (_e *mockkubernetesObject_Expecter[T]) SetOwnerReferences(ownerReferences interface{}) *mockkubernetesObject_SetOwnerReferences_Call[T] { - return &mockkubernetesObject_SetOwnerReferences_Call[T]{Call: _e.mock.On("SetOwnerReferences", ownerReferences)} -} - -func (_c *mockkubernetesObject_SetOwnerReferences_Call[T]) Run(run func(ownerReferences []v1.OwnerReference)) *mockkubernetesObject_SetOwnerReferences_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - var arg0 []v1.OwnerReference - if args[0] != nil { - arg0 = args[0].([]v1.OwnerReference) - } - run( - arg0, - ) - }) - return _c -} - -func (_c *mockkubernetesObject_SetOwnerReferences_Call[T]) Return() *mockkubernetesObject_SetOwnerReferences_Call[T] { - _c.Call.Return() - return _c -} - -func (_c *mockkubernetesObject_SetOwnerReferences_Call[T]) RunAndReturn(run func(ownerReferences []v1.OwnerReference)) *mockkubernetesObject_SetOwnerReferences_Call[T] { - _c.Run(run) - return _c -} - -// SetResourceVersion provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) SetResourceVersion(version string) { - _mock.Called(version) - return -} - -// mockkubernetesObject_SetResourceVersion_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetResourceVersion' -type mockkubernetesObject_SetResourceVersion_Call[T client.Object] struct { - *mock.Call -} - -// SetResourceVersion is a helper method to define mock.On call -// - version string -func (_e *mockkubernetesObject_Expecter[T]) SetResourceVersion(version interface{}) *mockkubernetesObject_SetResourceVersion_Call[T] { - return &mockkubernetesObject_SetResourceVersion_Call[T]{Call: _e.mock.On("SetResourceVersion", version)} -} - -func (_c *mockkubernetesObject_SetResourceVersion_Call[T]) Run(run func(version string)) *mockkubernetesObject_SetResourceVersion_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - var arg0 string - if args[0] != nil { - arg0 = args[0].(string) - } - run( - arg0, - ) - }) - return _c -} - -func (_c *mockkubernetesObject_SetResourceVersion_Call[T]) Return() *mockkubernetesObject_SetResourceVersion_Call[T] { - _c.Call.Return() - return _c -} - -func (_c *mockkubernetesObject_SetResourceVersion_Call[T]) RunAndReturn(run func(version string)) *mockkubernetesObject_SetResourceVersion_Call[T] { - _c.Run(run) - return _c -} - -// SetSelfLink provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) SetSelfLink(selfLink string) { - _mock.Called(selfLink) - return -} - -// mockkubernetesObject_SetSelfLink_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetSelfLink' -type mockkubernetesObject_SetSelfLink_Call[T client.Object] struct { - *mock.Call -} - -// SetSelfLink is a helper method to define mock.On call -// - selfLink string -func (_e *mockkubernetesObject_Expecter[T]) SetSelfLink(selfLink interface{}) *mockkubernetesObject_SetSelfLink_Call[T] { - return &mockkubernetesObject_SetSelfLink_Call[T]{Call: _e.mock.On("SetSelfLink", selfLink)} -} - -func (_c *mockkubernetesObject_SetSelfLink_Call[T]) Run(run func(selfLink string)) *mockkubernetesObject_SetSelfLink_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - var arg0 string - if args[0] != nil { - arg0 = args[0].(string) - } - run( - arg0, - ) - }) - return _c -} - -func (_c *mockkubernetesObject_SetSelfLink_Call[T]) Return() *mockkubernetesObject_SetSelfLink_Call[T] { - _c.Call.Return() - return _c -} - -func (_c *mockkubernetesObject_SetSelfLink_Call[T]) RunAndReturn(run func(selfLink string)) *mockkubernetesObject_SetSelfLink_Call[T] { - _c.Run(run) - return _c -} - -// SetUID provides a mock function for the type mockkubernetesObject -func (_mock *mockkubernetesObject[T]) SetUID(uid types.UID) { - _mock.Called(uid) - return -} - -// mockkubernetesObject_SetUID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetUID' -type mockkubernetesObject_SetUID_Call[T client.Object] struct { - *mock.Call -} - -// SetUID is a helper method to define mock.On call -// - uid types.UID -func (_e *mockkubernetesObject_Expecter[T]) SetUID(uid interface{}) *mockkubernetesObject_SetUID_Call[T] { - return &mockkubernetesObject_SetUID_Call[T]{Call: _e.mock.On("SetUID", uid)} -} - -func (_c *mockkubernetesObject_SetUID_Call[T]) Run(run func(uid types.UID)) *mockkubernetesObject_SetUID_Call[T] { - _c.Call.Run(func(args mock.Arguments) { - var arg0 types.UID - if args[0] != nil { - arg0 = args[0].(types.UID) - } - run( - arg0, - ) - }) - return _c -} - -func (_c *mockkubernetesObject_SetUID_Call[T]) Return() *mockkubernetesObject_SetUID_Call[T] { - _c.Call.Return() - return _c -} - -func (_c *mockkubernetesObject_SetUID_Call[T]) RunAndReturn(run func(uid types.UID)) *mockkubernetesObject_SetUID_Call[T] { - _c.Run(run) - return _c -} - -// newMockpatcher creates a new instance of mockpatcher. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func newMockpatcher(t interface { - mock.TestingT - Cleanup(func()) -}) *mockpatcher { - mock := &mockpatcher{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} - -// mockpatcher is an autogenerated mock type for the patcher type -type mockpatcher struct { - mock.Mock -} - -type mockpatcher_Expecter struct { - mock *mock.Mock -} - -func (_m *mockpatcher) EXPECT() *mockpatcher_Expecter { - return &mockpatcher_Expecter{mock: &_m.Mock} -} - -// Patch provides a mock function for the type mockpatcher -func (_mock *mockpatcher) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { - var tmpRet mock.Arguments - if len(opts) > 0 { - tmpRet = _mock.Called(ctx, obj, patch, opts) - } else { - tmpRet = _mock.Called(ctx, obj, patch) - } - ret := tmpRet - - if len(ret) == 0 { - panic("no return value specified for Patch") - } - - var r0 error - if returnFunc, ok := ret.Get(0).(func(context.Context, client.Object, client.Patch, ...client.PatchOption) error); ok { - r0 = returnFunc(ctx, obj, patch, opts...) - } else { - r0 = ret.Error(0) - } - return r0 -} - -// mockpatcher_Patch_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Patch' -type mockpatcher_Patch_Call struct { - *mock.Call -} - -// Patch is a helper method to define mock.On call -// - ctx context.Context -// - obj client.Object -// - patch client.Patch -// - opts ...client.PatchOption -func (_e *mockpatcher_Expecter) Patch(ctx interface{}, obj interface{}, patch interface{}, opts ...interface{}) *mockpatcher_Patch_Call { - return &mockpatcher_Patch_Call{Call: _e.mock.On("Patch", - append([]interface{}{ctx, obj, patch}, opts...)...)} -} - -func (_c *mockpatcher_Patch_Call) Run(run func(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption)) *mockpatcher_Patch_Call { - _c.Call.Run(func(args mock.Arguments) { - var arg0 context.Context - if args[0] != nil { - arg0 = args[0].(context.Context) - } - var arg1 client.Object - if args[1] != nil { - arg1 = args[1].(client.Object) - } - var arg2 client.Patch - if args[2] != nil { - arg2 = args[2].(client.Patch) - } - var arg3 []client.PatchOption - var variadicArgs []client.PatchOption - if len(args) > 3 { - variadicArgs = args[3].([]client.PatchOption) - } - arg3 = variadicArgs - run( - arg0, - arg1, - arg2, - arg3..., - ) - }) - return _c -} - -func (_c *mockpatcher_Patch_Call) Return(err error) *mockpatcher_Patch_Call { - _c.Call.Return(err) - return _c -} - -func (_c *mockpatcher_Patch_Call) RunAndReturn(run func(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error) *mockpatcher_Patch_Call { - _c.Call.Return(run) - return _c -} - -// newMocksubResourcePatcher creates a new instance of mocksubResourcePatcher. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func newMocksubResourcePatcher(t interface { - mock.TestingT - Cleanup(func()) -}) *mocksubResourcePatcher { - mock := &mocksubResourcePatcher{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} - -// mocksubResourcePatcher is an autogenerated mock type for the subResourcePatcher type -type mocksubResourcePatcher struct { - mock.Mock -} - -type mocksubResourcePatcher_Expecter struct { - mock *mock.Mock -} - -func (_m *mocksubResourcePatcher) EXPECT() *mocksubResourcePatcher_Expecter { - return &mocksubResourcePatcher_Expecter{mock: &_m.Mock} -} - -// Patch provides a mock function for the type mocksubResourcePatcher -func (_mock *mocksubResourcePatcher) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { - var tmpRet mock.Arguments - if len(opts) > 0 { - tmpRet = _mock.Called(ctx, obj, patch, opts) - } else { - tmpRet = _mock.Called(ctx, obj, patch) - } - ret := tmpRet - - if len(ret) == 0 { - panic("no return value specified for Patch") - } - - var r0 error - if returnFunc, ok := ret.Get(0).(func(context.Context, client.Object, client.Patch, ...client.SubResourcePatchOption) error); ok { - r0 = returnFunc(ctx, obj, patch, opts...) - } else { - r0 = ret.Error(0) - } - return r0 -} - -// mocksubResourcePatcher_Patch_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Patch' -type mocksubResourcePatcher_Patch_Call struct { - *mock.Call -} - -// Patch is a helper method to define mock.On call -// - ctx context.Context -// - obj client.Object -// - patch client.Patch -// - opts ...client.SubResourcePatchOption -func (_e *mocksubResourcePatcher_Expecter) Patch(ctx interface{}, obj interface{}, patch interface{}, opts ...interface{}) *mocksubResourcePatcher_Patch_Call { - return &mocksubResourcePatcher_Patch_Call{Call: _e.mock.On("Patch", - append([]interface{}{ctx, obj, patch}, opts...)...)} -} - -func (_c *mocksubResourcePatcher_Patch_Call) Run(run func(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption)) *mocksubResourcePatcher_Patch_Call { - _c.Call.Run(func(args mock.Arguments) { - var arg0 context.Context - if args[0] != nil { - arg0 = args[0].(context.Context) - } - var arg1 client.Object - if args[1] != nil { - arg1 = args[1].(client.Object) - } - var arg2 client.Patch - if args[2] != nil { - arg2 = args[2].(client.Patch) - } - var arg3 []client.SubResourcePatchOption - var variadicArgs []client.SubResourcePatchOption - if len(args) > 3 { - variadicArgs = args[3].([]client.SubResourcePatchOption) - } - arg3 = variadicArgs - run( - arg0, - arg1, - arg2, - arg3..., - ) - }) - return _c -} - -func (_c *mocksubResourcePatcher_Patch_Call) Return(err error) *mocksubResourcePatcher_Patch_Call { - _c.Call.Return(err) - return _c -} - -func (_c *mocksubResourcePatcher_Patch_Call) RunAndReturn(run func(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error) *mocksubResourcePatcher_Patch_Call { - _c.Call.Return(run) - return _c -} - // NewMockSecretResolver creates a new instance of MockSecretResolver. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewMockSecretResolver(t interface { @@ -1752,8 +98,8 @@ func (_c *MockSecretResolver_GetActionsService_Call) Run(run func(ctx context.Co return _c } -func (_c *MockSecretResolver_GetActionsService_Call) Return(client1 multiclient.Client, err error) *MockSecretResolver_GetActionsService_Call { - _c.Call.Return(client1, err) +func (_c *MockSecretResolver_GetActionsService_Call) Return(client multiclient.Client, err error) *MockSecretResolver_GetActionsService_Call { + _c.Call.Return(client, err) return _c } diff --git a/controllers/actions.github.com/resourcebuilder.go b/controllers/actions.github.com/resourcebuilder.go index 45ccf1245c..fd456b56f1 100644 --- a/controllers/actions.github.com/resourcebuilder.go +++ b/controllers/actions.github.com/resourcebuilder.go @@ -25,6 +25,9 @@ import ( corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" ) // secret constants @@ -43,6 +46,15 @@ var commonLabelKeys = [...]string{ LabelKeyGitHubRepository, } +// annotationKeyIntegrityHash is used as a hash of the important fields +// of each resource to determine if more drastic action should be taken. +// +// For example, annotations/labels are not something that should modify +// the behavior of a resource, while the change in spec is. Therefore, +// the spec hash should contain the spec fields in order to determine +// modifications. +const annotationKeyIntegrityHash = "actions.github.com/integrity-hash" + const labelValueKubernetesPartOf = "gha-runner-scale-set" var ( @@ -83,9 +95,27 @@ type SecretResolver interface { type ResourceBuilder struct { ExcludeLabelPropagationPrefixes []string SecretResolver + Scheme *runtime.Scheme +} + +func (b *ResourceBuilder) setSchemeIfUnset(scheme *runtime.Scheme) { + if b.Scheme == nil { + b.Scheme = scheme + } +} + +func (b *ResourceBuilder) setControllerReference(owner client.Object, object client.Object) error { + if b.Scheme == nil { + b.Scheme = runtime.NewScheme() + if err := v1alpha1.AddToScheme(b.Scheme); err != nil { + return err + } + } + + return ctrl.SetControllerReference(owner, object, b.Scheme) } -func (b *ResourceBuilder) newAutoScalingListener(autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet, ephemeralRunnerSet *v1alpha1.EphemeralRunnerSet, namespace, image string, imagePullSecrets []corev1.LocalObjectReference) (*v1alpha1.AutoscalingListener, error) { +func (b *ResourceBuilder) newAutoscalingListener(autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet, ephemeralRunnerSet *v1alpha1.EphemeralRunnerSet, namespace, image string, imagePullSecrets []corev1.LocalObjectReference) (*v1alpha1.AutoscalingListener, error) { runnerScaleSetID, err := strconv.Atoi(autoscalingRunnerSet.Annotations[runnerScaleSetIDAnnotationKey]) if err != nil { return nil, err @@ -100,6 +130,28 @@ func (b *ResourceBuilder) newAutoScalingListener(autoscalingRunnerSet *v1alpha1. effectiveMinRunners = *autoscalingRunnerSet.Spec.MinRunners } + spec := v1alpha1.AutoscalingListenerSpec{ + GitHubConfigURL: autoscalingRunnerSet.Spec.GitHubConfigUrl, + GitHubConfigSecret: autoscalingRunnerSet.Spec.GitHubConfigSecret, + VaultConfig: autoscalingRunnerSet.VaultConfig(), + RunnerScaleSetID: runnerScaleSetID, + AutoscalingRunnerSetNamespace: autoscalingRunnerSet.Namespace, + AutoscalingRunnerSetName: autoscalingRunnerSet.Name, + EphemeralRunnerSetName: ephemeralRunnerSet.Name, + MinRunners: effectiveMinRunners, + MaxRunners: effectiveMaxRunners, + Image: image, + ImagePullSecrets: imagePullSecrets, + Proxy: autoscalingRunnerSet.Spec.Proxy, + GitHubServerTLS: autoscalingRunnerSet.Spec.GitHubServerTLS, + Metrics: autoscalingRunnerSet.Spec.ListenerMetrics, + Template: autoscalingRunnerSet.Spec.ListenerTemplate, + ServiceAccountMetadata: autoscalingRunnerSet.Spec.ListenerServiceAccountMetadata, + RoleMetadata: autoscalingRunnerSet.Spec.ListenerRoleMetadata, + RoleBindingMetadata: autoscalingRunnerSet.Spec.ListenerRoleBindingMetadata, + ConfigSecretMetadata: autoscalingRunnerSet.Spec.ListenerConfigSecretMetadata, + } + labels := b.filterAndMergeLabels(autoscalingRunnerSet.Labels, map[string]string{ LabelKeyGitHubScaleSetNamespace: autoscalingRunnerSet.Namespace, LabelKeyGitHubScaleSetName: autoscalingRunnerSet.Name, @@ -113,8 +165,7 @@ func (b *ResourceBuilder) newAutoScalingListener(autoscalingRunnerSet *v1alpha1. } annotations := map[string]string{ - annotationKeyRunnerSpecHash: autoscalingRunnerSet.ListenerSpecHash(), - annotationKeyValuesHash: autoscalingRunnerSet.Annotations[annotationKeyValuesHash], + annotationKeyIntegrityHash: spec.Hash(), } if autoscalingRunnerSet.Spec.AutoscalingListenerMetadata != nil { @@ -129,27 +180,7 @@ func (b *ResourceBuilder) newAutoScalingListener(autoscalingRunnerSet *v1alpha1. Labels: labels, Annotations: annotations, }, - Spec: v1alpha1.AutoscalingListenerSpec{ - GitHubConfigUrl: autoscalingRunnerSet.Spec.GitHubConfigUrl, - GitHubConfigSecret: autoscalingRunnerSet.Spec.GitHubConfigSecret, - VaultConfig: autoscalingRunnerSet.VaultConfig(), - RunnerScaleSetId: runnerScaleSetID, - AutoscalingRunnerSetNamespace: autoscalingRunnerSet.Namespace, - AutoscalingRunnerSetName: autoscalingRunnerSet.Name, - EphemeralRunnerSetName: ephemeralRunnerSet.Name, - MinRunners: effectiveMinRunners, - MaxRunners: effectiveMaxRunners, - Image: image, - ImagePullSecrets: imagePullSecrets, - Proxy: autoscalingRunnerSet.Spec.Proxy, - GitHubServerTLS: autoscalingRunnerSet.Spec.GitHubServerTLS, - Metrics: autoscalingRunnerSet.Spec.ListenerMetrics, - Template: autoscalingRunnerSet.Spec.ListenerTemplate, - ServiceAccountMetadata: autoscalingRunnerSet.Spec.ListenerServiceAccountMetadata, - RoleMetadata: autoscalingRunnerSet.Spec.ListenerRoleMetadata, - RoleBindingMetadata: autoscalingRunnerSet.Spec.ListenerRoleBindingMetadata, - ConfigSecretMetadata: autoscalingRunnerSet.Spec.ListenerConfigSecretMetadata, - }, + Spec: spec, } return autoscalingListener, nil @@ -187,12 +218,12 @@ func (b *ResourceBuilder) newScaleSetListenerConfig(autoscalingListener *v1alpha } config := ghalistenerconfig.Config{ - ConfigureURL: autoscalingListener.Spec.GitHubConfigUrl, + ConfigureURL: autoscalingListener.Spec.GitHubConfigURL, EphemeralRunnerSetNamespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace, EphemeralRunnerSetName: autoscalingListener.Spec.EphemeralRunnerSetName, MaxRunners: autoscalingListener.Spec.MaxRunners, MinRunners: autoscalingListener.Spec.MinRunners, - RunnerScaleSetID: autoscalingListener.Spec.RunnerScaleSetId, + RunnerScaleSetID: autoscalingListener.Spec.RunnerScaleSetID, RunnerScaleSetName: autoscalingListener.Spec.AutoscalingRunnerSetName, ServerRootCA: cert, LogLevel: scaleSetListenerLogLevel, @@ -230,12 +261,12 @@ func (b *ResourceBuilder) newScaleSetListenerConfig(autoscalingListener *v1alpha labels = b.filterAndMergeLabels(autoscalingListener.Spec.ConfigSecretMetadata.Labels, nil) } - var annotations map[string]string + annotations := make(map[string]string) if autoscalingListener.Spec.ConfigSecretMetadata != nil && len(autoscalingListener.Spec.ConfigSecretMetadata.Annotations) > 0 { annotations = autoscalingListener.Spec.ConfigSecretMetadata.Annotations } - return &corev1.Secret{ + desiredSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: scaleSetListenerConfigName(autoscalingListener), Namespace: autoscalingListener.Namespace, @@ -245,17 +276,90 @@ func (b *ResourceBuilder) newScaleSetListenerConfig(autoscalingListener *v1alpha Data: map[string][]byte{ "config.json": buf.Bytes(), }, - }, nil + } + + desiredSecret.Annotations[annotationKeyIntegrityHash] = scaleSetListenerConfigIntegrityHash(desiredSecret) + + if err := b.setControllerReference(autoscalingListener, desiredSecret); err != nil { + return nil, fmt.Errorf("failed to set controller reference for listener config secret: %w", err) + } + + return desiredSecret, nil } -func (b *ResourceBuilder) newScaleSetListenerPod(autoscalingListener *v1alpha1.AutoscalingListener, podConfig *corev1.Secret, serviceAccount *corev1.ServiceAccount, metricsConfig *listenerMetricsServerConfig, envs ...corev1.EnvVar) (*corev1.Pod, error) { - listenerEnv := []corev1.EnvVar{ +func scaleSetListenerConfigIntegrityHash(secret *corev1.Secret) string { + type data struct { + Data map[string][]byte `json:"data,omitempty"` + } + + d := data{ + Data: secret.Data, + } + + return hash.ComputeTemplateHash(&d) +} + +func (b *ResourceBuilder) newScaleSetListenerPod( + autoscalingListener *v1alpha1.AutoscalingListener, + podConfig *corev1.Secret, + serviceAccount *corev1.ServiceAccount, + role *rbacv1.Role, + roleBinding *rbacv1.RoleBinding, + metricsConfig *listenerMetricsServerConfig, +) (*corev1.Pod, error) { + envs := []corev1.EnvVar{ { Name: "LISTENER_CONFIG_PATH", Value: "/etc/gha-listener/config.json", }, } - listenerEnv = append(listenerEnv, envs...) + + if autoscalingListener.Spec.Proxy != nil { + httpURL := corev1.EnvVar{ + Name: "http_proxy", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: proxyListenerSecretName(autoscalingListener), + }, + Key: "http_proxy", + }, + }, + } + if autoscalingListener.Spec.Proxy.HTTP != nil { + envs = append(envs, httpURL) + } + + httpsURL := corev1.EnvVar{ + Name: "https_proxy", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: proxyListenerSecretName(autoscalingListener), + }, + Key: "https_proxy", + }, + }, + } + if autoscalingListener.Spec.Proxy.HTTPS != nil { + envs = append(envs, httpsURL) + } + + noProxy := corev1.EnvVar{ + Name: "no_proxy", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: proxyListenerSecretName(autoscalingListener), + }, + Key: "no_proxy", + }, + }, + } + if len(autoscalingListener.Spec.Proxy.NoProxy) > 0 { + envs = append(envs, noProxy) + } + } var ports []corev1.ContainerPort if metricsConfig != nil && len(metricsConfig.addr) != 0 { @@ -276,7 +380,7 @@ func (b *ResourceBuilder) newScaleSetListenerPod(autoscalingListener *v1alpha1.A { Name: autoscalingListenerContainerName, Image: autoscalingListener.Spec.Image, - Env: listenerEnv, + Env: envs, Command: []string{ scaleSetListenerEntrypoint, }, @@ -314,23 +418,28 @@ func (b *ResourceBuilder) newScaleSetListenerPod(autoscalingListener *v1alpha1.A APIVersion: "v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: autoscalingListener.Name, - Namespace: autoscalingListener.Namespace, - Labels: labels, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: v1alpha1.GroupVersion.String(), - Kind: "AutoscalingListener", - UID: autoscalingListener.GetUID(), - Name: autoscalingListener.GetName(), - Controller: new(true), - BlockOwnerDeletion: new(true), - }, - }, + Name: autoscalingListener.Name, + Namespace: autoscalingListener.Namespace, + Labels: labels, + Annotations: make(map[string]string), }, Spec: podSpec, } + newRunnerScaleSetListenerPod.Annotations[annotationKeyIntegrityHash] = scaleSetListenerPodIntegrity( + newRunnerScaleSetListenerPod, + autoscalingListener, + podConfig, + serviceAccount, + role, + roleBinding, + metricsConfig, + ) + + if err := b.setControllerReference(autoscalingListener, newRunnerScaleSetListenerPod); err != nil { + return nil, fmt.Errorf("failed to set controller reference for listener pod: %w", err) + } + if autoscalingListener.Spec.Template != nil { mergeListenerPodWithTemplate(newRunnerScaleSetListenerPod, autoscalingListener.Spec.Template) } @@ -338,6 +447,38 @@ func (b *ResourceBuilder) newScaleSetListenerPod(autoscalingListener *v1alpha1.A return newRunnerScaleSetListenerPod, nil } +func scaleSetListenerPodIntegrity( + pod *corev1.Pod, + autoscalingListener *v1alpha1.AutoscalingListener, + podConfig *corev1.Secret, + serviceAccount *corev1.ServiceAccount, + role *rbacv1.Role, + roleBinding *rbacv1.RoleBinding, + metricsConfig *listenerMetricsServerConfig, +) string { + type data struct { + ListenerPodSpec *corev1.PodSpec `json:"listenerPodSpec,omitempty"` + AutoscalingListenerIntegrityHash string `json:"autoscalingListenerIntegrityHash"` + ConfigSecretIntegrityHash string `json:"configSecretIntegrityHash"` + ServiceAccountIntegrityHash string `json:"serviceAccountIntegrityHash"` + RoleIntegrityHash string `json:"roleIntegrityHash"` + RoleBindingIntegrityHash string `json:"roleBindingIntegrityHash"` + MetricsConfig *listenerMetricsServerConfig `json:"metricsConfig,omitempty"` + } + + d := data{ + ListenerPodSpec: &pod.Spec, + AutoscalingListenerIntegrityHash: autoscalingListener.Annotations[annotationKeyIntegrityHash], + ConfigSecretIntegrityHash: podConfig.Annotations[annotationKeyIntegrityHash], + ServiceAccountIntegrityHash: serviceAccount.Annotations[annotationKeyIntegrityHash], + RoleIntegrityHash: role.Annotations[annotationKeyIntegrityHash], + RoleBindingIntegrityHash: roleBinding.Annotations[annotationKeyIntegrityHash], + MetricsConfig: metricsConfig, + } + + return hash.ComputeTemplateHash(&d) +} + func mergeListenerPodWithTemplate(pod *corev1.Pod, tmpl *corev1.PodTemplateSpec) { if pod.Annotations == nil { pod.Annotations = make(map[string]string) @@ -455,7 +596,7 @@ func mergeListenerContainer(base, from *corev1.Container) { base.TTY = from.TTY } -func (b *ResourceBuilder) newScaleSetListenerServiceAccount(autoscalingListener *v1alpha1.AutoscalingListener) *corev1.ServiceAccount { +func (b *ResourceBuilder) newScaleSetListenerServiceAccount(autoscalingListener *v1alpha1.AutoscalingListener) (*corev1.ServiceAccount, error) { base := &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Name: autoscalingListener.Name, @@ -464,6 +605,7 @@ func (b *ResourceBuilder) newScaleSetListenerServiceAccount(autoscalingListener LabelKeyGitHubScaleSetNamespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace, LabelKeyGitHubScaleSetName: autoscalingListener.Spec.AutoscalingRunnerSetName, }), + Annotations: make(map[string]string), }, } @@ -472,22 +614,40 @@ func (b *ResourceBuilder) newScaleSetListenerServiceAccount(autoscalingListener base.Annotations = b.mergeAnnotations(autoscalingListener.Spec.ServiceAccountMetadata.Annotations, base.Annotations) } - return base + base.Annotations[annotationKeyIntegrityHash] = scaleSetListenerServiceAccountIntegrityHash(base) + + if err := b.setControllerReference(autoscalingListener, base); err != nil { + return nil, fmt.Errorf("failed to set controller reference for listener service account: %w", err) + } + + return base, nil } -func (b *ResourceBuilder) newScaleSetListenerRole(autoscalingListener *v1alpha1.AutoscalingListener) *rbacv1.Role { - rules := rulesForListenerRole([]string{autoscalingListener.Spec.EphemeralRunnerSetName}) - rulesHash := hash.ComputeTemplateHash(&rules) +func scaleSetListenerServiceAccountIntegrityHash(sa *corev1.ServiceAccount) string { + type data struct { + Secrets []corev1.ObjectReference `json:"secrets"` + ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets"` + AutomountServiceAccountToken *bool `json:"automountServiceAccountToken"` + } + + d := data{ + Secrets: sa.Secrets, + ImagePullSecrets: sa.ImagePullSecrets, + AutomountServiceAccountToken: sa.AutomountServiceAccountToken, + } + return hash.ComputeTemplateHash(&d) +} + +func (b *ResourceBuilder) newScaleSetListenerRole(autoscalingListener *v1alpha1.AutoscalingListener) *rbacv1.Role { labels := b.filterAndMergeLabels(autoscalingListener.Labels, map[string]string{ LabelKeyGitHubScaleSetNamespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace, LabelKeyGitHubScaleSetName: autoscalingListener.Spec.AutoscalingRunnerSetName, labelKeyListenerNamespace: autoscalingListener.Namespace, labelKeyListenerName: autoscalingListener.Name, - "role-policy-rules-hash": rulesHash, }) - var annotations map[string]string + annotations := make(map[string]string) if autoscalingListener.Spec.RoleMetadata != nil { labels = b.filterAndMergeLabels(autoscalingListener.Spec.RoleMetadata.Labels, labels) annotations = b.mergeAnnotations(autoscalingListener.Spec.RoleMetadata.Annotations, nil) @@ -500,18 +660,31 @@ func (b *ResourceBuilder) newScaleSetListenerRole(autoscalingListener *v1alpha1. Labels: labels, Annotations: annotations, }, - Rules: rules, + Rules: rulesForListenerRole([]string{autoscalingListener.Spec.EphemeralRunnerSetName}), } + newRole.Annotations[annotationKeyIntegrityHash] = scaleSetRoleIntegrityHash(newRole) + return newRole } +func scaleSetRoleIntegrityHash(role *rbacv1.Role) string { + type data struct { + Rules []rbacv1.PolicyRule `json:"rules"` + } + + d := data{ + Rules: role.Rules, + } + + return hash.ComputeTemplateHash(&d) +} + func (b *ResourceBuilder) newScaleSetListenerRoleBinding(autoscalingListener *v1alpha1.AutoscalingListener, listenerRole *rbacv1.Role, serviceAccount *corev1.ServiceAccount) *rbacv1.RoleBinding { roleRef := rbacv1.RoleRef{ Kind: "Role", Name: listenerRole.Name, } - roleRefHash := hash.ComputeTemplateHash(&roleRef) subjects := []rbacv1.Subject{ { @@ -520,19 +693,15 @@ func (b *ResourceBuilder) newScaleSetListenerRoleBinding(autoscalingListener *v1 Name: serviceAccount.Name, }, } - subjectHash := hash.ComputeTemplateHash(&subjects) labels := b.filterAndMergeLabels(autoscalingListener.Labels, map[string]string{ LabelKeyGitHubScaleSetNamespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace, LabelKeyGitHubScaleSetName: autoscalingListener.Spec.AutoscalingRunnerSetName, labelKeyListenerNamespace: autoscalingListener.Namespace, labelKeyListenerName: autoscalingListener.Name, - "role-binding-role-ref-hash": roleRefHash, - "role-binding-subject-hash": subjectHash, }) - var annotations map[string]string - + annotations := make(map[string]string) if autoscalingListener.Spec.RoleBindingMetadata != nil { labels = b.filterAndMergeLabels(autoscalingListener.Spec.RoleBindingMetadata.Labels, labels) annotations = autoscalingListener.Spec.RoleBindingMetadata.Annotations @@ -549,15 +718,45 @@ func (b *ResourceBuilder) newScaleSetListenerRoleBinding(autoscalingListener *v1 Subjects: subjects, } + newRoleBinding.Annotations[annotationKeyIntegrityHash] = scaleSetListenerRoleBindingIntegrityHash(newRoleBinding) + return newRoleBinding } +func scaleSetListenerRoleBindingIntegrityHash(rb *rbacv1.RoleBinding) string { + type data struct { + RoleRef rbacv1.RoleRef `json:"roleRef"` + Subjects []rbacv1.Subject `json:"subjects"` + } + + d := data{ + RoleRef: rb.RoleRef, + Subjects: rb.Subjects, + } + + return hash.ComputeTemplateHash(&d) +} + func (b *ResourceBuilder) newEphemeralRunnerSet(autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet) (*v1alpha1.EphemeralRunnerSet, error) { runnerScaleSetID, err := strconv.Atoi(autoscalingRunnerSet.Annotations[runnerScaleSetIDAnnotationKey]) if err != nil { return nil, err } - runnerSpecHash := autoscalingRunnerSet.RunnerSetSpecHash() + + spec := v1alpha1.EphemeralRunnerSetSpec{ + Replicas: 0, + EphemeralRunnerSpec: v1alpha1.EphemeralRunnerSpec{ + RunnerScaleSetID: runnerScaleSetID, + GitHubConfigURL: autoscalingRunnerSet.Spec.GitHubConfigUrl, + GitHubConfigSecret: autoscalingRunnerSet.Spec.GitHubConfigSecret, + Proxy: autoscalingRunnerSet.Spec.Proxy, + GitHubServerTLS: autoscalingRunnerSet.Spec.GitHubServerTLS, + PodTemplateSpec: autoscalingRunnerSet.Spec.Template, + VaultConfig: autoscalingRunnerSet.VaultConfig(), + EphemeralRunnerConfigSecretMetadata: autoscalingRunnerSet.Spec.EphemeralRunnerConfigSecretMetadata, + }, + EphemeralRunnerMetadata: autoscalingRunnerSet.Spec.EphemeralRunnerMetadata, + } labels := b.filterAndMergeLabels(autoscalingRunnerSet.Labels, map[string]string{ LabelKeyKubernetesPartOf: labelValueKubernetesPartOf, @@ -574,7 +773,6 @@ func (b *ResourceBuilder) newEphemeralRunnerSet(autoscalingRunnerSet *v1alpha1.A annotations := map[string]string{ AnnotationKeyGitHubRunnerGroupName: autoscalingRunnerSet.Annotations[AnnotationKeyGitHubRunnerGroupName], AnnotationKeyGitHubRunnerScaleSetName: autoscalingRunnerSet.Annotations[AnnotationKeyGitHubRunnerScaleSetName], - annotationKeyRunnerSpecHash: runnerSpecHash, } if autoscalingRunnerSet.Spec.EphemeralRunnerSetMetadata != nil { @@ -585,41 +783,70 @@ func (b *ResourceBuilder) newEphemeralRunnerSet(autoscalingRunnerSet *v1alpha1.A newEphemeralRunnerSet := &v1alpha1.EphemeralRunnerSet{ TypeMeta: metav1.TypeMeta{}, ObjectMeta: metav1.ObjectMeta{ - GenerateName: autoscalingRunnerSet.Name + "-", - Namespace: autoscalingRunnerSet.Namespace, - Labels: labels, - Annotations: annotations, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: v1alpha1.GroupVersion.String(), - Kind: "AutoscalingRunnerSet", - UID: autoscalingRunnerSet.GetUID(), - Name: autoscalingRunnerSet.GetName(), - Controller: new(true), - BlockOwnerDeletion: new(true), - }, - }, + Name: autoscalingRunnerSet.Name, + Namespace: autoscalingRunnerSet.Namespace, + Labels: labels, + Annotations: annotations, }, - Spec: v1alpha1.EphemeralRunnerSetSpec{ - Replicas: 0, - EphemeralRunnerSpec: v1alpha1.EphemeralRunnerSpec{ - RunnerScaleSetID: runnerScaleSetID, - GitHubConfigUrl: autoscalingRunnerSet.Spec.GitHubConfigUrl, - GitHubConfigSecret: autoscalingRunnerSet.Spec.GitHubConfigSecret, - Proxy: autoscalingRunnerSet.Spec.Proxy, - GitHubServerTLS: autoscalingRunnerSet.Spec.GitHubServerTLS, - PodTemplateSpec: autoscalingRunnerSet.Spec.Template, - VaultConfig: autoscalingRunnerSet.VaultConfig(), - EphemeralRunnerConfigSecretMetadata: autoscalingRunnerSet.Spec.EphemeralRunnerConfigSecretMetadata, + Spec: spec, + } + + newEphemeralRunnerSet.Annotations[annotationKeyIntegrityHash] = ephemeralRunnerSetIntegrityHash(newEphemeralRunnerSet) + + if err := b.setControllerReference(autoscalingRunnerSet, newEphemeralRunnerSet); err != nil { + return nil, fmt.Errorf("failed to set controller reference for ephemeral runner set: %w", err) + } + + return newEphemeralRunnerSet, nil +} + +func ephemeralRunnerSetIntegrityHash(ers *v1alpha1.EphemeralRunnerSet) string { + type data struct { + EphemeralRunnerSpec v1alpha1.EphemeralRunnerSpec `json:"ephemeralRunnerSpec"` + } + + d := data{ + EphemeralRunnerSpec: ers.Spec.EphemeralRunnerSpec, + } + return hash.ComputeTemplateHash(&d) +} + +func (b *ResourceBuilder) newAutoscalingListenerProxySecret(autoscalingListener *v1alpha1.AutoscalingListener, data map[string][]byte) (*corev1.Secret, error) { + newProxySecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: proxyListenerSecretName(autoscalingListener), + Namespace: autoscalingListener.Namespace, + Labels: map[string]string{ + LabelKeyGitHubScaleSetNamespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace, + LabelKeyGitHubScaleSetName: autoscalingListener.Spec.AutoscalingRunnerSetName, }, - EphemeralRunnerMetadata: autoscalingRunnerSet.Spec.EphemeralRunnerMetadata, + Annotations: make(map[string]string, 1), }, + Data: data, } - return newEphemeralRunnerSet, nil + newProxySecret.Annotations[annotationKeyIntegrityHash] = autoscalingListenerProxySecretIntegrityHash(newProxySecret) + + if err := b.setControllerReference(autoscalingListener, newProxySecret); err != nil { + return nil, fmt.Errorf("failed to set controller reference for listener proxy secret: %w", err) + } + + return newProxySecret, nil +} + +func autoscalingListenerProxySecretIntegrityHash(secret *corev1.Secret) string { + type data struct { + Data map[string][]byte `json:"data"` + } + + d := data{ + Data: secret.Data, + } + + return hash.ComputeTemplateHash(&d) } -func (b *ResourceBuilder) newEphemeralRunner(ephemeralRunnerSet *v1alpha1.EphemeralRunnerSet) *v1alpha1.EphemeralRunner { +func (b *ResourceBuilder) newEphemeralRunner(ephemeralRunnerSet *v1alpha1.EphemeralRunnerSet) (*v1alpha1.EphemeralRunner, error) { labels := make(map[string]string, len(ephemeralRunnerSet.Labels)) maps.Copy(labels, ephemeralRunnerSet.Labels) labels[LabelKeyKubernetesComponent] = "runner" @@ -633,7 +860,7 @@ func (b *ResourceBuilder) newEphemeralRunner(ephemeralRunnerSet *v1alpha1.Epheme annotations = b.mergeAnnotations(ephemeralRunnerSet.Spec.EphemeralRunnerMetadata.Annotations, annotations) } - return &v1alpha1.EphemeralRunner{ + ephemeralRunner := &v1alpha1.EphemeralRunner{ ObjectMeta: metav1.ObjectMeta{ GenerateName: ephemeralRunnerSet.Name + "-runner-", Namespace: ephemeralRunnerSet.Namespace, @@ -643,22 +870,17 @@ func (b *ResourceBuilder) newEphemeralRunner(ephemeralRunnerSet *v1alpha1.Epheme ephemeralRunnerFinalizerName, ephemeralRunnerActionsFinalizerName, }, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: v1alpha1.GroupVersion.String(), - Kind: "EphemeralRunnerSet", - UID: ephemeralRunnerSet.GetUID(), - Name: ephemeralRunnerSet.GetName(), - Controller: new(true), - BlockOwnerDeletion: new(true), - }, - }, }, Spec: ephemeralRunnerSet.Spec.EphemeralRunnerSpec, } + if err := b.setControllerReference(ephemeralRunnerSet, ephemeralRunner); err != nil { + return nil, fmt.Errorf("failed to set controller reference for ephemeral runner: %w", err) + } + + return ephemeralRunner, nil } -func (b *ResourceBuilder) newEphemeralRunnerPod(runner *v1alpha1.EphemeralRunner, secret *corev1.Secret, envs ...corev1.EnvVar) *corev1.Pod { +func (b *ResourceBuilder) newEphemeralRunnerPod(runner *v1alpha1.EphemeralRunner, secret *corev1.Secret, envs ...corev1.EnvVar) (*corev1.Pod, error) { var newPod corev1.Pod annotations := make(map[string]string, len(runner.Annotations)+len(runner.Spec.Annotations)) @@ -681,16 +903,6 @@ func (b *ResourceBuilder) newEphemeralRunnerPod(runner *v1alpha1.EphemeralRunner Namespace: runner.Namespace, Labels: labels, Annotations: annotations, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: v1alpha1.GroupVersion.String(), - Kind: "EphemeralRunner", - UID: runner.GetUID(), - Name: runner.GetName(), - Controller: new(true), - BlockOwnerDeletion: new(true), - }, - }, } newPod.ObjectMeta = objectMeta @@ -727,10 +939,14 @@ func (b *ResourceBuilder) newEphemeralRunnerPod(runner *v1alpha1.EphemeralRunner newPod.Spec.Containers = append(newPod.Spec.Containers, c) } - return &newPod + if err := b.setControllerReference(runner, &newPod); err != nil { + return nil, fmt.Errorf("failed to set controller reference for ephemeral runner pod: %w", err) + } + + return &newPod, nil } -func (b *ResourceBuilder) newEphemeralRunnerJitSecret(ephemeralRunner *v1alpha1.EphemeralRunner, jitConfig *scaleset.RunnerScaleSetJitRunnerConfig) *corev1.Secret { +func (b *ResourceBuilder) newEphemeralRunnerJitSecret(ephemeralRunner *v1alpha1.EphemeralRunner, jitConfig *scaleset.RunnerScaleSetJitRunnerConfig) (*corev1.Secret, error) { var ( labels map[string]string annotations map[string]string @@ -741,7 +957,7 @@ func (b *ResourceBuilder) newEphemeralRunnerJitSecret(ephemeralRunner *v1alpha1. annotations = ephemeralRunner.Spec.EphemeralRunnerConfigSecretMetadata.Annotations } - return &corev1.Secret{ + jitSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace, @@ -755,10 +971,50 @@ func (b *ResourceBuilder) newEphemeralRunnerJitSecret(ephemeralRunner *v1alpha1. "scaleSetId": []byte(strconv.Itoa(jitConfig.Runner.RunnerScaleSetID)), }, } + if err := b.setControllerReference(ephemeralRunner, jitSecret); err != nil { + return nil, fmt.Errorf("failed to set controller reference for ephemeral runner jit secret: %w", err) + } + + return jitSecret, nil +} + +func (b *ResourceBuilder) newEphemeralRunnerSetProxySecret(ephemeralRunnerSet *v1alpha1.EphemeralRunnerSet, data map[string][]byte) (*corev1.Secret, error) { + runnerPodProxySecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet), + Namespace: ephemeralRunnerSet.Namespace, + Labels: map[string]string{ + LabelKeyGitHubScaleSetName: ephemeralRunnerSet.Labels[LabelKeyGitHubScaleSetName], + LabelKeyGitHubScaleSetNamespace: ephemeralRunnerSet.Labels[LabelKeyGitHubScaleSetNamespace], + }, + Annotations: make(map[string]string, 1), + }, + Data: data, + } + + runnerPodProxySecret.Annotations[annotationKeyIntegrityHash] = ephemeralRunnerSetProxySecretZIdentityHash(runnerPodProxySecret) + + if err := b.setControllerReference(ephemeralRunnerSet, runnerPodProxySecret); err != nil { + return nil, fmt.Errorf("failed to set controller reference for ephemeral runner set proxy secret: %w", err) + } + + return runnerPodProxySecret, nil +} + +func ephemeralRunnerSetProxySecretZIdentityHash(secret *corev1.Secret) string { + type data struct { + Data map[string][]byte `json:"data"` + } + + d := data{ + Data: secret.Data, + } + + return hash.ComputeTemplateHash(&d) } func scaleSetListenerConfigName(autoscalingListener *v1alpha1.AutoscalingListener) string { - return fmt.Sprintf("%s-config", autoscalingListener.Name) + return autoscalingListener.Name + "-config" } func hashSuffix(namespace, runnerGroup, configURL string) string { diff --git a/controllers/actions.github.com/resourcebuilder_test.go b/controllers/actions.github.com/resourcebuilder_test.go index 6f916a6755..d08851173a 100644 --- a/controllers/actions.github.com/resourcebuilder_test.go +++ b/controllers/actions.github.com/resourcebuilder_test.go @@ -113,7 +113,7 @@ func TestMetadataPropagation(t *testing.T) { assert.Equal(t, labelValueKubernetesPartOf, ephemeralRunnerSet.Labels[LabelKeyKubernetesPartOf]) assert.Equal(t, "runner-set", ephemeralRunnerSet.Labels[LabelKeyKubernetesComponent]) assert.Equal(t, autoscalingRunnerSet.Labels[LabelKeyKubernetesVersion], ephemeralRunnerSet.Labels[LabelKeyKubernetesVersion]) - assert.NotEmpty(t, ephemeralRunnerSet.Annotations[annotationKeyRunnerSpecHash]) + assert.NotEmpty(t, ephemeralRunnerSet.Annotations[annotationKeyIntegrityHash]) assert.Equal(t, autoscalingRunnerSet.Name, ephemeralRunnerSet.Labels[LabelKeyGitHubScaleSetName]) assert.Equal(t, autoscalingRunnerSet.Namespace, ephemeralRunnerSet.Labels[LabelKeyGitHubScaleSetNamespace]) assert.Equal(t, "", ephemeralRunnerSet.Labels[LabelKeyGitHubEnterprise]) @@ -125,12 +125,12 @@ func TestMetadataPropagation(t *testing.T) { assert.Equal(t, "ephemeral-runner-set-label", ephemeralRunnerSet.Labels["test.com/ephemeral-runner-set-label"]) assert.Equal(t, "ephemeral-runner-set-annotation", ephemeralRunnerSet.Annotations["test.com/ephemeral-runner-set-annotation"]) - listener, err := b.newAutoScalingListener(&autoscalingRunnerSet, ephemeralRunnerSet, autoscalingRunnerSet.Namespace, "test:latest", nil) + listener, err := b.newAutoscalingListener(&autoscalingRunnerSet, ephemeralRunnerSet, autoscalingRunnerSet.Namespace, "test:latest", nil) require.NoError(t, err) assert.Equal(t, labelValueKubernetesPartOf, listener.Labels[LabelKeyKubernetesPartOf]) assert.Equal(t, "runner-scale-set-listener", listener.Labels[LabelKeyKubernetesComponent]) assert.Equal(t, autoscalingRunnerSet.Labels[LabelKeyKubernetesVersion], listener.Labels[LabelKeyKubernetesVersion]) - assert.NotEmpty(t, ephemeralRunnerSet.Annotations[annotationKeyRunnerSpecHash]) + assert.NotEmpty(t, ephemeralRunnerSet.Annotations[annotationKeyIntegrityHash]) assert.Equal(t, autoscalingRunnerSet.Name, listener.Labels[LabelKeyGitHubScaleSetName]) assert.Equal(t, autoscalingRunnerSet.Namespace, listener.Labels[LabelKeyGitHubScaleSetNamespace]) assert.Equal(t, "", listener.Labels[LabelKeyGitHubEnterprise]) @@ -145,7 +145,8 @@ func TestMetadataPropagation(t *testing.T) { assert.NotContains(t, listener.Labels, "directly.excluded.org/label") assert.Equal(t, "not-excluded-value", listener.Labels["directly.excluded.org/arbitrary"]) - listenerServiceAccount := b.newScaleSetListenerServiceAccount(listener) + listenerServiceAccount, err := b.newScaleSetListenerServiceAccount(listener) + require.NoError(t, err) assert.Equal(t, "listener-service-account-label", listenerServiceAccount.Labels["test.com/listener-service-account-label"]) assert.Equal(t, "listener-service-account-annotation", listenerServiceAccount.Annotations["test.com/listener-service-account-annotation"]) @@ -157,11 +158,18 @@ func TestMetadataPropagation(t *testing.T) { assert.Equal(t, "listener-role-binding-label", listenerRoleBinding.Labels["test.com/listener-role-binding-label"]) assert.Equal(t, "listener-role-binding-annotation", listenerRoleBinding.Annotations["test.com/listener-role-binding-annotation"]) - listenerPod, err := b.newScaleSetListenerPod(listener, &corev1.Secret{}, listenerServiceAccount, nil) + listenerPod, err := b.newScaleSetListenerPod( + listener, + &corev1.Secret{}, + listenerServiceAccount, + listenerRole, + listenerRoleBinding, + nil, + ) require.NoError(t, err) assert.Equal(t, listenerPod.Labels, listener.Labels) - ephemeralRunner := b.newEphemeralRunner(ephemeralRunnerSet) + ephemeralRunner, err := b.newEphemeralRunner(ephemeralRunnerSet) require.NoError(t, err) for _, key := range commonLabelKeys { @@ -176,7 +184,7 @@ func TestMetadataPropagation(t *testing.T) { assert.Equal(t, "ephemeral-runner-label", ephemeralRunner.Labels["test.com/ephemeral-runner-label"]) assert.Equal(t, "ephemeral-runner-annotation", ephemeralRunner.Annotations["test.com/ephemeral-runner-annotation"]) - runnerSecret := b.newEphemeralRunnerJitSecret(ephemeralRunner, &scaleset.RunnerScaleSetJitRunnerConfig{ + runnerSecret, err := b.newEphemeralRunnerJitSecret(ephemeralRunner, &scaleset.RunnerScaleSetJitRunnerConfig{ Runner: &scaleset.RunnerReference{ ID: 1, Name: "test", @@ -184,15 +192,44 @@ func TestMetadataPropagation(t *testing.T) { }, EncodedJITConfig: "", }) + require.NoError(t, err) assert.Equal(t, "ephemeral-runner-config-secret-label", runnerSecret.Labels["test.com/ephemeral-runner-config-secret-label"]) assert.Equal(t, "ephemeral-runner-config-secret-annotation", runnerSecret.Annotations["test.com/ephemeral-runner-config-secret-annotation"]) - pod := b.newEphemeralRunnerPod(ephemeralRunner, runnerSecret) + pod, err := b.newEphemeralRunnerPod(ephemeralRunner, runnerSecret) + require.NoError(t, err) for key := range ephemeralRunner.Labels { assert.Equal(t, ephemeralRunner.Labels[key], pod.Labels[key]) } } +func TestEphemeralRunnerSetProxySecretZIdentityHash(t *testing.T) { + ephemeralRunnerSet := &v1alpha1.EphemeralRunnerSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-scale-set", + Namespace: "test-ns", + Labels: map[string]string{ + LabelKeyGitHubScaleSetName: "test-scale-set", + LabelKeyGitHubScaleSetNamespace: "test-ns", + }, + }, + } + + var b ResourceBuilder + proxySecret, err := b.newEphemeralRunnerSetProxySecret(ephemeralRunnerSet, map[string][]byte{ + "http_proxy": []byte("http://proxy.example.com"), + }) + require.NoError(t, err) + + actualHash := proxySecret.Annotations[annotationKeyIntegrityHash] + assert.NotEmpty(t, actualHash) + assert.Equal(t, ephemeralRunnerSetProxySecretZIdentityHash(proxySecret), actualHash) + + changedProxySecret := proxySecret.DeepCopy() + changedProxySecret.Data["http_proxy"] = []byte("http://updated-proxy.example.com") + assert.NotEqual(t, actualHash, ephemeralRunnerSetProxySecretZIdentityHash(changedProxySecret)) +} + func TestGitHubURLTrimLabelValues(t *testing.T) { enterprise := strings.Repeat("a", 64) organization := strings.Repeat("b", 64) @@ -229,7 +266,7 @@ func TestGitHubURLTrimLabelValues(t *testing.T) { assert.True(t, strings.HasSuffix(ephemeralRunnerSet.Labels[LabelKeyGitHubOrganization], trimLabelVauleSuffix)) assert.True(t, strings.HasSuffix(ephemeralRunnerSet.Labels[LabelKeyGitHubRepository], trimLabelVauleSuffix)) - listener, err := b.newAutoScalingListener(autoscalingRunnerSet, ephemeralRunnerSet, autoscalingRunnerSet.Namespace, "test:latest", nil) + listener, err := b.newAutoscalingListener(autoscalingRunnerSet, ephemeralRunnerSet, autoscalingRunnerSet.Namespace, "test:latest", nil) require.NoError(t, err) assert.Len(t, listener.Labels[LabelKeyGitHubEnterprise], 0) assert.Len(t, listener.Labels[LabelKeyGitHubOrganization], 63) @@ -252,7 +289,7 @@ func TestGitHubURLTrimLabelValues(t *testing.T) { assert.Len(t, ephemeralRunnerSet.Labels[LabelKeyGitHubOrganization], 0) assert.Len(t, ephemeralRunnerSet.Labels[LabelKeyGitHubRepository], 0) - listener, err := b.newAutoScalingListener(autoscalingRunnerSet, ephemeralRunnerSet, autoscalingRunnerSet.Namespace, "test:latest", nil) + listener, err := b.newAutoscalingListener(autoscalingRunnerSet, ephemeralRunnerSet, autoscalingRunnerSet.Namespace, "test:latest", nil) require.NoError(t, err) assert.Len(t, listener.Labels[LabelKeyGitHubEnterprise], 63) assert.True(t, strings.HasSuffix(ephemeralRunnerSet.Labels[LabelKeyGitHubEnterprise], trimLabelVauleSuffix)) @@ -276,7 +313,7 @@ func TestOwnershipRelationships(t *testing.T) { runnerScaleSetIDAnnotationKey: "1", AnnotationKeyGitHubRunnerGroupName: "test-group", AnnotationKeyGitHubRunnerScaleSetName: "test-scale-set", - annotationKeyValuesHash: "test-hash", + annotationKeyIntegrityHash: "test-hash", }, }, Spec: v1alpha1.AutoscalingRunnerSetSpec{ @@ -292,13 +329,23 @@ func TestOwnershipRelationships(t *testing.T) { require.NoError(t, err) // Create and test Listener Pod ownership - listener, err := b.newAutoScalingListener(&autoscalingRunnerSet, ephemeralRunnerSet, autoscalingRunnerSet.Namespace, "test:latest", nil) + listener, err := b.newAutoscalingListener(&autoscalingRunnerSet, ephemeralRunnerSet, autoscalingRunnerSet.Namespace, "test:latest", nil) require.NoError(t, err) listener.UID = "test-listener-uid" - listenerServiceAccount := &corev1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{Name: "test-sa"}, - } - listenerPod, err := b.newScaleSetListenerPod(listener, &corev1.Secret{}, listenerServiceAccount, nil) + + listenerServiceAccount, err := b.newScaleSetListenerServiceAccount(listener) + require.NoError(t, err) + listenerRole := b.newScaleSetListenerRole(listener) + listenerRoleBinding := b.newScaleSetListenerRoleBinding(listener, listenerRole, listenerServiceAccount) + + listenerPod, err := b.newScaleSetListenerPod( + listener, + &corev1.Secret{}, + listenerServiceAccount, + listenerRole, + listenerRoleBinding, + nil, + ) require.NoError(t, err) require.Len(t, listenerPod.OwnerReferences, 1, "Listener Pod should have exactly one owner reference") @@ -321,7 +368,8 @@ func TestOwnershipRelationships(t *testing.T) { assert.Equal(t, true, *ownerRef.BlockOwnerDeletion, "BlockOwnerDeletion flag should be true") // Create EphemeralRunner - ephemeralRunner := b.newEphemeralRunner(ephemeralRunnerSet) + ephemeralRunner, err := b.newEphemeralRunner(ephemeralRunnerSet) + require.NoError(t, err) // Test EphemeralRunner ownership require.Len(t, ephemeralRunner.OwnerReferences, 1, "EphemeralRunner should have exactly one owner reference") @@ -339,7 +387,8 @@ func TestOwnershipRelationships(t *testing.T) { Name: "test-secret", }, } - pod := b.newEphemeralRunnerPod(ephemeralRunner, runnerSecret) + pod, err := b.newEphemeralRunnerPod(ephemeralRunner, runnerSecret) + require.NoError(t, err) // Test EphemeralRunnerPod ownership require.Len(t, pod.OwnerReferences, 1, "EphemeralRunnerPod should have exactly one owner reference") @@ -376,17 +425,23 @@ func TestListenerPodNodeSelector(t *testing.T) { ephemeralRunnerSet, err := b.newEphemeralRunnerSet(&autoscalingRunnerSet) require.NoError(t, err) - listener, err := b.newAutoScalingListener(&autoscalingRunnerSet, ephemeralRunnerSet, autoscalingRunnerSet.Namespace, "test:latest", nil) + listener, err := b.newAutoscalingListener(&autoscalingRunnerSet, ephemeralRunnerSet, autoscalingRunnerSet.Namespace, "test:latest", nil) require.NoError(t, err) - listenerServiceAccount := &corev1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - }, - } + listenerServiceAccount, err := b.newScaleSetListenerServiceAccount(listener) + require.NoError(t, err) + listenerRole := b.newScaleSetListenerRole(listener) + listenerRoleBinding := b.newScaleSetListenerRoleBinding(listener, listenerRole, listenerServiceAccount) t.Run("default listener pod has linux nodeSelector", func(t *testing.T) { - pod, err := b.newScaleSetListenerPod(listener, &corev1.Secret{}, listenerServiceAccount, nil) + pod, err := b.newScaleSetListenerPod( + listener, + &corev1.Secret{}, + listenerServiceAccount, + listenerRole, + listenerRoleBinding, + nil, + ) require.NoError(t, err) require.NotNil(t, pod.Spec.NodeSelector) assert.Equal(t, "linux", pod.Spec.NodeSelector[LabelKeyKubernetesOS], @@ -397,7 +452,14 @@ func TestListenerPodNodeSelector(t *testing.T) { listenerNoTemplate := listener.DeepCopy() listenerNoTemplate.Spec.Template = nil - pod, err := b.newScaleSetListenerPod(listenerNoTemplate, &corev1.Secret{}, listenerServiceAccount, nil) + pod, err := b.newScaleSetListenerPod( + listenerNoTemplate, + &corev1.Secret{}, + listenerServiceAccount, + listenerRole, + listenerRoleBinding, + nil, + ) require.NoError(t, err) require.NotNil(t, pod.Spec.NodeSelector) assert.Equal(t, "linux", pod.Spec.NodeSelector[LabelKeyKubernetesOS], @@ -415,7 +477,14 @@ func TestListenerPodNodeSelector(t *testing.T) { }, } - pod, err := b.newScaleSetListenerPod(listenerWithTemplate, &corev1.Secret{}, listenerServiceAccount, nil) + pod, err := b.newScaleSetListenerPod( + listenerWithTemplate, + &corev1.Secret{}, + listenerServiceAccount, + listenerRole, + listenerRoleBinding, + nil, + ) require.NoError(t, err) require.NotNil(t, pod.Spec.NodeSelector, "linux nodeSelector should not be cleared by template with nil nodeSelector") @@ -434,7 +503,14 @@ func TestListenerPodNodeSelector(t *testing.T) { }, } - pod, err := b.newScaleSetListenerPod(listenerWithTemplate, &corev1.Secret{}, listenerServiceAccount, nil) + pod, err := b.newScaleSetListenerPod( + listenerWithTemplate, + &corev1.Secret{}, + listenerServiceAccount, + listenerRole, + listenerRoleBinding, + nil, + ) require.NoError(t, err) require.NotNil(t, pod.Spec.NodeSelector) assert.Equal(t, "linux", pod.Spec.NodeSelector[LabelKeyKubernetesOS]) @@ -450,7 +526,14 @@ func TestListenerPodNodeSelector(t *testing.T) { }, } - pod, err := b.newScaleSetListenerPod(listenerWithTemplate, &corev1.Secret{}, listenerServiceAccount, nil) + pod, err := b.newScaleSetListenerPod( + listenerWithTemplate, + &corev1.Secret{}, + listenerServiceAccount, + listenerRole, + listenerRoleBinding, + nil, + ) require.NoError(t, err) // An explicitly set empty map is non-nil, so it overrides the default. // This is intentional: the user explicitly opted out of nodeSelector constraints. diff --git a/main.go b/main.go index 1b4c023c23..80c047e63f 100644 --- a/main.go +++ b/main.go @@ -91,7 +91,6 @@ func main() { autoScalingRunnerSetOnly bool enableLeaderElection bool disableAdmissionWebhook bool - updateStrategy string leaderElectionID string port int syncPeriod time.Duration @@ -160,7 +159,6 @@ func main() { flag.StringVar(&logLevel, "log-level", logging.LogLevelDebug, `The verbosity of the logging. Valid values are "debug", "info", "warn", "error". Defaults to "debug".`) flag.StringVar(&logFormat, "log-format", "text", `The log format. Valid options are "text" and "json". Defaults to "text"`) flag.BoolVar(&autoScalingRunnerSetOnly, "auto-scaling-runner-set-only", false, "Make controller only reconcile AutoRunnerScaleSet object.") - flag.StringVar(&updateStrategy, "update-strategy", "immediate", `Resources reconciliation strategy on upgrade with running/pending jobs. Valid values are: "immediate", "eventual". Defaults to "immediate".`) flag.Var(&autoScalerImagePullSecrets, "auto-scaler-image-pull-secrets", "The default image-pull secret name for auto-scaler listener container.") flag.IntVar(&k8sClientRateLimiterQPS, "k8s-client-rate-limiter-qps", 20, "The QPS value of the K8s client rate limiter.") flag.IntVar(&k8sClientRateLimiterBurst, "k8s-client-rate-limiter-burst", 30, "The burst value of the K8s client rate limiter.") @@ -211,14 +209,6 @@ func main() { defaultNamespaces[watchSingleNamespace] = cache.Config{} defaultNamespaces[managerNamespace] = cache.Config{} } - - switch updateStrategy { - case "eventual", "immediate": - log.Info(`Update strategy set to:`, "updateStrategy", updateStrategy) - default: - log.Info(`Update strategy not recognized. Defaulting to "immediately"`, "updateStrategy", updateStrategy) - updateStrategy = "immediate" - } } if actionsgithubcom.SetListenerLoggingParameters(logLevel, logFormat) { @@ -312,6 +302,7 @@ func main() { rb := actionsgithubcom.ResourceBuilder{ ExcludeLabelPropagationPrefixes: excludeLabelPropagationPrefixes, SecretResolver: secretResolver, + Scheme: mgr.GetScheme(), } log.Info("Resource builder initializing") @@ -320,7 +311,8 @@ func main() { switch workqueueRateLimiter { case "typed_rate_limiter": log.Info("Using typed rate limiter (per-item only, no global token bucket)") - controllerOpts = append(controllerOpts, + controllerOpts = append( + controllerOpts, actionsgithubcom.WithTypedRateLimiter(workqueue.DefaultTypedItemBasedRateLimiter[reconcile.Request]()), ) case "bucket_rate_limiter", "": @@ -336,7 +328,6 @@ func main() { Scheme: mgr.GetScheme(), ControllerNamespace: managerNamespace, DefaultRunnerScaleSetListenerImage: managerImage, - UpdateStrategy: actionsgithubcom.UpdateStrategy(updateStrategy), DefaultRunnerScaleSetListenerImagePullSecrets: autoScalerImagePullSecrets, ResourceBuilder: rb, }).SetupWithManager(mgr, controllerOpts...); err != nil { diff --git a/test/actions.github.com/init-with-min-runners-v2.test.sh b/test/actions.github.com/init-with-min-runners-v2.test.sh index c194ef2c3c..5730890419 100755 --- a/test/actions.github.com/init-with-min-runners-v2.test.sh +++ b/test/actions.github.com/init-with-min-runners-v2.test.sh @@ -25,7 +25,6 @@ function install_arc() { --namespace "arc-systems" \ --create-namespace \ --set controller.manager.container.image="${IMAGE_NAME}:${IMAGE_TAG}" \ - --set controller.manager.config.updateStrategy="eventual" \ "${ROOT_DIR}/charts/gha-runner-scale-set-controller-experimental" \ --debug diff --git a/test/actions.github.com/init-with-min-runners.test.sh b/test/actions.github.com/init-with-min-runners.test.sh index 7af6823271..a3c3940fb9 100755 --- a/test/actions.github.com/init-with-min-runners.test.sh +++ b/test/actions.github.com/init-with-min-runners.test.sh @@ -26,7 +26,6 @@ function install_arc() { --create-namespace \ --set image.repository="${IMAGE_NAME}" \ --set image.tag="${IMAGE_TAG}" \ - --set flags.updateStrategy="eventual" \ "${ROOT_DIR}/charts/gha-runner-scale-set-controller" \ --debug diff --git a/test/actions.github.com/update-gha-runner-scale-set-v2.test.sh b/test/actions.github.com/update-gha-runner-scale-set-v2.test.sh new file mode 100755 index 0000000000..208107d5ec --- /dev/null +++ b/test/actions.github.com/update-gha-runner-scale-set-v2.test.sh @@ -0,0 +1,453 @@ +#!/bin/bash + +set -euo pipefail + +DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" + +ROOT_DIR="$(realpath "${DIR}/../..")" + +source "${DIR}/helper.sh" || { + echo "Failed to source helper.sh" + exit 1 +} + +export VERSION="$(chart_version "${ROOT_DIR}/charts/gha-runner-scale-set-controller-experimental/Chart.yaml")" + +SCALE_SET_NAME="update-strategy-$(date '+%M%S')$(((RANDOM + 100) % 100 + 1))" +SCALE_SET_NAMESPACE="arc-runners" +WORKFLOW_FILE="arc-test-sleepy-matrix.yaml" +ARC_NAME="arc" +ARC_NAMESPACE="arc-systems" + +function install_arc() { + echo "Installing ARC" + helm install "${ARC_NAME}" \ + --namespace "${ARC_NAMESPACE}" \ + --create-namespace \ + --set controller.manager.container.image="${IMAGE_NAME}:${IMAGE_TAG}" \ + "${ROOT_DIR}/charts/gha-runner-scale-set-controller-experimental" \ + --debug + + if ! NAME="${ARC_NAME}" NAMESPACE="${ARC_NAMESPACE}" wait_for_arc; then + NAMESPACE="${ARC_NAMESPACE}" log_arc + return 1 + fi +} + +function install_scale_set() { + echo "Installing scale set ${SCALE_SET_NAME}/${SCALE_SET_NAMESPACE}" + helm install "${SCALE_SET_NAME}" \ + --namespace "${SCALE_SET_NAMESPACE}" \ + --create-namespace \ + --set controllerServiceAccount.name="${ARC_NAME}-gha-rs-controller" \ + --set controllerServiceAccount.namespace="${ARC_NAMESPACE}" \ + --set auth.url="https://github.com/${TARGET_ORG}/${TARGET_REPO}" \ + --set auth.githubToken="${GITHUB_TOKEN}" \ + "${ROOT_DIR}/charts/gha-runner-scale-set-experimental" \ + --version="${VERSION}" \ + --debug + + if ! NAME="${SCALE_SET_NAME}" NAMESPACE="${ARC_NAMESPACE}" wait_for_scale_set; then + NAMESPACE="${ARC_NAMESPACE}" log_arc + return 1 + fi +} + +function upgrade_scale_set() { + echo "Upgrading scale set ${SCALE_SET_NAME}/${SCALE_SET_NAMESPACE}" + + UPGRADE_MARKER="e2e-upgrade-${SCALE_SET_NAME}-$(date +%s)" + echo "Generated upgrade marker: ${UPGRADE_MARKER}" + + PATCH_APPLIED_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + echo "Captured PATCH_APPLIED_TIME: ${PATCH_APPLIED_TIME}" + + helm upgrade "${SCALE_SET_NAME}" \ + --namespace "${SCALE_SET_NAMESPACE}" \ + --set controllerServiceAccount.name="${ARC_NAME}-gha-rs-controller" \ + --set controllerServiceAccount.namespace="${ARC_NAMESPACE}" \ + --set auth.url="https://github.com/${TARGET_ORG}/${TARGET_REPO}" \ + --set auth.githubToken="${GITHUB_TOKEN}" \ + --set runner.container.image="ghcr.io/actions/actions-runner:latest" \ + --set runner.container.command={"/home/runner/run.sh"} \ + --set runner.env[0].name="TEST" \ + --set runner.env[0].value="E2E TESTS" \ + --set "runner.pod.metadata.labels.e2e\.arc/upgrade-marker=${UPGRADE_MARKER}" \ + "${ROOT_DIR}/charts/gha-runner-scale-set-experimental" \ + --version="${VERSION}" \ + --debug + +} + +function capture_pre_upgrade_state() { + echo "Capturing pre-upgrade state for scale set ${SCALE_SET_NAME}" + + # Capture listener pod UID + local listener_json + listener_json=$(kubectl get pods -n "${ARC_NAMESPACE}" \ + -l "actions.github.com/scale-set-name=${SCALE_SET_NAME}" \ + --field-selector=status.phase=Running \ + -o json) || { + echo "ERROR: kubectl failed to fetch listener pods" + echo "Namespace: ${ARC_NAMESPACE}" + echo "Selector: actions.github.com/scale-set-name=${SCALE_SET_NAME}" + return 1 + } + + local listener_count + listener_count=$(echo "${listener_json}" | jq '.items | length') + + if [ "${listener_count}" -ne 1 ]; then + echo "ERROR: Expected exactly 1 running listener pod, found ${listener_count}" + echo "Selector: actions.github.com/scale-set-name=${SCALE_SET_NAME}" + echo "Namespace: ${ARC_NAMESPACE}" + echo "Observed pods: ${listener_json}" + return 1 + fi + + PRE_UPGRADE_LISTENER_UID=$(echo "${listener_json}" | jq -r '.items[0].metadata.uid') + + # Capture EphemeralRunnerSet UID + local ers_json + ers_json=$(kubectl get autoscalingrunnersets -n "${SCALE_SET_NAMESPACE}" \ + -l "app.kubernetes.io/instance=${SCALE_SET_NAME}" \ + -o json) || { + echo "ERROR: kubectl failed to fetch EphemeralRunnerSet" + echo "Namespace: ${SCALE_SET_NAMESPACE}" + echo "Selector: app.kubernetes.io/instance=${SCALE_SET_NAME}" + return 1 + } + + local ers_count + ers_count=$(echo "${ers_json}" | jq '.items | length') + + if [ "${ers_count}" -ne 1 ]; then + echo "ERROR: Expected exactly 1 EphemeralRunnerSet, found ${ers_count}" + echo "Selector: app.kubernetes.io/instance=${SCALE_SET_NAME}" + echo "Namespace: ${SCALE_SET_NAMESPACE}" + echo "Observed resources: ${ers_json}" + return 1 + fi + + PRE_UPGRADE_ERS_UID=$(echo "${ers_json}" | jq -r '.items[0].metadata.uid') + + # Capture timestamp + PRE_UPGRADE_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + + echo "PRE_UPGRADE_LISTENER_UID=${PRE_UPGRADE_LISTENER_UID}" + echo "PRE_UPGRADE_ERS_UID=${PRE_UPGRADE_ERS_UID}" + echo "PRE_UPGRADE_TIME=${PRE_UPGRADE_TIME}" + + return 0 +} + +function assert_listener_stays_up() { + echo "Asserting listener remains continuously running with unchanged UID" + + if [ -z "${PRE_UPGRADE_LISTENER_UID:-}" ]; then + echo "ERROR: PRE_UPGRADE_LISTENER_UID not set. Call capture_pre_upgrade_state first." + return 1 + fi + + local count=0 + local max_iterations=120 + local sleep_interval=1 + + echo "Starting continuous verification for ${max_iterations}s (interval: ${sleep_interval}s)" + echo "Expected listener UID: ${PRE_UPGRADE_LISTENER_UID}" + echo "Selector: actions.github.com/scale-set-name=${SCALE_SET_NAME}" + echo "Namespace: ${ARC_NAMESPACE}" + + while [ "${count}" -lt "${max_iterations}" ]; do + # Fetch current listener pods + local listener_json + listener_json=$(kubectl get pods -n "${ARC_NAMESPACE}" \ + -l "actions.github.com/scale-set-name=${SCALE_SET_NAME}" \ + --field-selector=status.phase=Running \ + -o json) || { + echo "ERROR: kubectl failed to fetch listener pods during verification" + echo "Namespace: ${ARC_NAMESPACE}" + echo "Selector: actions.github.com/scale-set-name=${SCALE_SET_NAME}" + echo "Time elapsed: ${count}s" + return 1 + } + + local listener_count + listener_count=$(echo "${listener_json}" | jq '.items | length') + + # Check listener count is exactly 1 + if [ "${listener_count}" -ne 1 ]; then + echo "FAIL: Expected exactly 1 running listener, found ${listener_count}" + echo "Time elapsed: ${count}s" + echo "Expected UID: ${PRE_UPGRADE_LISTENER_UID}" + echo "Selector: actions.github.com/scale-set-name=${SCALE_SET_NAME}" + echo "Namespace: ${ARC_NAMESPACE}" + echo "" + echo "Listener pods detail:" + kubectl get pods -n "${ARC_NAMESPACE}" \ + -l "actions.github.com/scale-set-name=${SCALE_SET_NAME}" \ + -o json | jq '.items[] | {name: .metadata.name, uid: .metadata.uid, phase: .status.phase, creationTimestamp: .metadata.creationTimestamp}' || echo "WARNING: kubectl diagnostic command failed" + echo "" + echo "All pods in all namespaces:" + kubectl get pods -A || echo "WARNING: kubectl diagnostic command failed" + return 1 + fi + + # Extract current listener UID + local current_uid + current_uid=$(echo "${listener_json}" | jq -r '.items[0].metadata.uid') + + # Check UID has not changed + if [ "${current_uid}" != "${PRE_UPGRADE_LISTENER_UID}" ]; then + echo "FAIL: Listener UID changed from ${PRE_UPGRADE_LISTENER_UID} to ${current_uid}" + echo "Time elapsed: ${count}s" + echo "Expected UID: ${PRE_UPGRADE_LISTENER_UID}" + echo "Observed UID: ${current_uid}" + echo "Selector: actions.github.com/scale-set-name=${SCALE_SET_NAME}" + echo "Namespace: ${ARC_NAMESPACE}" + echo "" + echo "Current listener pod detail:" + kubectl get pods -n "${ARC_NAMESPACE}" \ + -l "actions.github.com/scale-set-name=${SCALE_SET_NAME}" \ + -o json | jq '.items[0] | {name: .metadata.name, uid: .metadata.uid, phase: .status.phase, creationTimestamp: .metadata.creationTimestamp}' || echo "WARNING: kubectl diagnostic command failed" + echo "" + echo "All pods in all namespaces:" + kubectl get pods -A || echo "WARNING: kubectl diagnostic command failed" + return 1 + fi + + # Progress indicator every 10 seconds + if [ $((count % 10)) -eq 0 ]; then + echo "Verification ongoing: ${count}s elapsed | Listener count: ${listener_count} | UID: ${current_uid}" + fi + + sleep "${sleep_interval}" + count=$((count + sleep_interval)) + done + + echo "SUCCESS: Listener remained continuously running for ${max_iterations}s with unchanged UID" + echo "Final listener UID: ${PRE_UPGRADE_LISTENER_UID}" + echo "Selector: actions.github.com/scale-set-name=${SCALE_SET_NAME}" + echo "Namespace: ${ARC_NAMESPACE}" + echo "" + echo "Final listener pod detail:" + kubectl get pods -n "${ARC_NAMESPACE}" \ + -l "actions.github.com/scale-set-name=${SCALE_SET_NAME}" \ + -o json | jq '.items[0] | {name: .metadata.name, uid: .metadata.uid, phase: .status.phase, creationTimestamp: .metadata.creationTimestamp}' || echo "WARNING: kubectl diagnostic command failed" + return 0 +} + +function assert_marked_runner_spawned_after_upgrade() { + echo "Asserting at least one runner pod with upgrade marker spawned after patch application" + + if [ -z "${UPGRADE_MARKER:-}" ]; then + echo "ERROR: UPGRADE_MARKER not set. Call upgrade_scale_set first." + return 1 + fi + + if [ -z "${PATCH_APPLIED_TIME:-}" ]; then + echo "ERROR: PATCH_APPLIED_TIME not set. Call upgrade_scale_set first." + return 1 + fi + + local count=0 + local max_iterations=120 + local sleep_interval=1 + + echo "Starting verification for ${max_iterations}s (interval: ${sleep_interval}s)" + echo "Expected marker: ${UPGRADE_MARKER}" + echo "Expected creationTimestamp > ${PATCH_APPLIED_TIME}" + echo "Selector: app.kubernetes.io/component=runner,e2e.arc/upgrade-marker=${UPGRADE_MARKER}" + echo "Namespace: ${SCALE_SET_NAMESPACE}" + + while [ "${count}" -lt "${max_iterations}" ]; do + # Fetch runner pods with marker label + local runner_json + runner_json=$(kubectl get pods -n "${SCALE_SET_NAMESPACE}" \ + -l "app.kubernetes.io/component=runner,e2e.arc/upgrade-marker=${UPGRADE_MARKER}" \ + -o json) || { + echo "ERROR: kubectl failed to fetch runner pods during verification" + echo "Namespace: ${SCALE_SET_NAMESPACE}" + echo "Selector: app.kubernetes.io/component=runner,e2e.arc/upgrade-marker=${UPGRADE_MARKER}" + echo "Time elapsed: ${count}s" + return 1 + } + + local runner_count + runner_count=$(echo "${runner_json}" | jq '.items | length') + + if [ "${runner_count}" -eq 0 ]; then + if [ $((count % 10)) -eq 0 ]; then + echo "No runners with marker found yet. Elapsed: ${count}s" + fi + sleep "${sleep_interval}" + count=$((count + sleep_interval)) + continue + fi + + # Filter pods created after PATCH_APPLIED_TIME + local matching_runners + matching_runners=$(echo "${runner_json}" | jq -r --arg patch_time "${PATCH_APPLIED_TIME}" \ + '.items[] | select(.metadata.creationTimestamp > $patch_time) | .metadata.name') + + local matching_count + matching_count=$(echo "${matching_runners}" | grep -c . || true) + + if [ "${matching_count}" -gt 0 ]; then + # Check if at least one is Running + local running_count + running_count=$(echo "${runner_json}" | jq -r --arg patch_time "${PATCH_APPLIED_TIME}" \ + '[.items[] | select(.metadata.creationTimestamp > $patch_time and .status.phase == "Running")] | length') + + if [ "${running_count}" -gt 0 ]; then + echo "SUCCESS: Found ${running_count} running runner(s) with marker spawned after upgrade" + echo "Marker: ${UPGRADE_MARKER}" + echo "Patch applied time: ${PATCH_APPLIED_TIME}" + echo "Matching pods:" + echo "${matching_runners}" + echo "" + echo "Detailed pod information:" + kubectl get pods -n "${SCALE_SET_NAMESPACE}" \ + -l "app.kubernetes.io/component=runner,e2e.arc/upgrade-marker=${UPGRADE_MARKER}" \ + -o json | jq '.items[] | select(.metadata.creationTimestamp > "'${PATCH_APPLIED_TIME}'") | {name: .metadata.name, uid: .metadata.uid, phase: .status.phase, creationTimestamp: .metadata.creationTimestamp, "upgrade-marker": .metadata.labels."e2e.arc/upgrade-marker"}' || echo "WARNING: kubectl diagnostic command failed" + return 0 + else + echo "Found ${matching_count} matching pod(s), but none are Running yet. Elapsed: ${count}s" + fi + fi + + # Progress indicator every 10 seconds + if [ $((count % 10)) -eq 0 ]; then + echo "Verification ongoing: ${count}s elapsed | Runners with marker: ${runner_count} | Matching time: ${matching_count}" + fi + + sleep "${sleep_interval}" + count=$((count + sleep_interval)) + done + + # Timeout - dump diagnostics + echo "FAIL: No running runner pod with marker spawned after upgrade within ${max_iterations}s" + echo "Expected marker: ${UPGRADE_MARKER}" + echo "Expected creationTimestamp > ${PATCH_APPLIED_TIME}" + echo "Selector: app.kubernetes.io/component=runner,e2e.arc/upgrade-marker=${UPGRADE_MARKER}" + echo "Namespace: ${SCALE_SET_NAMESPACE}" + echo "" + echo "Candidate pods with marker (regardless of timestamp):" + kubectl get pods -n "${SCALE_SET_NAMESPACE}" \ + -l "app.kubernetes.io/component=runner,e2e.arc/upgrade-marker=${UPGRADE_MARKER}" \ + -o json | jq -r '.items[] | "Pod: \(.metadata.name) | UID: \(.metadata.uid) | Created: \(.metadata.creationTimestamp) | Phase: \(.status.phase) | Marker: \(.metadata.labels."e2e.arc/upgrade-marker")"' || echo "WARNING: kubectl diagnostic command failed" + + echo "" + echo "All runner pods in namespace (for timestamp/label analysis):" + kubectl get pods -n "${SCALE_SET_NAMESPACE}" -l "app.kubernetes.io/component=runner" \ + -o json | jq '.items[] | {name: .metadata.name, uid: .metadata.uid, phase: .status.phase, creationTimestamp: .metadata.creationTimestamp, "upgrade-marker": .metadata.labels."e2e.arc/upgrade-marker"}' || echo "WARNING: kubectl diagnostic command failed" + + return 1 +} + +function assert_ephemeral_runner_set_stays_up() { + echo "Asserting EphemeralRunnerSet remains unchanged and stays up" + + if [ -z "${PRE_UPGRADE_ERS_UID:-}" ]; then + echo "ERROR: PRE_UPGRADE_ERS_UID not set. Call capture_pre_upgrade_state first." + return 1 + fi + + # Get current ERS + local ers_json + ers_json=$(kubectl get autoscalingrunnersets -n "${SCALE_SET_NAMESPACE}" \ + -l "app.kubernetes.io/instance=${SCALE_SET_NAME}" \ + -o json) || { + echo "ERROR: kubectl failed to fetch EphemeralRunnerSet" + echo "Namespace: ${SCALE_SET_NAMESPACE}" + echo "Selector: app.kubernetes.io/instance=${SCALE_SET_NAME}" + return 1 + } + + local ers_count + ers_count=$(echo "${ers_json}" | jq '.items | length') + + # Validate exactly one ERS exists + if [ "${ers_count}" -ne 1 ]; then + echo "FAIL: Expected exactly 1 EphemeralRunnerSet, found ${ers_count}" + echo "Expected UID: ${PRE_UPGRADE_ERS_UID}" + echo "Selector: app.kubernetes.io/instance=${SCALE_SET_NAME}" + echo "Namespace: ${SCALE_SET_NAMESPACE}" + echo "" + echo "Observed ERS resources:" + kubectl get autoscalingrunnersets -n "${SCALE_SET_NAMESPACE}" \ + -l "app.kubernetes.io/instance=${SCALE_SET_NAME}" \ + -o json | jq '.items[] | {name: .metadata.name, uid: .metadata.uid, generation: .metadata.generation, observedGeneration: .status.observedGeneration, creationTimestamp: .metadata.creationTimestamp}' || echo "WARNING: kubectl diagnostic command failed" + return 1 + fi + + # Extract current ERS UID + local current_ers_uid + current_ers_uid=$(echo "${ers_json}" | jq -r '.items[0].metadata.uid') + + # Validate UID has not changed + if [ "${current_ers_uid}" != "${PRE_UPGRADE_ERS_UID}" ]; then + echo "FAIL: ERS UID changed from ${PRE_UPGRADE_ERS_UID} to ${current_ers_uid}" + echo "ERS was recreated during upgrade, indicating pod replacement instead of true update" + echo "Expected UID: ${PRE_UPGRADE_ERS_UID}" + echo "Observed UID: ${current_ers_uid}" + echo "Selector: app.kubernetes.io/instance=${SCALE_SET_NAME}" + echo "Namespace: ${SCALE_SET_NAMESPACE}" + echo "" + echo "Full ERS metadata:" + kubectl get autoscalingrunnersets -n "${SCALE_SET_NAMESPACE}" \ + -l "app.kubernetes.io/instance=${SCALE_SET_NAME}" \ + -o json | jq '.items[0].metadata' || echo "WARNING: kubectl diagnostic command failed" + echo "" + echo "ERS status:" + kubectl get autoscalingrunnersets -n "${SCALE_SET_NAMESPACE}" \ + -l "app.kubernetes.io/instance=${SCALE_SET_NAME}" \ + -o json | jq '.items[0].status' || echo "WARNING: kubectl diagnostic command failed" + return 1 + fi + + # Optionally verify status.currentReplicas is queryable + local current_replicas + current_replicas=$(echo "${ers_json}" | jq -r '.items[0].status.currentReplicas // "N/A"') + local generation + generation=$(echo "${ers_json}" | jq -r '.items[0].metadata.generation // "N/A"') + local observedGeneration + observedGeneration=$(echo "${ers_json}" | jq -r '.items[0].status.observedGeneration // "N/A"') + + echo "SUCCESS: EphemeralRunnerSet remains unchanged" + echo "ERS UID: ${current_ers_uid}" + echo "Expected UID: ${PRE_UPGRADE_ERS_UID}" + echo "Generation: ${generation} | Observed Generation: ${observedGeneration}" + echo "Current Replicas: ${current_replicas}" + echo "Selector: app.kubernetes.io/instance=${SCALE_SET_NAME}" + echo "Namespace: ${SCALE_SET_NAMESPACE}" + echo "" + echo "ERS detail:" + kubectl get autoscalingrunnersets -n "${SCALE_SET_NAMESPACE}" \ + -l "app.kubernetes.io/instance=${SCALE_SET_NAME}" \ + -o json | jq '.items[0] | {name: .metadata.name, uid: .metadata.uid, generation: .metadata.generation, creationTimestamp: .metadata.creationTimestamp, observedGeneration: .status.observedGeneration, currentReplicas: .status.currentReplicas}' || echo "WARNING: kubectl diagnostic command failed" + return 0 +} + +function main() { + local failed=() + + build_image + create_cluster + install_arc + install_scale_set + + upgrade_scale_set || failed+=("upgrade_scale_set") + + WORKFLOW_FILE="${WORKFLOW_FILE}" SCALE_SET_NAME="${SCALE_SET_NAME}" run_workflow || failed+=("run_workflow") + + INSTALLATION_NAME="${SCALE_SET_NAME}" NAMESPACE="${SCALE_SET_NAMESPACE}" cleanup_scale_set || failed+=("cleanup_scale_set") + + NAMESPACE="${ARC_NAMESPACE}" log_arc || failed+=("log_arc") + + delete_cluster + + print_results "${failed[@]}" +} + +main diff --git a/test/actions.github.com/update-gha-runner-scale-set.test.sh b/test/actions.github.com/update-gha-runner-scale-set.test.sh new file mode 100755 index 0000000000..a7076340b1 --- /dev/null +++ b/test/actions.github.com/update-gha-runner-scale-set.test.sh @@ -0,0 +1,450 @@ +#!/bin/bash + +set -euo pipefail + +DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" + +ROOT_DIR="$(realpath "${DIR}/../..")" + +source "${DIR}/helper.sh" || { + echo "Failed to source helper.sh" + exit 1 +} + +export VERSION="$(chart_version "${ROOT_DIR}/charts/gha-runner-scale-set-controller/Chart.yaml")" + +SCALE_SET_NAME="update-strategy-$(date '+%M%S')$(((RANDOM + 100) % 100 + 1))" +SCALE_SET_NAMESPACE="arc-runners" +WORKFLOW_FILE="arc-test-sleepy-matrix.yaml" +ARC_NAME="arc" +ARC_NAMESPACE="arc-systems" + +function install_arc() { + echo "Installing ARC" + helm install "${ARC_NAME}" \ + --namespace "${ARC_NAMESPACE}" \ + --create-namespace \ + --set image.repository="${IMAGE_NAME}" \ + --set image.tag="${IMAGE_TAG}" \ + "${ROOT_DIR}/charts/gha-runner-scale-set-controller" \ + --debug + + if ! NAME="${ARC_NAME}" NAMESPACE="${ARC_NAMESPACE}" wait_for_arc; then + NAMESPACE="${ARC_NAMESPACE}" log_arc + return 1 + fi +} + +function install_scale_set() { + echo "Installing scale set ${SCALE_SET_NAME}/${SCALE_SET_NAMESPACE}" + helm install "${SCALE_SET_NAME}" \ + --namespace "${SCALE_SET_NAMESPACE}" \ + --create-namespace \ + --set githubConfigUrl="https://github.com/${TARGET_ORG}/${TARGET_REPO}" \ + --set githubConfigSecret.github_token="${GITHUB_TOKEN}" \ + "${ROOT_DIR}/charts/gha-runner-scale-set" \ + --debug + + if ! NAME="${SCALE_SET_NAME}" NAMESPACE="${ARC_NAMESPACE}" wait_for_scale_set; then + NAMESPACE="${ARC_NAMESPACE}" log_arc + return 1 + fi +} + +function upgrade_scale_set() { + echo "Upgrading scale set ${SCALE_SET_NAME}/${SCALE_SET_NAMESPACE}" + + UPGRADE_MARKER="e2e-upgrade-${SCALE_SET_NAME}-$(date +%s)" + echo "Generated upgrade marker: ${UPGRADE_MARKER}" + + PATCH_APPLIED_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + echo "Captured PATCH_APPLIED_TIME: ${PATCH_APPLIED_TIME}" + + helm upgrade "${SCALE_SET_NAME}" \ + --namespace "${SCALE_SET_NAMESPACE}" \ + --set githubConfigUrl="https://github.com/${TARGET_ORG}/${TARGET_REPO}" \ + --set githubConfigSecret.github_token="${GITHUB_TOKEN}" \ + --set template.spec.containers[0].name="runner" \ + --set template.spec.containers[0].image="ghcr.io/actions/actions-runner:latest" \ + --set template.spec.containers[0].command={"/home/runner/run.sh"} \ + --set template.spec.containers[0].env[0].name="TEST" \ + --set template.spec.containers[0].env[0].value="E2E TESTS" \ + --set "template.metadata.labels.e2e\.arc/upgrade-marker=${UPGRADE_MARKER}" \ + "${ROOT_DIR}/charts/gha-runner-scale-set" \ + --version="${VERSION}" \ + --debug + +} + +function capture_pre_upgrade_state() { + echo "Capturing pre-upgrade state for scale set ${SCALE_SET_NAME}" + + # Capture listener pod UID + local listener_json + listener_json=$(kubectl get pods -n "${ARC_NAMESPACE}" \ + -l "actions.github.com/scale-set-name=${SCALE_SET_NAME}" \ + --field-selector=status.phase=Running \ + -o json) || { + echo "ERROR: kubectl failed to fetch listener pods" + echo "Namespace: ${ARC_NAMESPACE}" + echo "Selector: actions.github.com/scale-set-name=${SCALE_SET_NAME}" + return 1 + } + + local listener_count + listener_count=$(echo "${listener_json}" | jq '.items | length') + + if [ "${listener_count}" -ne 1 ]; then + echo "ERROR: Expected exactly 1 running listener pod, found ${listener_count}" + echo "Selector: actions.github.com/scale-set-name=${SCALE_SET_NAME}" + echo "Namespace: ${ARC_NAMESPACE}" + echo "Observed pods: ${listener_json}" + return 1 + fi + + PRE_UPGRADE_LISTENER_UID=$(echo "${listener_json}" | jq -r '.items[0].metadata.uid') + + # Capture EphemeralRunnerSet UID + local ers_json + ers_json=$(kubectl get autoscalingrunnersets -n "${SCALE_SET_NAMESPACE}" \ + -l "app.kubernetes.io/instance=${SCALE_SET_NAME}" \ + -o json) || { + echo "ERROR: kubectl failed to fetch EphemeralRunnerSet" + echo "Namespace: ${SCALE_SET_NAMESPACE}" + echo "Selector: app.kubernetes.io/instance=${SCALE_SET_NAME}" + return 1 + } + + local ers_count + ers_count=$(echo "${ers_json}" | jq '.items | length') + + if [ "${ers_count}" -ne 1 ]; then + echo "ERROR: Expected exactly 1 EphemeralRunnerSet, found ${ers_count}" + echo "Selector: app.kubernetes.io/instance=${SCALE_SET_NAME}" + echo "Namespace: ${SCALE_SET_NAMESPACE}" + echo "Observed resources: ${ers_json}" + return 1 + fi + + PRE_UPGRADE_ERS_UID=$(echo "${ers_json}" | jq -r '.items[0].metadata.uid') + + # Capture timestamp + PRE_UPGRADE_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + + echo "PRE_UPGRADE_LISTENER_UID=${PRE_UPGRADE_LISTENER_UID}" + echo "PRE_UPGRADE_ERS_UID=${PRE_UPGRADE_ERS_UID}" + echo "PRE_UPGRADE_TIME=${PRE_UPGRADE_TIME}" + + return 0 +} + +function assert_listener_stays_up() { + echo "Asserting listener remains continuously running with unchanged UID" + + if [ -z "${PRE_UPGRADE_LISTENER_UID:-}" ]; then + echo "ERROR: PRE_UPGRADE_LISTENER_UID not set. Call capture_pre_upgrade_state first." + return 1 + fi + + local count=0 + local max_iterations=120 + local sleep_interval=1 + + echo "Starting continuous verification for ${max_iterations}s (interval: ${sleep_interval}s)" + echo "Expected listener UID: ${PRE_UPGRADE_LISTENER_UID}" + echo "Selector: actions.github.com/scale-set-name=${SCALE_SET_NAME}" + echo "Namespace: ${ARC_NAMESPACE}" + + while [ "${count}" -lt "${max_iterations}" ]; do + # Fetch current listener pods + local listener_json + listener_json=$(kubectl get pods -n "${ARC_NAMESPACE}" \ + -l "actions.github.com/scale-set-name=${SCALE_SET_NAME}" \ + --field-selector=status.phase=Running \ + -o json) || { + echo "ERROR: kubectl failed to fetch listener pods during verification" + echo "Namespace: ${ARC_NAMESPACE}" + echo "Selector: actions.github.com/scale-set-name=${SCALE_SET_NAME}" + echo "Time elapsed: ${count}s" + return 1 + } + + local listener_count + listener_count=$(echo "${listener_json}" | jq '.items | length') + + # Check listener count is exactly 1 + if [ "${listener_count}" -ne 1 ]; then + echo "FAIL: Expected exactly 1 running listener, found ${listener_count}" + echo "Time elapsed: ${count}s" + echo "Expected UID: ${PRE_UPGRADE_LISTENER_UID}" + echo "Selector: actions.github.com/scale-set-name=${SCALE_SET_NAME}" + echo "Namespace: ${ARC_NAMESPACE}" + echo "" + echo "Listener pods detail:" + kubectl get pods -n "${ARC_NAMESPACE}" \ + -l "actions.github.com/scale-set-name=${SCALE_SET_NAME}" \ + -o json | jq '.items[] | {name: .metadata.name, uid: .metadata.uid, phase: .status.phase, creationTimestamp: .metadata.creationTimestamp}' || echo "WARNING: kubectl diagnostic command failed" + echo "" + echo "All pods in all namespaces:" + kubectl get pods -A || echo "WARNING: kubectl diagnostic command failed" + return 1 + fi + + # Extract current listener UID + local current_uid + current_uid=$(echo "${listener_json}" | jq -r '.items[0].metadata.uid') + + # Check UID has not changed + if [ "${current_uid}" != "${PRE_UPGRADE_LISTENER_UID}" ]; then + echo "FAIL: Listener UID changed from ${PRE_UPGRADE_LISTENER_UID} to ${current_uid}" + echo "Time elapsed: ${count}s" + echo "Expected UID: ${PRE_UPGRADE_LISTENER_UID}" + echo "Observed UID: ${current_uid}" + echo "Selector: actions.github.com/scale-set-name=${SCALE_SET_NAME}" + echo "Namespace: ${ARC_NAMESPACE}" + echo "" + echo "Current listener pod detail:" + kubectl get pods -n "${ARC_NAMESPACE}" \ + -l "actions.github.com/scale-set-name=${SCALE_SET_NAME}" \ + -o json | jq '.items[0] | {name: .metadata.name, uid: .metadata.uid, phase: .status.phase, creationTimestamp: .metadata.creationTimestamp}' || echo "WARNING: kubectl diagnostic command failed" + echo "" + echo "All pods in all namespaces:" + kubectl get pods -A || echo "WARNING: kubectl diagnostic command failed" + return 1 + fi + + # Progress indicator every 10 seconds + if [ $((count % 10)) -eq 0 ]; then + echo "Verification ongoing: ${count}s elapsed | Listener count: ${listener_count} | UID: ${current_uid}" + fi + + sleep "${sleep_interval}" + count=$((count + sleep_interval)) + done + + echo "SUCCESS: Listener remained continuously running for ${max_iterations}s with unchanged UID" + echo "Final listener UID: ${PRE_UPGRADE_LISTENER_UID}" + echo "Selector: actions.github.com/scale-set-name=${SCALE_SET_NAME}" + echo "Namespace: ${ARC_NAMESPACE}" + echo "" + echo "Final listener pod detail:" + kubectl get pods -n "${ARC_NAMESPACE}" \ + -l "actions.github.com/scale-set-name=${SCALE_SET_NAME}" \ + -o json | jq '.items[0] | {name: .metadata.name, uid: .metadata.uid, phase: .status.phase, creationTimestamp: .metadata.creationTimestamp}' || echo "WARNING: kubectl diagnostic command failed" + return 0 +} + +function assert_marked_runner_spawned_after_upgrade() { + echo "Asserting at least one runner pod with upgrade marker spawned after patch application" + + if [ -z "${UPGRADE_MARKER:-}" ]; then + echo "ERROR: UPGRADE_MARKER not set. Call upgrade_scale_set first." + return 1 + fi + + if [ -z "${PATCH_APPLIED_TIME:-}" ]; then + echo "ERROR: PATCH_APPLIED_TIME not set. Call upgrade_scale_set first." + return 1 + fi + + local count=0 + local max_iterations=120 + local sleep_interval=1 + + echo "Starting verification for ${max_iterations}s (interval: ${sleep_interval}s)" + echo "Expected marker: ${UPGRADE_MARKER}" + echo "Expected creationTimestamp > ${PATCH_APPLIED_TIME}" + echo "Selector: app.kubernetes.io/component=runner,e2e.arc/upgrade-marker=${UPGRADE_MARKER}" + echo "Namespace: ${SCALE_SET_NAMESPACE}" + + while [ "${count}" -lt "${max_iterations}" ]; do + # Fetch runner pods with marker label + local runner_json + runner_json=$(kubectl get pods -n "${SCALE_SET_NAMESPACE}" \ + -l "app.kubernetes.io/component=runner,e2e.arc/upgrade-marker=${UPGRADE_MARKER}" \ + -o json) || { + echo "ERROR: kubectl failed to fetch runner pods during verification" + echo "Namespace: ${SCALE_SET_NAMESPACE}" + echo "Selector: app.kubernetes.io/component=runner,e2e.arc/upgrade-marker=${UPGRADE_MARKER}" + echo "Time elapsed: ${count}s" + return 1 + } + + local runner_count + runner_count=$(echo "${runner_json}" | jq '.items | length') + + if [ "${runner_count}" -eq 0 ]; then + if [ $((count % 10)) -eq 0 ]; then + echo "No runners with marker found yet. Elapsed: ${count}s" + fi + sleep "${sleep_interval}" + count=$((count + sleep_interval)) + continue + fi + + # Filter pods created after PATCH_APPLIED_TIME + local matching_runners + matching_runners=$(echo "${runner_json}" | jq -r --arg patch_time "${PATCH_APPLIED_TIME}" \ + '.items[] | select(.metadata.creationTimestamp > $patch_time) | .metadata.name') + + local matching_count + matching_count=$(echo "${matching_runners}" | grep -c . || true) + + if [ "${matching_count}" -gt 0 ]; then + # Check if at least one is Running + local running_count + running_count=$(echo "${runner_json}" | jq -r --arg patch_time "${PATCH_APPLIED_TIME}" \ + '[.items[] | select(.metadata.creationTimestamp > $patch_time and .status.phase == "Running")] | length') + + if [ "${running_count}" -gt 0 ]; then + echo "SUCCESS: Found ${running_count} running runner(s) with marker spawned after upgrade" + echo "Marker: ${UPGRADE_MARKER}" + echo "Patch applied time: ${PATCH_APPLIED_TIME}" + echo "Matching pods:" + echo "${matching_runners}" + echo "" + echo "Detailed pod information:" + kubectl get pods -n "${SCALE_SET_NAMESPACE}" \ + -l "app.kubernetes.io/component=runner,e2e.arc/upgrade-marker=${UPGRADE_MARKER}" \ + -o json | jq '.items[] | select(.metadata.creationTimestamp > "'${PATCH_APPLIED_TIME}'") | {name: .metadata.name, uid: .metadata.uid, phase: .status.phase, creationTimestamp: .metadata.creationTimestamp, "upgrade-marker": .metadata.labels."e2e.arc/upgrade-marker"}' || echo "WARNING: kubectl diagnostic command failed" + return 0 + else + echo "Found ${matching_count} matching pod(s), but none are Running yet. Elapsed: ${count}s" + fi + fi + + # Progress indicator every 10 seconds + if [ $((count % 10)) -eq 0 ]; then + echo "Verification ongoing: ${count}s elapsed | Runners with marker: ${runner_count} | Matching time: ${matching_count}" + fi + + sleep "${sleep_interval}" + count=$((count + sleep_interval)) + done + + # Timeout - dump diagnostics + echo "FAIL: No running runner pod with marker spawned after upgrade within ${max_iterations}s" + echo "Expected marker: ${UPGRADE_MARKER}" + echo "Expected creationTimestamp > ${PATCH_APPLIED_TIME}" + echo "Selector: app.kubernetes.io/component=runner,e2e.arc/upgrade-marker=${UPGRADE_MARKER}" + echo "Namespace: ${SCALE_SET_NAMESPACE}" + echo "" + echo "Candidate pods with marker (regardless of timestamp):" + kubectl get pods -n "${SCALE_SET_NAMESPACE}" \ + -l "app.kubernetes.io/component=runner,e2e.arc/upgrade-marker=${UPGRADE_MARKER}" \ + -o json | jq -r '.items[] | "Pod: \(.metadata.name) | UID: \(.metadata.uid) | Created: \(.metadata.creationTimestamp) | Phase: \(.status.phase) | Marker: \(.metadata.labels."e2e.arc/upgrade-marker")"' || echo "WARNING: kubectl diagnostic command failed" + + echo "" + echo "All runner pods in namespace (for timestamp/label analysis):" + kubectl get pods -n "${SCALE_SET_NAMESPACE}" -l "app.kubernetes.io/component=runner" \ + -o json | jq '.items[] | {name: .metadata.name, uid: .metadata.uid, phase: .status.phase, creationTimestamp: .metadata.creationTimestamp, "upgrade-marker": .metadata.labels."e2e.arc/upgrade-marker"}' || echo "WARNING: kubectl diagnostic command failed" + + return 1 +} + +function assert_ephemeral_runner_set_stays_up() { + echo "Asserting EphemeralRunnerSet remains unchanged and stays up" + + if [ -z "${PRE_UPGRADE_ERS_UID:-}" ]; then + echo "ERROR: PRE_UPGRADE_ERS_UID not set. Call capture_pre_upgrade_state first." + return 1 + fi + + # Get current ERS + local ers_json + ers_json=$(kubectl get autoscalingrunnersets -n "${SCALE_SET_NAMESPACE}" \ + -l "app.kubernetes.io/instance=${SCALE_SET_NAME}" \ + -o json) || { + echo "ERROR: kubectl failed to fetch EphemeralRunnerSet" + echo "Namespace: ${SCALE_SET_NAMESPACE}" + echo "Selector: app.kubernetes.io/instance=${SCALE_SET_NAME}" + return 1 + } + + local ers_count + ers_count=$(echo "${ers_json}" | jq '.items | length') + + # Validate exactly one ERS exists + if [ "${ers_count}" -ne 1 ]; then + echo "FAIL: Expected exactly 1 EphemeralRunnerSet, found ${ers_count}" + echo "Expected UID: ${PRE_UPGRADE_ERS_UID}" + echo "Selector: app.kubernetes.io/instance=${SCALE_SET_NAME}" + echo "Namespace: ${SCALE_SET_NAMESPACE}" + echo "" + echo "Observed ERS resources:" + kubectl get autoscalingrunnersets -n "${SCALE_SET_NAMESPACE}" \ + -l "app.kubernetes.io/instance=${SCALE_SET_NAME}" \ + -o json | jq '.items[] | {name: .metadata.name, uid: .metadata.uid, generation: .metadata.generation, observedGeneration: .status.observedGeneration, creationTimestamp: .metadata.creationTimestamp}' || echo "WARNING: kubectl diagnostic command failed" + return 1 + fi + + # Extract current ERS UID + local current_ers_uid + current_ers_uid=$(echo "${ers_json}" | jq -r '.items[0].metadata.uid') + + # Validate UID has not changed + if [ "${current_ers_uid}" != "${PRE_UPGRADE_ERS_UID}" ]; then + echo "FAIL: ERS UID changed from ${PRE_UPGRADE_ERS_UID} to ${current_ers_uid}" + echo "ERS was recreated during upgrade, indicating pod replacement instead of true update" + echo "Expected UID: ${PRE_UPGRADE_ERS_UID}" + echo "Observed UID: ${current_ers_uid}" + echo "Selector: app.kubernetes.io/instance=${SCALE_SET_NAME}" + echo "Namespace: ${SCALE_SET_NAMESPACE}" + echo "" + echo "Full ERS metadata:" + kubectl get autoscalingrunnersets -n "${SCALE_SET_NAMESPACE}" \ + -l "app.kubernetes.io/instance=${SCALE_SET_NAME}" \ + -o json | jq '.items[0].metadata' || echo "WARNING: kubectl diagnostic command failed" + echo "" + echo "ERS status:" + kubectl get autoscalingrunnersets -n "${SCALE_SET_NAMESPACE}" \ + -l "app.kubernetes.io/instance=${SCALE_SET_NAME}" \ + -o json | jq '.items[0].status' || echo "WARNING: kubectl diagnostic command failed" + return 1 + fi + + # Optionally verify status.currentReplicas is queryable + local current_replicas + current_replicas=$(echo "${ers_json}" | jq -r '.items[0].status.currentReplicas // "N/A"') + local generation + generation=$(echo "${ers_json}" | jq -r '.items[0].metadata.generation // "N/A"') + local observedGeneration + observedGeneration=$(echo "${ers_json}" | jq -r '.items[0].status.observedGeneration // "N/A"') + + echo "SUCCESS: EphemeralRunnerSet remains unchanged" + echo "ERS UID: ${current_ers_uid}" + echo "Expected UID: ${PRE_UPGRADE_ERS_UID}" + echo "Generation: ${generation} | Observed Generation: ${observedGeneration}" + echo "Current Replicas: ${current_replicas}" + echo "Selector: app.kubernetes.io/instance=${SCALE_SET_NAME}" + echo "Namespace: ${SCALE_SET_NAMESPACE}" + echo "" + echo "ERS detail:" + kubectl get autoscalingrunnersets -n "${SCALE_SET_NAMESPACE}" \ + -l "app.kubernetes.io/instance=${SCALE_SET_NAME}" \ + -o json | jq '.items[0] | {name: .metadata.name, uid: .metadata.uid, generation: .metadata.generation, creationTimestamp: .metadata.creationTimestamp, observedGeneration: .status.observedGeneration, currentReplicas: .status.currentReplicas}' || echo "WARNING: kubectl diagnostic command failed" + return 0 +} + +function main() { + local failed=() + + build_image + create_cluster + install_arc + install_scale_set + + upgrade_scale_set || failed+=("upgrade_scale_set") + + WORKFLOW_FILE="${WORKFLOW_FILE}" SCALE_SET_NAME="${SCALE_SET_NAME}" run_workflow || failed+=("run_workflow") + + INSTALLATION_NAME="${SCALE_SET_NAME}" NAMESPACE="${SCALE_SET_NAMESPACE}" cleanup_scale_set || failed+=("cleanup_scale_set") + + NAMESPACE="${ARC_NAMESPACE}" log_arc || failed+=("log_arc") + + delete_cluster + + print_results "${failed[@]}" +} + +main diff --git a/test/actions.github.com/update-strategy-v2.test.sh b/test/actions.github.com/update-strategy-v2.test.sh deleted file mode 100755 index 8d27a64509..0000000000 --- a/test/actions.github.com/update-strategy-v2.test.sh +++ /dev/null @@ -1,150 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" - -ROOT_DIR="$(realpath "${DIR}/../..")" - -source "${DIR}/helper.sh" || { - echo "Failed to source helper.sh" - exit 1 -} - -export VERSION="$(chart_version "${ROOT_DIR}/charts/gha-runner-scale-set-controller-experimental/Chart.yaml")" - -SCALE_SET_NAME="update-strategy-$(date '+%M%S')$(((RANDOM + 100) % 100 + 1))" -SCALE_SET_NAMESPACE="arc-runners" -WORKFLOW_FILE="arc-test-sleepy-matrix.yaml" -ARC_NAME="arc" -ARC_NAMESPACE="arc-systems" - -function install_arc() { - echo "Installing ARC" - helm install "${ARC_NAME}" \ - --namespace "${ARC_NAMESPACE}" \ - --create-namespace \ - --set controller.manager.container.image="${IMAGE_NAME}:${IMAGE_TAG}" \ - --set controller.manager.config.updateStrategy="eventual" \ - "${ROOT_DIR}/charts/gha-runner-scale-set-controller-experimental" \ - --debug - - if ! NAME="${ARC_NAME}" NAMESPACE="${ARC_NAMESPACE}" wait_for_arc; then - NAMESPACE="${ARC_NAMESPACE}" log_arc - return 1 - fi -} - -function install_scale_set() { - echo "Installing scale set ${SCALE_SET_NAME}/${SCALE_SET_NAMESPACE}" - helm install "${SCALE_SET_NAME}" \ - --namespace "${SCALE_SET_NAMESPACE}" \ - --create-namespace \ - --set controllerServiceAccount.name="${ARC_NAME}-gha-rs-controller" \ - --set controllerServiceAccount.namespace="${ARC_NAMESPACE}" \ - --set auth.url="https://github.com/${TARGET_ORG}/${TARGET_REPO}" \ - --set auth.githubToken="${GITHUB_TOKEN}" \ - "${ROOT_DIR}/charts/gha-runner-scale-set-experimental" \ - --version="${VERSION}" \ - --debug - - if ! NAME="${SCALE_SET_NAME}" NAMESPACE="${ARC_NAMESPACE}" wait_for_scale_set; then - NAMESPACE="${ARC_NAMESPACE}" log_arc - return 1 - fi -} - -function upgrade_scale_set() { - echo "Upgrading scale set ${SCALE_SET_NAME}/${SCALE_SET_NAMESPACE}" - helm upgrade "${SCALE_SET_NAME}" \ - --namespace "${SCALE_SET_NAMESPACE}" \ - --set controllerServiceAccount.name="${ARC_NAME}-gha-rs-controller" \ - --set controllerServiceAccount.namespace="${ARC_NAMESPACE}" \ - --set auth.url="https://github.com/${TARGET_ORG}/${TARGET_REPO}" \ - --set auth.githubToken="${GITHUB_TOKEN}" \ - --set runner.container.image="ghcr.io/actions/actions-runner:latest" \ - --set runner.container.command={"/home/runner/run.sh"} \ - --set runner.env[0].name="TEST" \ - --set runner.env[0].value="E2E TESTS" \ - "${ROOT_DIR}/charts/gha-runner-scale-set-experimental" \ - --version="${VERSION}" \ - --debug - -} - -function assert_listener_deleted() { - local count=0 - while true; do - LISTENER_COUNT="$(kubectl get pods -l actions.github.com/scale-set-name="${SCALE_SET_NAME}" -n "${ARC_NAMESPACE}" --field-selector=status.phase=Running -o=jsonpath='{.items}' | jq 'length')" - RUNNERS_COUNT="$(kubectl get pods -l app.kubernetes.io/component=runner -n "${SCALE_SET_NAMESPACE}" --field-selector=status.phase=Running -o=jsonpath='{.items}' | jq 'length')" - RESOURCES="$(kubectl get pods -A)" - - if [ "${LISTENER_COUNT}" -eq 0 ]; then - echo "Listener has been deleted" - echo "${RESOURCES}" - return 0 - fi - if [ "${count}" -ge 60 ]; then - echo "Timeout waiting for listener to be deleted" - echo "${RESOURCES}" - return 1 - fi - - echo "Waiting for listener to be deleted" - echo "Listener count: ${LISTENER_COUNT} target: 0 | Runners count: ${RUNNERS_COUNT} target: 3" - - sleep 1 - count=$((count + 1)) - done -} - -function assert_listener_recreated() { - count=0 - while true; do - LISTENER_COUNT="$(kubectl get pods -l actions.github.com/scale-set-name="${SCALE_SET_NAME}" -n "${ARC_NAMESPACE}" --field-selector=status.phase=Running -o=jsonpath='{.items}' | jq 'length')" - RUNNERS_COUNT="$(kubectl get pods -l app.kubernetes.io/component=runner -n "${SCALE_SET_NAMESPACE}" --field-selector=status.phase=Running -o=jsonpath='{.items}' | jq 'length')" - RESOURCES="$(kubectl get pods -A)" - - if [ "${LISTENER_COUNT}" -eq 1 ]; then - echo "Listener is up!" - echo "${RESOURCES}" - return 0 - fi - if [ "${count}" -ge 120 ]; then - echo "Timeout waiting for listener to be recreated" - echo "${RESOURCES}" - return 1 - fi - - echo "Waiting for listener to be recreated" - echo "Listener count: ${LISTENER_COUNT} target: 1 | Runners count: ${RUNNERS_COUNT} target: 0" - - sleep 1 - count=$((count + 1)) - done -} - -function main() { - local failed=() - - build_image - create_cluster - install_arc - install_scale_set - - WORKFLOW_FILE="${WORKFLOW_FILE}" SCALE_SET_NAME="${SCALE_SET_NAME}" run_workflow || failed+=("run_workflow") - - upgrade_scale_set || failed+=("upgrade_scale_set") - assert_listener_deleted || failed+=("assert_listener_deleted") - assert_listener_recreated || failed+=("assert_listener_recreated") - - INSTALLATION_NAME="${SCALE_SET_NAME}" NAMESPACE="${SCALE_SET_NAMESPACE}" cleanup_scale_set || failed+=("cleanup_scale_set") - - NAMESPACE="${ARC_NAMESPACE}" log_arc || failed+=("log_arc") - - delete_cluster - - print_results "${failed[@]}" -} - -main diff --git a/test/actions.github.com/update-strategy.test.sh b/test/actions.github.com/update-strategy.test.sh deleted file mode 100755 index 7ddff6e9ee..0000000000 --- a/test/actions.github.com/update-strategy.test.sh +++ /dev/null @@ -1,147 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" - -ROOT_DIR="$(realpath "${DIR}/../..")" - -source "${DIR}/helper.sh" || { - echo "Failed to source helper.sh" - exit 1 -} - -export VERSION="$(chart_version "${ROOT_DIR}/charts/gha-runner-scale-set-controller/Chart.yaml")" - -SCALE_SET_NAME="update-strategy-$(date '+%M%S')$(((RANDOM + 100) % 100 + 1))" -SCALE_SET_NAMESPACE="arc-runners" -WORKFLOW_FILE="arc-test-sleepy-matrix.yaml" -ARC_NAME="arc" -ARC_NAMESPACE="arc-systems" - -function install_arc() { - echo "Installing ARC" - helm install "${ARC_NAME}" \ - --namespace "${ARC_NAMESPACE}" \ - --create-namespace \ - --set image.repository="${IMAGE_NAME}" \ - --set image.tag="${IMAGE_TAG}" \ - --set flags.updateStrategy="eventual" \ - "${ROOT_DIR}/charts/gha-runner-scale-set-controller" \ - --debug - - if ! NAME="${ARC_NAME}" NAMESPACE="${ARC_NAMESPACE}" wait_for_arc; then - NAMESPACE="${ARC_NAMESPACE}" log_arc - return 1 - fi -} - -function install_scale_set() { - echo "Installing scale set ${SCALE_SET_NAME}/${SCALE_SET_NAMESPACE}" - helm install "${SCALE_SET_NAME}" \ - --namespace "${SCALE_SET_NAMESPACE}" \ - --create-namespace \ - --set githubConfigUrl="https://github.com/${TARGET_ORG}/${TARGET_REPO}" \ - --set githubConfigSecret.github_token="${GITHUB_TOKEN}" \ - "${ROOT_DIR}/charts/gha-runner-scale-set" \ - --debug - - if ! NAME="${SCALE_SET_NAME}" NAMESPACE="${ARC_NAMESPACE}" wait_for_scale_set; then - NAMESPACE="${ARC_NAMESPACE}" log_arc - return 1 - fi -} - -function upgrade_scale_set() { - echo "Upgrading scale set ${SCALE_SET_NAME}/${SCALE_SET_NAMESPACE}" - helm upgrade "${SCALE_SET_NAME}" \ - --namespace "${SCALE_SET_NAMESPACE}" \ - --set githubConfigUrl="https://github.com/${TARGET_ORG}/${TARGET_REPO}" \ - --set githubConfigSecret.github_token="${GITHUB_TOKEN}" \ - --set template.spec.containers[0].name="runner" \ - --set template.spec.containers[0].image="ghcr.io/actions/actions-runner:latest" \ - --set template.spec.containers[0].command={"/home/runner/run.sh"} \ - --set template.spec.containers[0].env[0].name="TEST" \ - --set template.spec.containers[0].env[0].value="E2E TESTS" \ - "${ROOT_DIR}/charts/gha-runner-scale-set" \ - --version="${VERSION}" \ - --debug - -} - -function assert_listener_deleted() { - local count=0 - while true; do - LISTENER_COUNT="$(kubectl get pods -l actions.github.com/scale-set-name="${SCALE_SET_NAME}" -n "${ARC_NAMESPACE}" --field-selector=status.phase=Running -o=jsonpath='{.items}' | jq 'length')" - RUNNERS_COUNT="$(kubectl get pods -l app.kubernetes.io/component=runner -n "${SCALE_SET_NAMESPACE}" --field-selector=status.phase=Running -o=jsonpath='{.items}' | jq 'length')" - RESOURCES="$(kubectl get pods -A)" - - if [ "${LISTENER_COUNT}" -eq 0 ]; then - echo "Listener has been deleted" - echo "${RESOURCES}" - return 0 - fi - if [ "${count}" -ge 60 ]; then - echo "Timeout waiting for listener to be deleted" - echo "${RESOURCES}" - return 1 - fi - - echo "Waiting for listener to be deleted" - echo "Listener count: ${LISTENER_COUNT} target: 0 | Runners count: ${RUNNERS_COUNT} target: 3" - - sleep 1 - count=$((count + 1)) - done -} - -function assert_listener_recreated() { - count=0 - while true; do - LISTENER_COUNT="$(kubectl get pods -l actions.github.com/scale-set-name="${SCALE_SET_NAME}" -n "${ARC_NAMESPACE}" --field-selector=status.phase=Running -o=jsonpath='{.items}' | jq 'length')" - RUNNERS_COUNT="$(kubectl get pods -l app.kubernetes.io/component=runner -n "${SCALE_SET_NAMESPACE}" --field-selector=status.phase=Running -o=jsonpath='{.items}' | jq 'length')" - RESOURCES="$(kubectl get pods -A)" - - if [ "${LISTENER_COUNT}" -eq 1 ]; then - echo "Listener is up!" - echo "${RESOURCES}" - return 0 - fi - if [ "${count}" -ge 120 ]; then - echo "Timeout waiting for listener to be recreated" - echo "${RESOURCES}" - return 1 - fi - - echo "Waiting for listener to be recreated" - echo "Listener count: ${LISTENER_COUNT} target: 1 | Runners count: ${RUNNERS_COUNT} target: 0" - - sleep 1 - count=$((count + 1)) - done -} - -function main() { - local failed=() - - build_image - create_cluster - install_arc - install_scale_set - - WORKFLOW_FILE="${WORKFLOW_FILE}" SCALE_SET_NAME="${SCALE_SET_NAME}" run_workflow || failed+=("run_workflow") - - upgrade_scale_set || failed+=("upgrade_scale_set") - assert_listener_deleted || failed+=("assert_listener_deleted") - assert_listener_recreated || failed+=("assert_listener_recreated") - - INSTALLATION_NAME="${SCALE_SET_NAME}" NAMESPACE="${SCALE_SET_NAMESPACE}" cleanup_scale_set || failed+=("cleanup_scale_set") - - NAMESPACE="${ARC_NAMESPACE}" log_arc || failed+=("log_arc") - - delete_cluster - - print_results "${failed[@]}" -} - -main