diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 073f19cd..a8089cef 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: ${REGISTRY_DOCKER_URL}/stackstate/stackstate-agent-integrations-runner:20241120-py311 +image: ${REGISTRY_DOCKER_URL}/stackstate/stackstate-agent-integrations-runner:${RUNNER_IMAGE_TAG} stages: - build - test @@ -6,7 +6,8 @@ stages: variables: CONDA_ENV: "stackstate-agent-integrations-py-3" - PYTHON_VERSION: 3.13.13 + PYTHON_VERSION: 3.13.14 + RUNNER_IMAGE_TAG: "20260625-py313" .rules: - &pull_requests @@ -24,7 +25,7 @@ variables: - $CHECK/* - $CHECK/**/* - &master_branch - if: '$CI_COMMIT_BRANCH == "stackstate-7.71.2"' + if: '$CI_COMMIT_BRANCH == "stackstate-7.78.2"' - &release_branch if: $CI_COMMIT_TAG - &base_manual_changes @@ -48,7 +49,7 @@ variables: - splunk_base/* - splunk_base/**/* - &master_manual_branch - if: '$CI_COMMIT_BRANCH == "stackstate-7.71.2"' + if: '$CI_COMMIT_BRANCH == "stackstate-7.78.2"' when: manual allow_failure: true - &release_manual_branch @@ -120,18 +121,41 @@ print_env: interruptible: true docker: - <<: *manual_job_rules image: ${REGISTRY_DOCKER_URL}/library/docker:20-git stage: build + rules: + - if: $CI_COMMIT_MESSAGE =~ /Build new docker image/i + when: on_success + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + changes: + - .setup-scripts/image/**/* + - .setup-scripts/setup_env.sh + when: manual + allow_failure: true + - if: $CI_EXTERNAL_PULL_REQUEST_IID + when: manual + allow_failure: true + - if: '$CI_COMMIT_BRANCH == "stackstate-7.78.2"' + when: manual + allow_failure: true + - if: $CI_COMMIT_TAG + when: manual + allow_failure: true script: - echo "${docker_password}" | docker login --username=${docker_user} --password-stdin docker.io - echo "${quay_password}" | docker login --username=${quay_user} --password-stdin quay.io - echo "${REGISTRY_PASSWORD}" | docker login --username=${REGISTRY_USER} --password-stdin ${REGISTRY_HOST} - apk add make - cd .setup-scripts/image - - make build - - make push - - if [ "${CI_COMMIT_REF_NAME}" = "stackstate-7.71.2" ]; then make tag_latest; make push_latest; fi + # Pull base from docker.io during build; mirror may not have 3.13.14-bookworm yet. + - make build push PYTHON_BASE_IMAGE=python:3.13.14-bookworm + - RUNNER_TAG=$(make -s print_tag) + - echo "Published ${REGISTRY_DOCKER_URL}/stackstate/stackstate-agent-integrations-runner:${RUNNER_TAG}" + - echo "Set RUNNER_IMAGE_TAG to ${RUNNER_TAG} (setup_env.sh/conda_env.ps1 already target python3.13/3.13.14), then re-run the pipeline." + - | + if [ "${CI_COMMIT_REF_NAME}" = "stackstate-7.78.2" ]; then + make tag_latest push_latest + fi services: - alias: docker command: @@ -386,12 +410,29 @@ test_static_health: DOCKER_DRIVER: overlay2 publish-checks-dev: + <<: *linux_env stage: publish interruptible: true + needs: + - job: linux_deps + artifacts: true script: - .setup-scripts/setup_artifact_registry.sh - .setup-scripts/setup_artifact_publishing.sh - export VERSION=`./.setup-scripts/version.sh` - echo "__version__ = \"$VERSION\"" > stackstate_checks_dev/stackstate_checks/dev/__about__.py - - cd stackstate_checks_dev && python setup.py sdist bdist_wheel upload -r gitlab - <<: *manual_job_rules + - pip install twine + - cd stackstate_checks_dev && python setup.py sdist bdist_wheel && twine upload --repository gitlab dist/* + rules: + - if: '$CI_COMMIT_BRANCH =~ /^STAC-/' + when: manual + allow_failure: true + - <<: *pull_requests + when: manual + allow_failure: true + - <<: *master_branch + when: manual + allow_failure: true + - <<: *release_branch + when: manual + allow_failure: true diff --git a/.python-version b/.python-version index 655354de..0cf960df 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.13.13 +3.13.14 diff --git a/.setup-scripts/conda_env.ps1 b/.setup-scripts/conda_env.ps1 index 59d3c359..33028b1b 100644 --- a/.setup-scripts/conda_env.ps1 +++ b/.setup-scripts/conda_env.ps1 @@ -1,16 +1,27 @@ $envName = $args[0] $pythonVersion = $args[1] if ($pythonVersion -eq '3') { - $pythonVersion = '3.11' + $pythonVersion = '3.13.14' } -$env_name = conda env list | grep $envName | awk '{print $1}' -if (($env_name -ne $null) -and ($env_name -eq $envName)) { + +$versionsFile = Join-Path $PSScriptRoot 'python_tool_versions.env' +if (Test-Path $versionsFile) { + Get-Content $versionsFile | ForEach-Object { + if ($_ -match '^\s*([A-Z_]+)\s*=\s*(.+?)\s*$') { + Set-Variable -Name $matches[1] -Value $matches[2] -Scope Script + } + } +} +if (-not $PIP_VERSION) { $PIP_VERSION = '24.3.1' } +if (-not $SETUPTOOLS_VERSION) { $SETUPTOOLS_VERSION = '75.8.2' } + +$envExists = conda env list | Select-String -Pattern "^\s*$([regex]::Escape($envName))\s" +if ($envExists) { Write-Output "Virtual Environment '$envName' already exists" return } -$DD_PIP_VERSION = '20.3.4' -$DD_SETUPTOOLS_VERSION = '44.1.1' + conda create -n $envName python python=$pythonVersion -y conda activate $envName -pip install --user -i https://pypi.python.org/simple pip==$DD_PIP_VERSION -pip install --ignore-installed setuptools==$DD_SETUPTOOLS_VERSION +pip install --user -i https://pypi.python.org/simple "pip==$PIP_VERSION" +pip install --ignore-installed "setuptools==$SETUPTOOLS_VERSION" diff --git a/.setup-scripts/image/Dockerfile b/.setup-scripts/image/Dockerfile index f3fcb82d..aa9de252 100644 --- a/.setup-scripts/image/Dockerfile +++ b/.setup-scripts/image/Dockerfile @@ -1,11 +1,18 @@ -FROM registry.tooling.stackstate.io/docker/python:3.11.14-bullseye +# Match agent embedded CPython 3.13.14. Docker Hub publishes 3.13.14 on bookworm, not bullseye. +# CI `docker` job passes PYTHON_BASE_IMAGE=python:3.13.14-bookworm (docker.io is logged in there). +ARG PYTHON_BASE_IMAGE=registry.tooling.stackstate.io/docker/python:3.13.14-bookworm +FROM ${PYTHON_BASE_IMAGE} +ARG PIP_VERSION=24.3.1 +ARG SETUPTOOLS_VERSION=75.8.2 RUN apt-get update && \ - apt-get install -y virtualenv apt-transport-https ca-certificates curl gnupg2 software-properties-common && \ + apt-get install -y apt-transport-https ca-certificates curl gnupg2 lsb-release && \ apt-get clean && \ - pip3 install -U pip setuptools codecov wheel + pip3 install "pip==${PIP_VERSION}" "setuptools==${SETUPTOOLS_VERSION}" virtualenv codecov wheel -RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - -RUN add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" && \ +RUN install -m 0755 -d /etc/apt/keyrings && \ + curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc && \ + chmod a+r /etc/apt/keyrings/docker.asc && \ + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null && \ apt-get update && \ apt-get install -y docker-ce docker-ce-cli containerd.io && \ apt-get clean diff --git a/.setup-scripts/image/Makefile b/.setup-scripts/image/Makefile index 71572d35..644bcc65 100644 --- a/.setup-scripts/image/Makefile +++ b/.setup-scripts/image/Makefile @@ -1,4 +1,10 @@ RUNTIMETAG := $(shell date +%Y%m%d) +PYTHON_SUFFIX := py313 +IMAGE_REPO ?= stackstate/stackstate-agent-integrations-runner +# CI `docker` job overrides with python:3.13.14-bookworm (docker.io login); local default is the mirror. +PYTHON_BASE_IMAGE ?= registry.tooling.stackstate.io/docker/python:3.13.14-bookworm +include ../python_tool_versions.env +export #build: # docker build -t stackstate/stackstate-agent-integrations-runner:$(RUNTIMETAG) . @@ -16,12 +22,20 @@ RUNTIMETAG := $(shell date +%Y%m%d) # vvv New runner build: - docker build -t stackstate/stackstate-agent-integrations-runner:$(RUNTIMETAG)-py311 . + docker build --build-arg PYTHON_BASE_IMAGE=$(PYTHON_BASE_IMAGE) \ + --build-arg PIP_VERSION=$(PIP_VERSION) \ + --build-arg SETUPTOOLS_VERSION=$(SETUPTOOLS_VERSION) \ + -t $(IMAGE_REPO):$(RUNTIMETAG)-$(PYTHON_SUFFIX) . + push: - docker push stackstate/stackstate-agent-integrations-runner:$(RUNTIMETAG)-py311 + docker push $(IMAGE_REPO):$(RUNTIMETAG)-$(PYTHON_SUFFIX) tag_latest: - docker tag stackstate/stackstate-agent-integrations-runner:$(RUNTIMETAG)-py311 stackstate/stackstate-agent-integrations-runner:latest-py311 + docker tag $(IMAGE_REPO):$(RUNTIMETAG)-$(PYTHON_SUFFIX) $(IMAGE_REPO):latest-$(PYTHON_SUFFIX) push_latest: - docker push stackstate/stackstate-agent-integrations-runner:latest-py311 \ No newline at end of file + docker push $(IMAGE_REPO):latest-$(PYTHON_SUFFIX) + +# Print the tag CI should set as RUNNER_IMAGE_TAG after a successful publish. +print_tag: + @echo $(RUNTIMETAG)-$(PYTHON_SUFFIX) diff --git a/.setup-scripts/python_tool_versions.env b/.setup-scripts/python_tool_versions.env new file mode 100644 index 00000000..f4ccde95 --- /dev/null +++ b/.setup-scripts/python_tool_versions.env @@ -0,0 +1,3 @@ +# Python 3.13-compatible tool pins shared by CI setup, runner image, and tox. +PIP_VERSION=24.3.1 +SETUPTOOLS_VERSION=75.8.2 diff --git a/.setup-scripts/setup_artifact_publishing.sh b/.setup-scripts/setup_artifact_publishing.sh index 93052699..34c54255 100755 --- a/.setup-scripts/setup_artifact_publishing.sh +++ b/.setup-scripts/setup_artifact_publishing.sh @@ -13,7 +13,12 @@ if [ -n "$missing" ]; then fi echo "→ Configuring .pypirc for publishing to GitLab Package Registry..." -echo "GitLab PyPI URL: $GITLAB_PACKAGE_REGISTRY_PYPI_URL" + +PYPI_REPOSITORY_URL="${GITLAB_PACKAGE_REGISTRY_PYPI_URL}" +if [[ "${PYPI_REPOSITORY_URL}" != http://* && "${PYPI_REPOSITORY_URL}" != https://* ]]; then + PYPI_REPOSITORY_URL="https://${PYPI_REPOSITORY_URL}" +fi +echo "GitLab PyPI URL: ${PYPI_REPOSITORY_URL}" # setup .pypirc cat > ~/.pypirc < list -> str). + assert isinstance(entity.properties.customHostMetadata, list) + assert entity.properties.customHostMetadata == [ + {'value': 'VM4', 'key': 'ENVIRONMENT'}, + {'value': 'VM', 'key': 'VM'}, + ] + # Downstream serialization (used to build topology) must succeed and keep the shape. + dumped = entity.model_dump(mode="json", exclude_none=True) + assert dumped['properties']['customHostMetadata'] == [ + {'value': 'VM4', 'key': 'ENVIRONMENT'}, + {'value': 'VM', 'key': 'VM'}, + ] + + +def test_host_entity_customhostmetadata_dict_handling(): + """ + Test that HostEntity still accepts customHostMetadata as a dict (original/mock format). + """ + from stackstate_checks.dynatrace_topology.entity_data_types import HostEntity + + test_data = { + 'entityId': 'HOST-0C4456C4494C5BF5', + 'type': 'HOST', + 'displayName': 'lspe003490.eu.rabonet.com', + 'properties': { + 'customHostMetadata': { + 'ENVIRONMENT': 'VM4', + 'VM': 'VM', + } + } + } + + entity = HostEntity.model_validate(test_data) + + assert isinstance(entity.properties.customHostMetadata, dict) + assert entity.properties.customHostMetadata['ENVIRONMENT'] == 'VM4' + assert entity.properties.customHostMetadata['VM'] == 'VM' + + +def test_host_entity_customhostmetadata_string_handling(): + """ + Test that HostEntity tolerates customHostMetadata delivered as a plain string. + """ + from stackstate_checks.dynatrace_topology.entity_data_types import HostEntity + + test_data = { + 'entityId': 'HOST-0C4456C4494C5BF5', + 'type': 'HOST', + 'displayName': 'lspe003490.eu.rabonet.com', + 'properties': { + 'customHostMetadata': 'ENVIRONMENT=VM4' + } + } + + entity = HostEntity.model_validate(test_data) + + assert isinstance(entity.properties.customHostMetadata, str) + assert entity.properties.customHostMetadata == 'ENVIRONMENT=VM4' + + +def test_host_entity_customhostmetadata_other_type_fallback(): + """ + Test that a customHostMetadata value that is neither dict, list nor string falls back + to its string representation instead of dropping the host entity. + """ + from stackstate_checks.dynatrace_topology.entity_data_types import HostEntity + + test_data = { + 'entityId': 'HOST-0C4456C4494C5BF5', + 'type': 'HOST', + 'displayName': 'lspe003490.eu.rabonet.com', + 'properties': { + 'customHostMetadata': 42 + } + } + + entity = HostEntity.model_validate(test_data) + + assert isinstance(entity.properties.customHostMetadata, str) + assert entity.properties.customHostMetadata == '42' + + def test_entity_cache_resets_between_runs(requests_mock, dynatrace_check, topology, aggregator): """ Verify that the in-memory entity cache is cleared at the start of each check run. diff --git a/kubelet/tox.ini b/kubelet/tox.ini index 1da311ff..c13793de 100755 --- a/kubelet/tox.ini +++ b/kubelet/tox.ini @@ -5,7 +5,7 @@ envlist = py3 [testenv] -pip_version = pip==23.3.1 +pip_version = pip==24.3.1 dd_check_style = true usedevelop = true platform = linux|darwin|win32 diff --git a/openmetrics/tox.ini b/openmetrics/tox.ini index 34da2298..2e9abb90 100644 --- a/openmetrics/tox.ini +++ b/openmetrics/tox.ini @@ -7,7 +7,7 @@ envlist = flake8 [testenv] -pip_version = pip==23.3.1 +pip_version = pip==24.3.1 usedevelop = true platform = linux|darwin|win32 deps = diff --git a/servicenow/tests/test_servicenow.py b/servicenow/tests/test_servicenow.py index dd76327b..58e2b320 100644 --- a/servicenow/tests/test_servicenow.py +++ b/servicenow/tests/test_servicenow.py @@ -13,7 +13,6 @@ import mock import pytest import requests -from six import PY3 from stackstate_checks.base import AgentIntegrationTestUtil, AgentCheck, TopologyInstance from stackstate_checks.base.errors import CheckException @@ -723,12 +722,14 @@ def test_get_json_error_msg(self, mock_request_get): url='http://test.org') msg_py3 = 'Json parse error: "Expecting property name enclosed in double quotes: ' \ 'line 11 column 5 (char 232)" in response from url http://test.org' + msg_py3_13 = 'Json parse error: "Illegal trailing comma before end of object: ' \ + 'line 10 column 8 (char 226)" in response from url http://test.org' msg_py2 = 'Json parse error: "Expecting property name: ' \ 'line 11 column 5 (char 232)" in response from url http://test.org' - expected_msg = msg_py3 if PY3 else msg_py2 + expected_msgs = {msg_py2, msg_py3, msg_py3_13} with self.assertRaises(CheckException) as context: self.check._get_json(url, 10, {}, auth) - self.assertEqual(expected_msg, str(context.exception)) + self.assertIn(str(context.exception), expected_msgs) def test_process_components_encoding_errors(self): """ diff --git a/servicenow/tox.ini b/servicenow/tox.ini index e9b568c8..e3428858 100644 --- a/servicenow/tox.ini +++ b/servicenow/tox.ini @@ -7,7 +7,7 @@ envlist = flake8 [testenv] -pip_version = pip==23.3.1 +pip_version = pip==24.3.1 usedevelop = true platform = linux|darwin|win32 deps = diff --git a/splunk_base/tox.ini b/splunk_base/tox.ini index 2d029795..231d7401 100644 --- a/splunk_base/tox.ini +++ b/splunk_base/tox.ini @@ -6,7 +6,7 @@ envlist = flake8 [testenv] -pip_version = pip==23.3.1 +pip_version = pip==24.3.1 usedevelop = true platform = linux|darwin|win32 deps = diff --git a/splunk_health/tox.ini b/splunk_health/tox.ini index db477176..acda99e5 100644 --- a/splunk_health/tox.ini +++ b/splunk_health/tox.ini @@ -6,7 +6,7 @@ envlist = flake8 [testenv] -pip_version = pip==23.3.1 +pip_version = pip==24.3.1 usedevelop = true platform = linux|darwin|win32 passenv = diff --git a/splunk_metric/tox.ini b/splunk_metric/tox.ini index ec817186..6ca2610a 100644 --- a/splunk_metric/tox.ini +++ b/splunk_metric/tox.ini @@ -12,7 +12,7 @@ markers = unit [testenv] -pip_version = pip==23.3.1 +pip_version = pip==24.3.1 usedevelop = true platform = linux|darwin|win32 passenv = diff --git a/splunk_topology/tox.ini b/splunk_topology/tox.ini index db477176..acda99e5 100644 --- a/splunk_topology/tox.ini +++ b/splunk_topology/tox.ini @@ -6,7 +6,7 @@ envlist = flake8 [testenv] -pip_version = pip==23.3.1 +pip_version = pip==24.3.1 usedevelop = true platform = linux|darwin|win32 passenv = diff --git a/stackstate_checks_dev/tox.ini b/stackstate_checks_dev/tox.ini index 103947d4..06fb9daa 100644 --- a/stackstate_checks_dev/tox.ini +++ b/stackstate_checks_dev/tox.ini @@ -7,7 +7,7 @@ envlist = flake8 [testenv] -pip_version = pip==23.3.1 +pip_version = pip==24.3.1 usedevelop = true skip_install = true platform = linux|darwin|win32 diff --git a/static_health/tox.ini b/static_health/tox.ini index 2feace8c..20fbf39d 100644 --- a/static_health/tox.ini +++ b/static_health/tox.ini @@ -7,7 +7,7 @@ envlist = flake8 [testenv] -pip_version = pip==23.3.1 +pip_version = pip==24.3.1 usedevelop = true platform = linux|darwin|win32 deps = diff --git a/static_topology/tox.ini b/static_topology/tox.ini index 07475eb0..0a974d62 100644 --- a/static_topology/tox.ini +++ b/static_topology/tox.ini @@ -7,7 +7,7 @@ envlist = flake8 [testenv] -pip_version = pip==23.3.1 +pip_version = pip==24.3.1 usedevelop = true platform = linux|darwin|win32 deps = diff --git a/vsphere/tox.ini b/vsphere/tox.ini index 56fea2a5..3b0e5544 100644 --- a/vsphere/tox.ini +++ b/vsphere/tox.ini @@ -7,7 +7,7 @@ envlist = flake8 [testenv] -pip_version = pip==23.2.1 +pip_version = pip==24.3.1 usedevelop = true platform = linux|darwin|win32 deps = diff --git a/zabbix/tox.ini b/zabbix/tox.ini index f40cb76e..6792cdc5 100644 --- a/zabbix/tox.ini +++ b/zabbix/tox.ini @@ -7,7 +7,7 @@ envlist = flake8 [testenv] -pip_version = pip==23.3.1 +pip_version = pip==24.3.1 usedevelop = true platform = linux|darwin|win32 deps =