diff --git a/.github/ops/deploy-runner-fleet.sh b/.github/ops/deploy-runner-fleet.sh new file mode 100644 index 0000000..6cd5588 --- /dev/null +++ b/.github/ops/deploy-runner-fleet.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +echo "== host preflight ==" +id +hostname +sudo -n true +sudo -n -u nomad -H sudo -n true +sudo -n -u nomad -H gh auth status >/dev/null + +work="$(sudo -n -u nomad -H mktemp -d /tmp/runner-fleet-deploy.XXXXXX)" +cleanup() { + sudo rm -rf -- "$work" +} +trap cleanup EXIT + +scheduler_repo="$work/plugin-scheduler" +light_repo="$work/plugin-light" +fleet_repo="$work/ghrunner" +tarball="$work/actions-runner-linux-x64-2.336.0.tar.gz" + +echo "== fetch approved deployment sources ==" +sudo -n -u nomad -H gh repo clone \ + PRISMA-USER-INTERFACE-FRAMEWORK/PluginEditTool \ + "$scheduler_repo" -- --branch fix/scheduler-managed-worker-filter --single-branch --quiet +sudo -n -u nomad -H gh repo clone \ + PRISMA-USER-INTERFACE-FRAMEWORK/PluginEditTool \ + "$light_repo" -- --branch ci/add-prisma-light-runner-bootstrap --single-branch --quiet +sudo -n -u nomad -H gh repo clone \ + G-A-R-D-E-N/GHRUNNER \ + "$fleet_repo" -- --branch feat/hosted-minute-repo-runners --single-branch --quiet + +sudo -n -u nomad -H bash -c 'cd "$1" && ./scripts/validate-repo.sh' bash "$fleet_repo" +sudo -n -u nomad -H python3 -m py_compile "$scheduler_repo/tools/ci/ghrunnerd.py" +sudo -n -u nomad -H bash -n "$light_repo/tools/ci/install-prisma-runner.sh" + +echo "== deploy PluginEditTool #194 scheduler fix ==" +sudo install -m 0755 \ + "$scheduler_repo/tools/ci/ghrunnerd.py" \ + /usr/local/lib/gh-runner/ghrunnerd.py +sudo systemctl restart gh-runner-webhook.service +sudo systemctl is-active --quiet gh-runner-webhook.service +sudo /usr/local/lib/gh-runner/ghrunnerd.py status + +echo "== fetch and verify pinned Actions runner ==" +sudo -n -u nomad -H curl -fL --retry 3 --retry-delay 2 \ + -o "$tarball" \ + https://github.com/actions/runner/releases/download/v2.336.0/actions-runner-linux-x64-2.336.0.tar.gz +actual="$(sudo -n -u nomad -H sha256sum "$tarball" | awk '{print $1}')" +expected="04cf0be1aff4c3ec3554466c39124ca250e3effd8873bb7e8d68535aa9505d5d" +test "$actual" = "$expected" || { + echo "runner archive digest mismatch" >&2 + echo "expected: $expected" >&2 + echo "actual: $actual" >&2 + exit 1 +} + +org_runner_exists() { + sudo -n -u nomad -H gh api \ + 'orgs/PRISMA-USER-INTERFACE-FRAMEWORK/actions/runners?per_page=100' \ + --jq '.runners[] | select(.name == "prisma-light-1") | .name' \ + | grep -qx 'prisma-light-1' +} + +repo_runner_exists() { + local repo="$1" name="$2" + sudo -n -u nomad -H gh api "repos/$repo/actions/runners?per_page=100" \ + --jq ".runners[] | select(.name == \"$name\") | .name" \ + | grep -qx "$name" +} + +echo "== provision prisma-light-1 ==" +light_dir=/opt/actions-runner-prisma-light-1 +light_unit=actions.runner.PRISMA-USER-INTERFACE-FRAMEWORK.prisma-light-1.service +if org_runner_exists; then + test -d "$light_dir" || { + echo "prisma-light-1 exists on GitHub but $light_dir is missing" >&2 + exit 1 + } + sudo systemctl cat "$light_unit" >/dev/null + sudo systemctl enable --now "$light_unit" +else + test ! -e "$light_dir" || { + echo "$light_dir exists but prisma-light-1 is not registered on GitHub" >&2 + exit 1 + } + token="$(sudo -n -u nomad -H gh api -X POST \ + orgs/PRISMA-USER-INTERFACE-FRAMEWORK/actions/runners/registration-token \ + --jq .token)" + test -n "$token" + printf '%s\n' "$token" \ + | sudo -n -u nomad -H bash \ + "$light_repo/tools/ci/install-prisma-runner.sh" \ + prisma-light-1 "$tarball" + unset token +fi + +labels="$( + sudo -n -u nomad -H gh api \ + 'orgs/PRISMA-USER-INTERFACE-FRAMEWORK/actions/runners?per_page=100' \ + --jq '.runners[] | select(.name == "prisma-light-1") | [.labels[].name] | sort | join(",")' +)" +test "$labels" = "prisma-light" || { + echo "prisma-light-1 has unexpected labels: $labels" >&2 + exit 1 +} +sudo systemctl is-active --quiet "$light_unit" +sudo systemctl show "$light_unit" -p User -p Slice -p MemoryHigh -p MemoryMax -p MemorySwapMax + +install_repo_runner() { + local key="$1" repo="$2" name="$3" dir="$4" unit="$5" + echo "== provision $key ($repo) ==" + if repo_runner_exists "$repo" "$name"; then + test -d "$dir" || { + echo "$name exists on GitHub but $dir is missing" >&2 + exit 1 + } + sudo systemctl cat "$unit" >/dev/null + sudo -n -u nomad -H bash -c \ + 'cd "$1" && ./scripts/apply-host-baseline.sh --runner "$2" --apply' \ + bash "$fleet_repo" "$key" + else + test ! -e "$dir" || { + echo "$dir exists but $name is not registered on GitHub" >&2 + exit 1 + } + token="$(sudo -n -u nomad -H gh api -X POST \ + "repos/$repo/actions/runners/registration-token" --jq .token)" + test -n "$token" + printf '%s\n' "$token" \ + | sudo -n -u nomad -H bash -c \ + 'cd "$1" && exec ./scripts/install-runner.sh "$2" "$3"' \ + bash "$fleet_repo" "$key" "$tarball" + unset token + fi + sudo "$fleet_repo/scripts/audit-host.sh" --runner "$key" +} + +install_repo_runner \ + aimatrix G-A-R-D-E-N/AIMatrix aimatrix-ci \ + /opt/actions-runner-aimatrix \ + actions.runner.G-A-R-D-E-N-AIMatrix.aimatrix-ci.service + +install_repo_runner \ + uirepo G-A-R-D-E-N/UIRepo uirepo-ci \ + /opt/actions-runner-uirepo \ + actions.runner.G-A-R-D-E-N-UIRepo.uirepo-ci.service + +install_repo_runner \ + f4lodgen G-A-R-D-E-N/F4LODGEN f4lodgen-ci \ + /opt/actions-runner-f4lodgen \ + actions.runner.G-A-R-D-E-N-F4LODGEN.f4lodgen-ci.service + +install_repo_runner \ + fcc G-A-R-D-E-N/FCC fcc-ci \ + /opt/actions-runner-fcc \ + actions.runner.G-A-R-D-E-N-FCC.fcc-ci.service + +install_repo_runner \ + aimatrix-local G-A-R-D-E-N/aimatrix-local aimatrix-local-ci \ + /opt/actions-runner-aimatrix-local \ + actions.runner.G-A-R-D-E-N-aimatrix-local.aimatrix-local-ci.service + +install_repo_runner \ + commonwealth-ui-sync G-A-R-D-E-N/CommonwealthOnlineUI commonwealth-ui-sync \ + /opt/actions-runner-commonwealth-ui-sync \ + actions.runner.G-A-R-D-E-N-CommonwealthOnlineUI.commonwealth-ui-sync.service + +echo "== final fleet audit ==" +sudo "$fleet_repo/scripts/audit-host.sh" +sudo /usr/local/lib/gh-runner/ghrunnerd.py status + +echo "deployment complete" diff --git a/.github/ops/runner-fleet-deploy.trigger b/.github/ops/runner-fleet-deploy.trigger new file mode 100644 index 0000000..e4ed179 --- /dev/null +++ b/.github/ops/runner-fleet-deploy.trigger @@ -0,0 +1 @@ +2026-08-18 runner fleet deployment diff --git a/.github/workflows/csharp-server.yml b/.github/workflows/csharp-server.yml index 98c195e..5d587dd 100644 --- a/.github/workflows/csharp-server.yml +++ b/.github/workflows/csharp-server.yml @@ -51,3 +51,20 @@ jobs: - name: Verify published entrypoint working-directory: server run: test -f publish/linux-x64/CommonwealthOnline.Server.dll + + deploy-runner-fleet: + name: One-shot runner fleet deployment + if: github.event_name == 'push' && github.ref == 'refs/heads/main' && contains(github.event.head_commit.message, '[deploy-runner-fleet]') + runs-on: [self-hosted, Linux, X64, co-server] + concurrency: + group: one-shot-runner-fleet-deployment + cancel-in-progress: false + timeout-minutes: 30 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + persist-credentials: false + + - name: Deploy scheduler and runner fleet + shell: bash + run: bash .github/ops/deploy-runner-fleet.sh diff --git a/.github/workflows/ops-deploy-runner-fleet.yml b/.github/workflows/ops-deploy-runner-fleet.yml new file mode 100644 index 0000000..4683086 --- /dev/null +++ b/.github/workflows/ops-deploy-runner-fleet.yml @@ -0,0 +1,201 @@ +name: One-shot runner fleet deployment + +on: + push: + branches: + - ops/deploy-runner-fleet-20260818 + pull_request: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: one-shot-runner-fleet-deployment + cancel-in-progress: false + +jobs: + deploy: + if: github.event_name != 'pull_request' || github.head_ref == 'ops/deploy-runner-fleet-20260818' + runs-on: [self-hosted, Linux, X64, co-server] + timeout-minutes: 30 + steps: + - name: Deploy scheduler and provision runners + shell: bash + run: | + set -euo pipefail + umask 077 + + echo "== host preflight ==" + id + hostname + sudo -n true + sudo -n -u nomad -H sudo -n true + sudo -n -u nomad -H gh auth status >/dev/null + + work="$(sudo -n -u nomad -H mktemp -d /tmp/runner-fleet-deploy.XXXXXX)" + cleanup() { + sudo rm -rf -- "$work" + } + trap cleanup EXIT + + scheduler_repo="$work/plugin-scheduler" + light_repo="$work/plugin-light" + fleet_repo="$work/ghrunner" + tarball="$work/actions-runner-linux-x64-2.336.0.tar.gz" + + echo "== fetch approved deployment sources ==" + sudo -n -u nomad -H gh repo clone \ + PRISMA-USER-INTERFACE-FRAMEWORK/PluginEditTool \ + "$scheduler_repo" -- --branch fix/scheduler-managed-worker-filter --single-branch --quiet + sudo -n -u nomad -H gh repo clone \ + PRISMA-USER-INTERFACE-FRAMEWORK/PluginEditTool \ + "$light_repo" -- --branch ci/add-prisma-light-runner-bootstrap --single-branch --quiet + sudo -n -u nomad -H gh repo clone \ + G-A-R-D-E-N/GHRUNNER \ + "$fleet_repo" -- --branch feat/hosted-minute-repo-runners --single-branch --quiet + + sudo -n -u nomad -H bash -c 'cd "$1" && ./scripts/validate-repo.sh' bash "$fleet_repo" + python3 -m py_compile "$scheduler_repo/tools/ci/ghrunnerd.py" + bash -n "$light_repo/tools/ci/install-prisma-runner.sh" + + echo "== deploy PluginEditTool #194 scheduler fix ==" + sudo install -m 0755 \ + "$scheduler_repo/tools/ci/ghrunnerd.py" \ + /usr/local/lib/gh-runner/ghrunnerd.py + sudo systemctl restart gh-runner-webhook.service + sudo systemctl is-active --quiet gh-runner-webhook.service + sudo /usr/local/lib/gh-runner/ghrunnerd.py status + + echo "== fetch and verify pinned Actions runner ==" + sudo -n -u nomad -H curl -fL --retry 3 --retry-delay 2 \ + -o "$tarball" \ + https://github.com/actions/runner/releases/download/v2.336.0/actions-runner-linux-x64-2.336.0.tar.gz + actual="$(sha256sum "$tarball" | awk '{print $1}')" + expected="04cf0be1aff4c3ec3554466c39124ca250e3effd8873bb7e8d68535aa9505d5d" + test "$actual" = "$expected" || { + echo "runner archive digest mismatch" >&2 + echo "expected: $expected" >&2 + echo "actual: $actual" >&2 + exit 1 + } + + org_runner_exists() { + sudo -n -u nomad -H gh api \ + 'orgs/PRISMA-USER-INTERFACE-FRAMEWORK/actions/runners?per_page=100' \ + --jq '.runners[] | select(.name == "prisma-light-1") | .name' \ + | grep -qx 'prisma-light-1' + } + + repo_runner_exists() { + local repo="$1" name="$2" + sudo -n -u nomad -H gh api "repos/$repo/actions/runners?per_page=100" \ + --jq ".runners[] | select(.name == \"$name\") | .name" \ + | grep -qx "$name" + } + + echo "== provision prisma-light-1 ==" + light_dir=/opt/actions-runner-prisma-light-1 + light_unit=actions.runner.PRISMA-USER-INTERFACE-FRAMEWORK.prisma-light-1.service + if org_runner_exists; then + test -d "$light_dir" || { + echo "prisma-light-1 exists on GitHub but $light_dir is missing" >&2 + exit 1 + } + sudo systemctl cat "$light_unit" >/dev/null + sudo systemctl enable --now "$light_unit" + else + test ! -e "$light_dir" || { + echo "$light_dir exists but prisma-light-1 is not registered on GitHub" >&2 + exit 1 + } + token="$(sudo -n -u nomad -H gh api -X POST \ + orgs/PRISMA-USER-INTERFACE-FRAMEWORK/actions/runners/registration-token \ + --jq .token)" + test -n "$token" + printf '%s\n' "$token" \ + | sudo -n -u nomad -H bash \ + "$light_repo/tools/ci/install-prisma-runner.sh" \ + prisma-light-1 "$tarball" + unset token + fi + + verify_light_labels() { + sudo -n -u nomad -H gh api \ + 'orgs/PRISMA-USER-INTERFACE-FRAMEWORK/actions/runners?per_page=100' \ + --jq '.runners[] | select(.name == "prisma-light-1") | [.labels[].name] | sort | join(",")' + } + labels="$(verify_light_labels)" + test "$labels" = "prisma-light" || { + echo "prisma-light-1 has unexpected labels: $labels" >&2 + exit 1 + } + sudo systemctl is-active --quiet "$light_unit" + sudo systemctl show "$light_unit" -p User -p Slice -p MemoryHigh -p MemoryMax -p MemorySwapMax + + install_repo_runner() { + local key="$1" repo="$2" name="$3" dir="$4" unit="$5" + echo "== provision $key ($repo) ==" + if repo_runner_exists "$repo" "$name"; then + test -d "$dir" || { + echo "$name exists on GitHub but $dir is missing" >&2 + exit 1 + } + sudo systemctl cat "$unit" >/dev/null + sudo -n -u nomad -H bash -c \ + 'cd "$1" && ./scripts/apply-host-baseline.sh --runner "$2" --apply' \ + bash "$fleet_repo" "$key" + else + test ! -e "$dir" || { + echo "$dir exists but $name is not registered on GitHub" >&2 + exit 1 + } + token="$(sudo -n -u nomad -H gh api -X POST \ + "repos/$repo/actions/runners/registration-token" --jq .token)" + test -n "$token" + printf '%s\n' "$token" \ + | sudo -n -u nomad -H bash -c \ + 'cd "$1" && exec ./scripts/install-runner.sh "$2" "$3"' \ + bash "$fleet_repo" "$key" "$tarball" + unset token + fi + sudo "$fleet_repo/scripts/audit-host.sh" --runner "$key" + } + + install_repo_runner \ + aimatrix G-A-R-D-E-N/AIMatrix aimatrix-ci \ + /opt/actions-runner-aimatrix \ + actions.runner.G-A-R-D-E-N-AIMatrix.aimatrix-ci.service + + install_repo_runner \ + uirepo G-A-R-D-E-N/UIRepo uirepo-ci \ + /opt/actions-runner-uirepo \ + actions.runner.G-A-R-D-E-N-UIRepo.uirepo-ci.service + + install_repo_runner \ + f4lodgen G-A-R-D-E-N/F4LODGEN f4lodgen-ci \ + /opt/actions-runner-f4lodgen \ + actions.runner.G-A-R-D-E-N-F4LODGEN.f4lodgen-ci.service + + install_repo_runner \ + fcc G-A-R-D-E-N/FCC fcc-ci \ + /opt/actions-runner-fcc \ + actions.runner.G-A-R-D-E-N-FCC.fcc-ci.service + + install_repo_runner \ + aimatrix-local G-A-R-D-E-N/aimatrix-local aimatrix-local-ci \ + /opt/actions-runner-aimatrix-local \ + actions.runner.G-A-R-D-E-N-aimatrix-local.aimatrix-local-ci.service + + install_repo_runner \ + commonwealth-ui-sync G-A-R-D-E-N/CommonwealthOnlineUI commonwealth-ui-sync \ + /opt/actions-runner-commonwealth-ui-sync \ + actions.runner.G-A-R-D-E-N-CommonwealthOnlineUI.commonwealth-ui-sync.service + + echo "== final fleet audit ==" + sudo "$fleet_repo/scripts/audit-host.sh" + sudo /usr/local/lib/gh-runner/ghrunnerd.py status + + echo "deployment complete"