27 changed files with 255498 additions and 165 deletions

No files matched your search

+22
View File
@@ -0,0 +1,22 @@
The MIT License (MIT)
Copyright (c) 2018 GitHub, Inc. and contributors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
@@ -0,0 +1,8 @@
{
"name": "ci-action-gitea-download-artifact",
"source": "https://github.com/christopherhx/gitea-download-artifact.git",
"requested_ref": "v4",
"commit": "75635f32b4c1c41c4b3d64e8f85210112ed4c9c7",
"path": "/private/tmp/gitea-migration-20261105/mirrors/ci-action-gitea-download-artifact.git",
"destination": "https://git.ermnvldmr.com/deytenit/ci-action-gitea-download-artifact"
}
@@ -0,0 +1,40 @@
name: 'Download a Build Artifact'
description: 'Download a build artifact that was previously uploaded in the workflow by the upload-artifact action'
author: 'GitHub'
inputs:
name:
description: 'Name of the artifact to download. If unspecified, all artifacts for the run are downloaded.'
required: false
path:
description: 'Destination path. Supports basic tilde expansion. Defaults to $GITHUB_WORKSPACE'
required: false
pattern:
description: 'A glob pattern matching the artifacts that should be downloaded. Ignored if name is specified.'
required: false
merge-multiple:
description: 'When multiple artifacts are matched, this changes the behavior of the destination directories.
If true, the downloaded artifacts will be in the same directory specified by path.
If false, the downloaded artifacts will be extracted into individual named directories within the specified path.'
required: false
default: 'false'
github-token:
description: 'The GitHub token used to authenticate with the GitHub API.
This is required when downloading artifacts from a different repository or from a different workflow run.
If this is not specified, the action will attempt to download artifacts from the current repository and the current workflow run.'
required: false
repository:
description: 'The repository owner and the repository name joined together by "/".
If github-token is specified, this is the repository that artifacts will be downloaded from.'
required: false
default: ${{ github.repository }}
run-id:
description: 'The id of the workflow run where the desired download artifact was uploaded from.
If github-token is specified, this is the run that artifacts will be downloaded from.'
required: false
default: ${{ github.run_id }}
outputs:
download-path:
description: 'Path of artifact download'
runs:
using: 'node20'
main: 'dist/index.js'
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,47 @@
{
"name": "download-artifact",
"version": "4.0.1",
"description": "Download an Actions Artifact from a workflow run",
"main": "dist/index.js",
"scripts": {
"build": "tsc",
"release": "ncc build src/download-artifact.ts && git add -f dist/",
"check-all": "concurrently \"npm:format-check\" \"npm:lint\" \"npm:build\"",
"format": "prettier --write **/*.ts",
"format-check": "prettier --check **/*.ts",
"lint": "eslint **/*.ts"
},
"repository": {
"type": "git",
"url": "git+https://github.com/actions/download-artifact.git"
},
"keywords": [
"Actions",
"GitHub",
"Artifacts",
"Download"
],
"author": "GitHub",
"license": "MIT",
"bugs": {
"url": "https://github.com/actions/download-artifact/issues"
},
"homepage": "https://github.com/actions/download-artifact#readme",
"dependencies": {
"@actions/artifact": "^2.0.1",
"@actions/core": "^1.10.0",
"@actions/github": "^5.1.1",
"minimatch": "^9.0.3"
},
"devDependencies": {
"@types/node": "^12.12.6",
"@typescript-eslint/eslint-plugin": "^6.14.0",
"@vercel/ncc": "^0.33.4",
"concurrently": "^5.2.0",
"eslint": "^8.55.0",
"eslint-plugin-github": "^4.10.1",
"eslint-plugin-prettier": "^5.0.1",
"prettier": "^3.1.1",
"typescript": "^5.3.3"
}
}
+22
View File
@@ -0,0 +1,22 @@
The MIT License (MIT)
Copyright (c) 2018 GitHub, Inc. and contributors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
@@ -0,0 +1,8 @@
{
"name": "ci-action-gitea-upload-artifact",
"source": "https://github.com/christopherhx/gitea-upload-artifact.git",
"requested_ref": "v4",
"commit": "81f940d004763f986ba3582c007fd842dd5cb0d7",
"path": "/private/tmp/gitea-migration-20261105/mirrors/ci-action-gitea-upload-artifact.git",
"destination": "https://git.ermnvldmr.com/deytenit/ci-action-gitea-upload-artifact"
}
+61
View File
@@ -0,0 +1,61 @@
name: 'Upload a Build Artifact'
description: 'Upload a build artifact that can be used by subsequent workflow steps'
author: 'GitHub'
inputs:
name:
description: 'Artifact name'
default: 'artifact'
path:
description: 'A file, directory or wildcard pattern that describes what to upload'
required: true
if-no-files-found:
description: >
The desired behavior if no files are found using the provided path.
Available Options:
warn: Output a warning but do not fail the action
error: Fail the action with an error message
ignore: Do not output any warnings or errors, the action does not fail
default: 'warn'
retention-days:
description: >
Duration after which artifact will expire in days. 0 means using default retention.
Minimum 1 day.
Maximum 90 days unless changed from the repository settings page.
compression-level:
description: >
The level of compression for Zlib to be applied to the artifact archive.
The value can range from 0 to 9:
- 0: No compression
- 1: Best speed
- 6: Default compression (same as GNU Gzip)
- 9: Best compression
Higher levels will result in better compression, but will take longer to complete.
For large files that are not easily compressed, a value of 0 is recommended for significantly faster uploads.
default: '6'
overwrite:
description: >
If true, an artifact with a matching name will be deleted before a new one is uploaded.
If false, the action will fail if an artifact for the given name already exists.
Does not fail if the artifact does not exist.
default: 'false'
outputs:
artifact-id:
description: >
A unique identifier for the artifact that was just uploaded. Empty if the artifact upload failed.
This ID can be used as input to other APIs to download, delete or get more information about an artifact: https://docs.github.com/en/rest/actions/artifacts
artifact-url:
description: >
A download URL for the artifact that was just uploaded. Empty if the artifact upload failed.
This download URL only works for requests Authenticated with GitHub. Anonymous downloads will be prompted to first login.
If an anonymous download URL is needed than a short time restricted URL can be generated using the download artifact API: https://docs.github.com/en/rest/actions/artifacts#download-an-artifact
This URL will be valid for as long as the artifact exists and the workflow run and repository exists. Once an artifact has expired this URL will no longer work.
Common uses cases for such a download URL can be adding download links to artifacts in descriptions or comments on pull requests or issues.
runs:
using: 'node20'
main: 'dist/index.js'
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,54 @@
{
"name": "upload-artifact",
"version": "4.2.0",
"description": "Upload an Actions Artifact in a workflow run",
"main": "dist/index.js",
"scripts": {
"build": "tsc",
"release": "ncc build src/upload-artifact.ts && git add -f dist/index.js",
"check-all": "concurrently \"npm:format-check\" \"npm:lint\" \"npm:test\" \"npm:build\"",
"format": "prettier --write **/*.ts",
"format-check": "prettier --check **/*.ts",
"lint": "eslint **/*.ts",
"test": "jest --testTimeout 10000"
},
"repository": {
"type": "git",
"url": "git+https://github.com/actions/upload-artifact.git"
},
"keywords": [
"Actions",
"GitHub",
"Artifacts",
"Upload"
],
"author": "GitHub",
"license": "MIT",
"bugs": {
"url": "https://github.com/actions/upload-artifact/issues"
},
"homepage": "https://github.com/actions/upload-artifact#readme",
"dependencies": {
"@actions/artifact": "^2.1.0",
"@actions/core": "^1.10.0",
"@actions/github": "^6.0.0",
"@actions/glob": "^0.3.0",
"@actions/io": "^1.1.2"
},
"devDependencies": {
"@types/jest": "^29.2.5",
"@types/node": "^18.11.18",
"@typescript-eslint/parser": "^5.48.0",
"@vercel/ncc": "^0.36.0",
"concurrently": "^7.6.0",
"eslint": "^8.31.0",
"eslint-plugin-github": "^4.6.0",
"eslint-plugin-jest": "^27.2.0",
"glob": "^8.0.3",
"jest": "^29.3.1",
"jest-circus": "^29.3.1",
"prettier": "^2.8.1",
"ts-jest": "^29.0.3",
"typescript": "^4.9.4"
}
}
@@ -0,0 +1,179 @@
name: ci/packer - main/manual-build-golden-image
'on':
workflow_dispatch:
inputs:
upload_s3:
description: Upload built image to specified static S3 Object Storage
type: boolean
default: true
concurrency:
group: ci-golden-image
cancel-in-progress: false
permissions:
contents: read
jobs:
validate-scripts:
name: 'Validate: Provisioner Scripts'
runs-on: apex-debian13
steps:
- name: Checkout repository
run: "set -euo pipefail\ncase \"$CHECKOUT_REF\" in -*) exit 1;; esac\nGITHUB_SERVER_URL=${GITHUB_SERVER_URL%/}\n\
case \"$GITHUB_SERVER_URL\" in https://git.ermnvldmr.com) ;; *) exit 1;; esac\n\
AUTH=$(printf 'deytenit:%s' \"$CHECKOUT_TOKEN\" | base64 -w0)\nHEADER=\"http.${GITHUB_SERVER_URL}/.extraheader=AUTHORIZATION:\
\ basic $AUTH\"\ngit init \"$GITHUB_WORKSPACE\"\ncd \"$GITHUB_WORKSPACE\"\n\
git remote add origin \"$GITHUB_SERVER_URL/$GITHUB_REPOSITORY.git\"\ngit -c\
\ \"$HEADER\" fetch --tags origin '+refs/heads/*:refs/remotes/origin/*'\n\
git -c \"$HEADER\" fetch origin \"$CHECKOUT_REF\"\ngit checkout --detach FETCH_HEAD\n\
test \"$(git rev-parse HEAD)\" = \"$GITHUB_SHA\"\nif [ \"$CHECKOUT_SUBMODULES\"\
\ = recursive ]; then\n key_file=$(mktemp \"${RUNNER_TEMP:-/tmp}/hextra-key.XXXXXX\"\
)\n trap 'rm -f \"$key_file\"' EXIT\n umask 077\n printf '%s\\n'\
\ \"$HEXTRA_READ_KEY\" > \"$key_file\"\n chmod 600 \"$key_file\"\n export\
\ GIT_SSH_COMMAND=\"ssh -T -i $key_file -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes\
\ -o HostKeyAlgorithms=ssh-ed25519 -o UserKnownHostsFile=$GITHUB_WORKSPACE/.gitea/known_hosts\"\
\n git submodule update --init --recursive\nfi\nif [ \"$PERSIST_CREDENTIALS\"\
\ = true ]; then\n git config \"http.${GITHUB_SERVER_URL}/.extraheader\"\
\ \"AUTHORIZATION: basic $AUTH\"\nfi\n"
env:
CHECKOUT_REF: ${{ github.sha }}
CHECKOUT_SUBMODULES: 'false'
CHECKOUT_TOKEN: ${{ github.token }}
PERSIST_CREDENTIALS: 'false'
HEXTRA_READ_KEY: ''
- name: Setup Python
run: python3 -c "import sys; assert sys.version_info >= (3, 10)"
- name: Install Developer Dependencies
run: python3 -m pip install ruff==0.17.2 "PyYAML>=6.0,<7"
- name: Validate Shell Script Syntax
run: "for script in ci/packer/scripts/*.sh ci/scripts/init-node/provisioning/seed.sh;\
\ do\n bash -n \"$script\"\ndone\nmake check\n"
- name: Setup Packer
run: 'packer version
packer plugins installed | grep qemu'
- name: Validate Packer Template
run: 'packer plugins installed | grep qemu
packer validate ci/packer/apex-debian-13.pkr.hcl
'
build-image:
name: 'Build: Golden Image'
needs: validate-scripts
runs-on: apex-debian13
outputs:
image_name: ${{ steps.vars.outputs.image_name }}
steps:
- name: Checkout repository
run: "set -euo pipefail\ncase \"$CHECKOUT_REF\" in -*) exit 1;; esac\nGITHUB_SERVER_URL=${GITHUB_SERVER_URL%/}\n\
case \"$GITHUB_SERVER_URL\" in https://git.ermnvldmr.com) ;; *) exit 1;; esac\n\
AUTH=$(printf 'deytenit:%s' \"$CHECKOUT_TOKEN\" | base64 -w0)\nHEADER=\"http.${GITHUB_SERVER_URL}/.extraheader=AUTHORIZATION:\
\ basic $AUTH\"\ngit init \"$GITHUB_WORKSPACE\"\ncd \"$GITHUB_WORKSPACE\"\n\
git remote add origin \"$GITHUB_SERVER_URL/$GITHUB_REPOSITORY.git\"\ngit -c\
\ \"$HEADER\" fetch --tags origin '+refs/heads/*:refs/remotes/origin/*'\n\
git -c \"$HEADER\" fetch origin \"$CHECKOUT_REF\"\ngit checkout --detach FETCH_HEAD\n\
test \"$(git rev-parse HEAD)\" = \"$GITHUB_SHA\"\nif [ \"$CHECKOUT_SUBMODULES\"\
\ = recursive ]; then\n key_file=$(mktemp \"${RUNNER_TEMP:-/tmp}/hextra-key.XXXXXX\"\
)\n trap 'rm -f \"$key_file\"' EXIT\n umask 077\n printf '%s\\n'\
\ \"$HEXTRA_READ_KEY\" > \"$key_file\"\n chmod 600 \"$key_file\"\n export\
\ GIT_SSH_COMMAND=\"ssh -T -i $key_file -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes\
\ -o HostKeyAlgorithms=ssh-ed25519 -o UserKnownHostsFile=$GITHUB_WORKSPACE/.gitea/known_hosts\"\
\n git submodule update --init --recursive\nfi\nif [ \"$PERSIST_CREDENTIALS\"\
\ = true ]; then\n git config \"http.${GITHUB_SERVER_URL}/.extraheader\"\
\ \"AUTHORIZATION: basic $AUTH\"\nfi\n"
env:
CHECKOUT_REF: ${{ github.sha }}
CHECKOUT_SUBMODULES: 'false'
CHECKOUT_TOKEN: ${{ github.token }}
PERSIST_CREDENTIALS: 'false'
HEXTRA_READ_KEY: ''
- name: Compute Image Metadata
id: vars
run: 'SHORT_SHA="$(echo ${{ github.sha }} | cut -c1-7)"
IMAGE_NAME="com-ermnvldmr-apex-debian-13-amd64-${SHORT_SHA}.qcow2"
echo "image_name=${IMAGE_NAME}" >> "$GITHUB_OUTPUT"
'
- name: Install QEMU & Build Tools
run: 'sudo apt-get update
sudo apt-get install -y qemu-system-x86 qemu-utils xorriso
sudo chmod 666 /dev/kvm 2>/dev/null || true
'
- name: Setup Packer
run: 'packer version
packer plugins installed | grep qemu'
- name: Build Debian 13 Image with Packer
run: 'packer plugins installed | grep qemu
packer build -var "vm_name=${{ steps.vars.outputs.image_name }}" -var "provision_revision=${{
github.sha }}" ci/packer/apex-debian-13.pkr.hcl
'
- name: Upload Image Artifact
uses: ./.gitea/actions/upload-artifact
with:
name: golden-image
path: output-qemu/${{ steps.vars.outputs.image_name }}
retention-days: 30
upload-s3:
name: 'Upload: S3 Storage'
needs: build-image
if: inputs.upload_s3 == true
runs-on: apex-debian13
steps:
- run: "set -euo pipefail\ncase \"$CHECKOUT_REF\" in -*) exit 1;; esac\nGITHUB_SERVER_URL=${GITHUB_SERVER_URL%/}\n\
case \"$GITHUB_SERVER_URL\" in https://git.ermnvldmr.com) ;; *) exit 1;; esac\n\
AUTH=$(printf 'deytenit:%s' \"$CHECKOUT_TOKEN\" | base64 -w0)\nHEADER=\"http.${GITHUB_SERVER_URL}/.extraheader=AUTHORIZATION:\
\ basic $AUTH\"\ngit init \"$GITHUB_WORKSPACE\"\ncd \"$GITHUB_WORKSPACE\"\n\
git remote add origin \"$GITHUB_SERVER_URL/$GITHUB_REPOSITORY.git\"\ngit -c\
\ \"$HEADER\" fetch --tags origin '+refs/heads/*:refs/remotes/origin/*'\n\
git -c \"$HEADER\" fetch origin \"$CHECKOUT_REF\"\ngit checkout --detach FETCH_HEAD\n\
test \"$(git rev-parse HEAD)\" = \"$GITHUB_SHA\"\nif [ \"$CHECKOUT_SUBMODULES\"\
\ = recursive ]; then\n key_file=$(mktemp \"${RUNNER_TEMP:-/tmp}/hextra-key.XXXXXX\"\
)\n trap 'rm -f \"$key_file\"' EXIT\n umask 077\n printf '%s\\n'\
\ \"$HEXTRA_READ_KEY\" > \"$key_file\"\n chmod 600 \"$key_file\"\n export\
\ GIT_SSH_COMMAND=\"ssh -T -i $key_file -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes\
\ -o HostKeyAlgorithms=ssh-ed25519 -o UserKnownHostsFile=$GITHUB_WORKSPACE/.gitea/known_hosts\"\
\n git submodule update --init --recursive\nfi\nif [ \"$PERSIST_CREDENTIALS\"\
\ = true ]; then\n git config \"http.${GITHUB_SERVER_URL}/.extraheader\"\
\ \"AUTHORIZATION: basic $AUTH\"\nfi\n"
env:
CHECKOUT_REF: ${{ github.sha }}
CHECKOUT_SUBMODULES: 'false'
CHECKOUT_TOKEN: ${{ github.token }}
PERSIST_CREDENTIALS: 'false'
HEXTRA_READ_KEY: ''
- name: Download Image Artifact
uses: ./.gitea/actions/download-artifact
with:
name: golden-image
path: output-qemu
- name: Upload to Static S3 (/dev/images/)
env:
AWS_ACCESS_KEY_ID: ${{ secrets.STATIC_AWS_ACCESS_KEY_ID || secrets.COM_ERMNVLDMR_STATIC_AWS_ACCESS_KEY_ID
}}
AWS_SECRET_ACCESS_KEY: ${{ secrets.STATIC_AWS_SECRET_ACCESS_KEY || secrets.COM_ERMNVLDMR_STATIC_AWS_SECRET_ACCESS_KEY
}}
AWS_DEFAULT_REGION: ${{ secrets.STATIC_AWS_REGION || secrets.COM_ERMNVLDMR_STATIC_S3_REGION
|| 'auto' }}
S3_ENDPOINT: ${{ secrets.STATIC_S3_ENDPOINT_URL || secrets.COM_ERMNVLDMR_STATIC_S3_ENDPOINT_URL
}}
S3_BUCKET: ${{ secrets.STATIC_S3_BUCKET || secrets.COM_ERMNVLDMR_STATIC_S3_BUCKET
}}
IMAGE_NAME: ${{ needs.build-image.outputs.image_name }}
run: "if [ -n \"${S3_BUCKET}\" ] && [ -n \"${AWS_ACCESS_KEY_ID}\" ]; then\n\
\ ENDPOINT_FLAG=\"\"\n if [ -n \"${S3_ENDPOINT}\" ]; then\n ENDPOINT_FLAG=\"\
--endpoint-url=${S3_ENDPOINT}\"\n fi\n echo \"Cleaning up older builds in\
\ dev/images/com-ermnvldmr-apex-debian-13-amd64-*.qcow2 to maintain quota...\"\
\n aws ${ENDPOINT_FLAG} s3 rm \"s3://${S3_BUCKET}/dev/images/\" --recursive\
\ --exclude \"*\" --include \"com-ermnvldmr-apex-debian-13-amd64-*.qcow2\"\
\ || true\n echo \"Uploading ${IMAGE_NAME}...\"\n aws ${ENDPOINT_FLAG} s3\
\ cp \"output-qemu/${IMAGE_NAME}\" \"s3://${S3_BUCKET}/dev/images/${IMAGE_NAME}\"\
\n echo \"Golden image uploaded successfully to s3://${S3_BUCKET}/dev/images/${IMAGE_NAME}\"\
\nelse\n echo \"S3 credentials not configured; skipping upload. Image artifact\
\ preserved in GitHub Actions run.\"\nfi\n"
@@ -1,124 +0,0 @@
name: ci/packer - main/manual-build-golden-image
on:
workflow_dispatch:
inputs:
upload_s3:
description: "Upload built image to specified static S3 Object Storage"
type: boolean
default: true
concurrency:
group: ci-golden-image
cancel-in-progress: false
permissions:
contents: read
jobs:
validate-scripts:
name: "Validate: Provisioner Scripts"
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.10"
- name: Install Developer Dependencies
run: python3 -m pip install ruff==0.17.2 "PyYAML>=6.0,<7"
- name: Validate Shell Script Syntax
run: |
for script in ci/packer/scripts/*.sh ci/scripts/init-node/provisioning/seed.sh; do
bash -n "$script"
done
make check
- name: Setup Packer
uses: hashicorp/setup-packer@main
with:
version: "latest"
- name: Validate Packer Template
run: |
packer init ci/packer/apex-debian-13.pkr.hcl
packer validate ci/packer/apex-debian-13.pkr.hcl
build-image:
name: "Build: Golden Image"
needs: validate-scripts
runs-on: ubuntu-24.04
outputs:
image_name: ${{ steps.vars.outputs.image_name }}
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Compute Image Metadata
id: vars
run: |
SHORT_SHA="$(echo ${{ github.sha }} | cut -c1-7)"
IMAGE_NAME="com-ermnvldmr-apex-debian-13-amd64-${SHORT_SHA}.qcow2"
echo "image_name=${IMAGE_NAME}" >> "$GITHUB_OUTPUT"
- name: Install QEMU & Build Tools
run: |
sudo apt-get update
sudo apt-get install -y qemu-system-x86 qemu-utils xorriso
sudo chmod 666 /dev/kvm 2>/dev/null || true
- name: Setup Packer
uses: hashicorp/setup-packer@main
with:
version: "latest"
- name: Build Debian 13 Image with Packer
run: |
packer init ci/packer/apex-debian-13.pkr.hcl
packer build -var "vm_name=${{ steps.vars.outputs.image_name }}" -var "provision_revision=${{ github.sha }}" ci/packer/apex-debian-13.pkr.hcl
- name: Upload Image Artifact
uses: actions/upload-artifact@v4
with:
name: golden-image
path: output-qemu/${{ steps.vars.outputs.image_name }}
retention-days: 2
upload-s3:
name: "Upload: S3 Storage"
needs: build-image
if: inputs.upload_s3 == true
runs-on: ubuntu-latest
steps:
- name: Download Image Artifact
uses: actions/download-artifact@v4
with:
name: golden-image
path: output-qemu
- name: Upload to Static S3 (/dev/images/)
env:
AWS_ACCESS_KEY_ID: ${{ secrets.STATIC_AWS_ACCESS_KEY_ID || secrets.COM_ERMNVLDMR_STATIC_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.STATIC_AWS_SECRET_ACCESS_KEY || secrets.COM_ERMNVLDMR_STATIC_AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.STATIC_AWS_REGION || secrets.COM_ERMNVLDMR_STATIC_S3_REGION || 'auto' }}
S3_ENDPOINT: ${{ secrets.STATIC_S3_ENDPOINT_URL || secrets.COM_ERMNVLDMR_STATIC_S3_ENDPOINT_URL }}
S3_BUCKET: ${{ secrets.STATIC_S3_BUCKET || secrets.COM_ERMNVLDMR_STATIC_S3_BUCKET }}
IMAGE_NAME: ${{ needs.build-image.outputs.image_name }}
run: |
if [ -n "${S3_BUCKET}" ] && [ -n "${AWS_ACCESS_KEY_ID}" ]; then
ENDPOINT_FLAG=""
if [ -n "${S3_ENDPOINT}" ]; then
ENDPOINT_FLAG="--endpoint-url=${S3_ENDPOINT}"
fi
echo "Cleaning up older builds in dev/images/com-ermnvldmr-apex-debian-13-amd64-*.qcow2 to maintain quota..."
aws ${ENDPOINT_FLAG} s3 rm "s3://${S3_BUCKET}/dev/images/" --recursive --exclude "*" --include "com-ermnvldmr-apex-debian-13-amd64-*.qcow2" || true
echo "Uploading ${IMAGE_NAME}..."
aws ${ENDPOINT_FLAG} s3 cp "output-qemu/${IMAGE_NAME}" "s3://${S3_BUCKET}/dev/images/${IMAGE_NAME}"
echo "Golden image uploaded successfully to s3://${S3_BUCKET}/dev/images/${IMAGE_NAME}"
else
echo "S3 credentials not configured; skipping upload. Image artifact preserved in GitHub Actions run."
fi
+2
View File
@@ -16,3 +16,5 @@ __pycache__/
*.env
.env
*.bak
._*
.DS_Store
+3 -3
View File
@@ -10,12 +10,12 @@ Local infrastructure operations for independently owned nodes, with a Python sta
- **Live Documentation**: <https://docs.ermnvldmr.com/en/apex/> (also available in [Russian](https://docs.ermnvldmr.com/ru/apex/))
- **Getting Started Guide**: <https://docs.ermnvldmr.com/en/apex/getting-started/>
- **Report an Issue**: <https://github.com/deytenit/apex.ermnvldmr.com/issues>
- **Report an Issue**: <https://git.ermnvldmr.com/deytenit/apex.ermnvldmr.com/issues>
- **License**: [BSD-2-Clause](#copyright--license)
```sh
# Clone your node's repository (which pins apex as commons/)
git clone --recurse-submodules git@github.com:you/selfhost.example.com.git
git clone --recurse-submodules ssh://git@git.example.com:22222/infrastructure/selfhost.example.com.git
cd selfhost.example.com && ./init.sh && source ~/.bashrc
# Configure node identity
@@ -138,7 +138,7 @@ Adding shared actions or improving the engine.
## Communication & Support
- **Issues**: <https://github.com/deytenit/apex.ermnvldmr.com/issues>
- **Issues**: <https://git.ermnvldmr.com/deytenit/apex.ermnvldmr.com/issues>
- **Email**: <personal@ermnvldmr.com>
## Copyright & License
+48 -3
View File
@@ -1,5 +1,5 @@
# actions/backup/run.py
"""Label-driven restic backup via resticontainer (github.com/deytenit/resticontainer).
"""Label-driven restic backup via resticontainer (git.ermnvldmr.com/deytenit/resticontainer).
Runs the resticontainer core service one-shot: init-if-needed -> backup (discovers
`restic.*`-labelled services, runs their pre-hooks/stops, snapshots the union of the
resolved host paths in a single restic run) -> forget --prune -> Telegram. Every
@@ -7,12 +7,44 @@ non-`backup` subcommand passes straight through to restic, so init/snapshots/for
behave exactly as before."""
import json
import os
from pathlib import Path
import time
from engine.descriptor import Meta, Arg
METADATA = Meta(summary="Run the resticontainer label-driven backup (init-if-needed, backup, forget --prune).",
args=[Arg("telegram_bot_url", "Telegram Bot URL for notifications")])
TITLE = "Restic Host Backup"
def write_backup_status(path, state, timestamp):
path = Path(path)
previous = json.loads(path.read_text()) if path.exists() else {}
status = {"state": state, "updated_at": timestamp,
"last_success": timestamp if state == "success" else previous.get("last_success", 0)}
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_name(path.name + f".{os.getpid()}.tmp")
temporary.write_text(json.dumps(status) + "\n")
temporary.chmod(0o644)
temporary.replace(path)
def restore_containers(ctx, container_ids):
"""Recover previously running backup targets, retrying namespace dependents."""
pending = list(container_ids)
if not pending:
return
for _ in range(len(pending) + 1):
cp = ctx.sys.run(["docker", "inspect", *pending], capture=True)
stopped = [c["Id"] for c in json.loads(cp.stdout) if not c["State"]["Running"]]
if not stopped:
return
progress = False
for container_id in stopped:
cp = ctx.sys.run(["docker", "start", container_id], check=False, capture=True)
progress = progress or cp.returncode == 0
if not progress:
raise RuntimeError("Backup left containers stopped: " + ", ".join(stopped))
pending = stopped
raise RuntimeError("Backup containers did not stay running: " + ", ".join(pending))
def run(ctx, args):
log, node, url = ctx.log, ctx.node.name, args.telegram_bot_url
core = ctx.paths.core
@@ -42,6 +74,8 @@ def run(ctx, args):
except (ValueError, TypeError):
return None
status_path = Path("/var/lib/apex/backup-status.json")
write_backup_status(status_path, "running", time.time())
try:
log.info("Checking restic repository availability...")
if restic("snapshots", check=False).returncode == 0:
@@ -50,8 +84,16 @@ def run(ctx, args):
ctx.notify.error(TITLE, url, "restic init failed"); raise SystemExit(1)
before = host_snapshot_ids()
log.info("Backing up `restic.*`-labelled services (resticontainer discovers paths)...")
if restic("backup", "--host", host, "--tag", tag, "--compression", "max",
check=False).returncode != 0:
targets = ctx.sys.run(["docker", "ps", "-q", "--no-trunc",
"--filter", "label=restic.enable=true",
"--filter", "label=restic.backup.stop=true"], capture=True).stdout.split()
try:
result = restic("backup", "--host", host, "--tag", tag, "--compression", "max",
check=False)
finally:
# Resticontainer may start a network-namespace dependent before its owner.
restore_containers(ctx, targets)
if result.returncode != 0:
ctx.notify.error(TITLE, url, "restic backup failed"); raise SystemExit(1)
# Guard: the label-driven backup is a silent no-op if no service carries the
# `restic.enable` label (resticontainer finds zero paths and skips restic).
@@ -70,9 +112,12 @@ def run(ctx, args):
log.success("Restic backup completed.")
if not ctx.notify.success(TITLE, url, f"Restic backup + retention succeeded for {node}"):
raise SystemExit(1) # bash exited 1 when the success telegram failed to send
write_backup_status(status_path, "success", time.time())
except SystemExit:
write_backup_status(status_path, "failed", time.time())
raise
except Exception as e:
write_backup_status(status_path, "failed", time.time())
log.error(f"{type(e).__name__}: {e}")
ctx.notify.error(TITLE, url, f"backup failed: {e}")
raise SystemExit(1) from e
+77
View File
@@ -0,0 +1,77 @@
"""Deploy the opt-in native Alloy service using shared host/guest configs."""
import json
from pathlib import Path
import re
import tempfile
from engine.descriptor import Flag, Meta
METADATA = Meta(summary="Configure native Alloy from configs/alloy/config.json.",
args=[Flag("--dry-run", "Validate the deployment plan without changes.")])
def deployment(ctx):
settings = Path(ctx.paths.configs) / "alloy/config.json"
if not settings.exists():
return None
values = json.loads(settings.read_text())
role = values.get("role")
if role not in ("host", "guest"):
raise ValueError("Alloy role must be host or guest")
env_file = values.get("environment_file", "")
if not re.fullmatch(r"/[A-Za-z0-9_./-]+", env_file):
raise ValueError("Alloy environment_file must be an absolute path without whitespace")
source = Path(ctx.paths.commons) / "etc/alloy" / (
"native-host.alloy" if role == "host" else "native.alloy")
config = source.read_text()
extra = Path(ctx.paths.configs) / "alloy/extra.alloy"
if extra.is_file():
config += "\n" + extra.read_text()
unit = ("[Service]\nEnvironmentFile=" + env_file + "\n"
'ExecStart=\nExecStart=/usr/bin/alloy run --storage.path=/var/lib/alloy --server.http.listen-addr=127.0.0.1:12346 /etc/alloy/config.alloy\n'
+ ("User=root\nGroup=root\n" if role == "guest" else "") +
"SupplementaryGroups=systemd-journal" + (" docker" if role == "guest" else "") + "\n")
return config, unit, env_file
def run(ctx, args):
plan = deployment(ctx)
if plan is None:
ctx.log.info("Native Alloy is not configured for this node; skipping.")
return
config, unit, env_file = plan
if args.dry_run:
ctx.log.info("Native Alloy deployment plan validated; no changes made.")
return
ctx.sys.sudo(["test", "-f", env_file])
# Validate with the actual service environment without printing credentials.
validator = '''import os, stat, subprocess, sys
from pathlib import Path
# Environment files deliberately use plain KEY=value, with no shell evaluation.
secret = Path(sys.argv[1])
info = secret.stat()
if info.st_uid != 0 or stat.S_IMODE(info.st_mode) & 0o077:
raise RuntimeError('Alloy environment file must be root-owned with mode 0600 or stricter')
for line in secret.read_text().splitlines():
if not line.strip() or line.lstrip().startswith('#'): continue
key, value = line.split('=', 1)
os.environ[key] = value
subprocess.run(['alloy', 'validate', sys.argv[2]], check=True)
'''
with tempfile.TemporaryDirectory(prefix="apex-alloy-") as directory:
config_path = Path(directory) / "config.alloy"
unit_path = Path(directory) / "apex.conf"
config_path.write_text(config)
unit_path.write_text(unit)
ctx.sys.sudo(["python3", "-c", validator, env_file, str(config_path)])
ctx.sys.sudo(["install", "-d", "-m", "0755", "/etc/alloy",
"/etc/systemd/system/alloy.service.d"])
ctx.sys.sudo(["install", "-m", "0644", str(config_path), "/etc/alloy/config.alloy"])
ctx.sys.sudo(["install", "-m", "0644", str(unit_path),
"/etc/systemd/system/alloy.service.d/apex.conf"])
ctx.sys.sudo(["systemctl", "unmask", "alloy.service"])
ctx.sys.daemon_reload()
ctx.sys.sudo(["systemctl", "enable", "alloy.service"])
ctx.sys.sudo(["systemctl", "restart", "alloy.service"])
ctx.sys.sudo(["systemctl", "is-active", "--quiet", "alloy.service"])
ctx.log.success("Configured native Alloy.")
+27 -32
View File
@@ -1,64 +1,59 @@
# actions/sync/repository.py
"""Port of sync/repository — capture-up commit + rebase onto origin/main + force-push."""
"""Commit node configuration and synchronize main directly with its remote."""
import os
from datetime import datetime
from engine.descriptor import Meta, Arg
from engine.identity import read_env
METADATA = Meta(summary="Capture-up: commit node changes, rebase onto origin/main, force-push sync/<node>.",
args=[Arg("telegram_bot_url", "Telegram Bot URL for notifications")])
TITLE = "Annual Remote Sync"
METADATA = Meta(summary="Commit node changes, rebase onto origin/main, and push main.",
args=[Arg("telegram_bot_url", "Optional notification URL", required=False)])
TITLE = "Repository Sync"
def run(ctx, args):
log, node, url = ctx.log, ctx.node.name, args.telegram_bot_url
log, node = ctx.log, ctx.node.name
url = args.telegram_bot_url
repo, commons = ctx.paths.repo_root, ctx.paths.commons
ne = read_env(os.path.join(repo, "node.env")) # per-node bot identity (generic defaults)
ne = read_env(os.path.join(repo, "node.env"))
author = ne.get("APEX_GIT_AUTHOR_NAME", "apex [bot]")
email = ne.get("APEX_GIT_AUTHOR_EMAIL", "apex@localhost")
env = dict(os.environ,
GIT_AUTHOR_NAME=author, GIT_AUTHOR_EMAIL=email,
env = dict(os.environ, GIT_AUTHOR_NAME=author, GIT_AUTHOR_EMAIL=email,
GIT_COMMITTER_NAME=author, GIT_COMMITTER_EMAIL=email)
def git(*a, check=True):
return ctx.sys.run(["git", "-C", repo, *a], check=check, env=env)
def git(*a, check=True, capture=False):
return ctx.sys.run(["git", "-C", repo, *a], check=check, capture=capture, env=env)
try:
branch = git("symbolic-ref", "--quiet", "--short", "HEAD", capture=True).stdout.strip()
if branch != "main":
raise RuntimeError("Repository sync requires main; switch branches explicitly first.")
cp = ctx.sys.run(["git", "-C", commons, "status", "--porcelain"], capture=True)
if (cp.stdout or "").strip():
log.error(f"Commons submodule at {commons} has uncommitted changes. Aborting.")
raise SystemExit(1)
raise RuntimeError(f"Commons submodule at {commons} has uncommitted changes.")
if not os.path.isdir(ctx.paths.compositions):
# Without this, `git add -A` would stage a deleted node tree and an
# unattended cron run would commit + force-push the deletion.
log.error(f"Directory '{ctx.paths.compositions}' does not exist. Cannot sync.")
raise SystemExit(1)
branch = f"sync/{node}"
ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if ctx.sys.ok(["git", "-C", repo, "rev-parse", "--verify", branch]):
git("checkout", branch)
else:
git("checkout", "-b", branch)
raise RuntimeError(f"Directory '{ctx.paths.compositions}' does not exist.")
expected = git("rev-parse", "HEAD:commons", capture=True).stdout.strip()
actual = ctx.sys.run(["git", "-C", commons, "rev-parse", "HEAD"], capture=True).stdout.strip()
if actual != expected:
raise RuntimeError("Commons checkout differs from the committed pin; update the submodule before retrying.")
git("add", "-A")
git("submodule", "update", "--init", "--recursive")
git("add", "commons")
if git("diff", "--cached", "--quiet", check=False).returncode != 0:
ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
git("commit", "-m", f"[{node}] Auto-commit: {ts}")
else:
log.info("No changes to commit.")
git("fetch", "origin", "main")
if git("rebase", "origin/main", check=False).returncode != 0:
log.error("Rebase conflict — aborting rebase; manual intervention required.")
git("rebase", "--abort", check=False)
ctx.notify.error(TITLE, url, "Rebase conflict. Manual intervention required.")
raise SystemExit(1)
raise RuntimeError("Rebase conflict; local commits retained for manual resolution.")
git("submodule", "update", "--init", "--recursive")
git("push", "--force-with-lease", "origin", branch)
log.success(f"Pushed {branch}.")
if not ctx.notify.success(TITLE, url, f"Sync to '{branch}' completed."):
git("push", "origin", "HEAD:refs/heads/main")
log.success("Synchronized main.")
if url and not ctx.notify.success(TITLE, url, "Sync to main completed."):
raise SystemExit(1)
except SystemExit:
raise
except Exception as e:
log.error(f"{type(e).__name__}: {e}")
ctx.notify.error(TITLE, url, f"Sync failed: {e}")
if url:
ctx.notify.error(TITLE, url, f"Sync failed: {e}")
raise SystemExit(1) from e
+3 -2
View File
@@ -12,8 +12,6 @@ def run(ctx, args):
log, s = ctx.log, ctx.sys
repo = ctx.paths.repo_root
rc_ip = _validate_anchor_ips(ctx)
if not shutil.which("docker"):
log.error("docker is not installed or not in PATH."); return rc_ip or 1
files_string, targets = "", ["."]
if args.hook:
@@ -25,6 +23,9 @@ def run(ctx, args):
log.info("No staged docker-compose files to lint."); return rc_ip
targets = [f"/workdir/{f}" for f in files_string.splitlines()]
if not shutil.which("docker"):
log.error("docker is not installed or not in PATH."); return rc_ip or 1
log.info("Running dclint --fix (via docker)...")
env_args = []
for k in ("HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy", "NO_PROXY", "no_proxy"):
+1 -1
View File
@@ -19,7 +19,7 @@ write_files:
permissions: '0600'
content: |
APEX_REPO_NAME="node1.example.com"
APEX_REPO_URL="https://github.com/you/node1.example.com.git"
APEX_REPO_URL="https://git.example.com/infrastructure/node1.example.com.git"
APEX_TIER1_MODE="directory"
APEX_TIER1_DEVICE=""
APEX_TIER2_MODE="directory"
+1
View File
@@ -79,6 +79,7 @@ build {
scripts = [
"ci/packer/scripts/01-base-system.sh",
"ci/packer/scripts/02-storage-docker.sh",
"ci/packer/scripts/02-alloy-package.sh",
"ci/packer/scripts/03-bootstrap-helper.sh",
"ci/packer/scripts/04-cleanup.sh"
]
+18
View File
@@ -0,0 +1,18 @@
#!/usr/bin/env bash
set -euo pipefail
# Service configuration and credentials are deployed later by apex configure/alloy.
alloy_version=1.21.3
alloy_sha256=dbbd705de8ab37dec0cbd6361a3fc7e754d6b2854b57ce672a83e6366199ba73
[[ "$(dpkg --print-architecture)" == amd64 ]]
alloy_deb=$(mktemp --suffix=.deb)
trap 'rm -f "$alloy_deb"' EXIT
curl --fail --location --retry 3 --output "$alloy_deb" \
"https://github.com/grafana/alloy/releases/download/v${alloy_version}/alloy-${alloy_version}-1.amd64.deb"
printf '%s %s\n' "$alloy_sha256" "$alloy_deb" | sha256sum --check --status
systemctl mask alloy.service
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "$alloy_deb"
systemctl disable --now alloy.service
systemctl unmask alloy.service
! systemctl is-active --quiet alloy.service
! systemctl is-enabled --quiet alloy.service
+102
View File
@@ -0,0 +1,102 @@
// Shared native Grafana Alloy configuration for the physical host.
// Metrics and logs use the endpoints and credentials supplied through
// APEX_PROM_PUSH_URL, APEX_LOKI_PUSH_URL, and APEX_OBS_BASIC_AUTH_USER/PASS.
// ----------------------------------------------------------------------------
// METRICS
// ----------------------------------------------------------------------------
// Native filesystem views; physical capacity is collected on the Proxmox host.
prometheus.exporter.unix "node" {
enable_collectors = ["textfile"]
textfile {
directory = "/var/lib/alloy/textfile"
}
procfs_path = "/proc"
sysfs_path = "/sys"
rootfs_path = "/"
}
// Container metrics (cadvisor-equivalent) via the docker socket.
// Stamp a static `node` label onto each exporter's targets.
discovery.relabel "node_metrics" {
targets = prometheus.exporter.unix.node.targets
rule {
action = "replace"
target_label = "node"
replacement = sys.env("APEX_NODE_HOST")
}
// Stamp a stable `instance` (the node's public host) instead of the default,
// which is the Alloy container id/address.
rule {
action = "replace"
target_label = "instance"
replacement = sys.env("APEX_NODE_HOST")
}
}
prometheus.scrape "node" {
targets = discovery.relabel.node_metrics.output
forward_to = [prometheus.remote_write.default.receiver]
scrape_interval = "30s"
}
// The configured Prometheus-compatible endpoint must accept remote writes.
prometheus.remote_write "default" {
endpoint {
url = sys.env("APEX_PROM_PUSH_URL")
basic_auth {
username = sys.env("APEX_OBS_BASIC_AUTH_USER")
password = sys.env("APEX_OBS_BASIC_AUTH_PASS")
}
queue_config {
capacity = 5000
max_shards = 10
min_shards = 1
max_samples_per_send = 1000
batch_send_deadline = "5s"
min_backoff = "100ms"
max_backoff = "10s"
}
}
wal {
truncate_frequency = "30m"
min_keepalive_time = "5m"
max_keepalive_time = "30m"
}
}
// ----------------------------------------------------------------------------
// LOGS
// ----------------------------------------------------------------------------
// systemd journal (host services, ufw, sshd, crowdsec, etc.).
loki.source.journal "systemd" {
path = "/var/log/journal"
max_age = "12h"
labels = {
node = sys.env("APEX_NODE_HOST"),
job = "systemd-journal",
}
forward_to = [loki.write.default.receiver]
}
// All docker containers' stdout/stderr.
// Push logs to the central Loki.
loki.write "default" {
endpoint {
url = sys.env("APEX_LOKI_PUSH_URL")
basic_auth {
username = sys.env("APEX_OBS_BASIC_AUTH_USER")
password = sys.env("APEX_OBS_BASIC_AUTH_PASS")
}
}
}
+128
View File
@@ -0,0 +1,128 @@
// Shared native Grafana Alloy configuration for Docker guests.
// Metrics and logs use the endpoints and credentials supplied through
// APEX_PROM_PUSH_URL, APEX_LOKI_PUSH_URL, and APEX_OBS_BASIC_AUTH_USER/PASS.
// ----------------------------------------------------------------------------
// METRICS
// ----------------------------------------------------------------------------
// Native filesystem views; physical capacity is collected on the Proxmox host.
// Container metrics (cadvisor-equivalent) via the docker socket.
prometheus.exporter.cadvisor "containers" {
docker_host = "unix:///var/run/docker.sock"
store_container_labels = false
allowlisted_container_labels = [
"com.docker.compose.project",
"com.docker.compose.service",
]
docker_only = true
enabled_metrics = [
"cpu",
"cpuLoad",
"diskIO",
"memory",
"network",
"oom_event",
]
}
// Stamp a static `node` label onto each exporter's targets.
discovery.relabel "container_metrics" {
targets = prometheus.exporter.cadvisor.containers.targets
rule {
action = "replace"
target_label = "node"
replacement = sys.env("APEX_NODE_HOST")
}
rule {
action = "replace"
target_label = "instance"
replacement = sys.env("APEX_NODE_HOST")
}
}
prometheus.scrape "containers" {
targets = discovery.relabel.container_metrics.output
forward_to = [prometheus.remote_write.default.receiver]
scrape_interval = "30s"
}
// The configured Prometheus-compatible endpoint must accept remote writes.
prometheus.remote_write "default" {
endpoint {
url = sys.env("APEX_PROM_PUSH_URL")
basic_auth {
username = sys.env("APEX_OBS_BASIC_AUTH_USER")
password = sys.env("APEX_OBS_BASIC_AUTH_PASS")
}
queue_config {
capacity = 5000
max_shards = 10
min_shards = 1
max_samples_per_send = 1000
batch_send_deadline = "5s"
min_backoff = "100ms"
max_backoff = "10s"
}
}
wal {
truncate_frequency = "30m"
min_keepalive_time = "5m"
max_keepalive_time = "30m"
}
}
// ----------------------------------------------------------------------------
// LOGS
// ----------------------------------------------------------------------------
// systemd journal (host services, ufw, sshd, crowdsec, etc.).
loki.source.journal "systemd" {
path = "/var/log/journal"
max_age = "12h"
labels = {
node = sys.env("APEX_NODE_HOST"),
job = "systemd-journal",
}
forward_to = [loki.write.default.receiver]
}
// All docker containers' stdout/stderr.
discovery.docker "containers" {
host = "unix:///var/run/docker.sock"
}
discovery.relabel "container_logs" {
targets = discovery.docker.containers.targets
rule {
action = "replace"
source_labels = ["__meta_docker_container_name"]
regex = "/?(.*)"
target_label = "container"
}
rule {
action = "replace"
target_label = "node"
replacement = sys.env("APEX_NODE_HOST")
}
}
loki.source.docker "containers" {
host = "unix:///var/run/docker.sock"
targets = discovery.relabel.container_logs.output
relabel_rules = discovery.relabel.container_logs.rules
forward_to = [loki.write.default.receiver]
}
// Push logs to the central Loki.
loki.write "default" {
endpoint {
url = sys.env("APEX_LOKI_PUSH_URL")
basic_auth {
username = sys.env("APEX_OBS_BASIC_AUTH_USER")
password = sys.env("APEX_OBS_BASIC_AUTH_PASS")
}
}
}
+49
View File
@@ -0,0 +1,49 @@
import json
from pathlib import Path
from types import SimpleNamespace
import unittest
import tempfile
from unittest.mock import Mock
from engine import overlay
backup = overlay.load_module(str(Path(__file__).resolve().parents[1] / 'actions/backup/run.py'),
'test_backup_action')
class BackupRecovery(unittest.TestCase):
def test_failed_attempt_preserves_last_success_and_records_new_failure(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / 'status.json'
backup.write_backup_status(path, 'success', 100)
backup.write_backup_status(path, 'running', 200)
backup.write_backup_status(path, 'failed', 210)
status = json.loads(path.read_text())
self.assertEqual(status['last_success'], 100)
self.assertEqual(status['state'], 'failed')
self.assertEqual(status['updated_at'], 210)
def test_recovers_namespace_owner_before_dependent_without_starting_other_containers(self):
running = {'komf': False, 'komga': False, 'intentionally-stopped': False}
def docker(command, **kwargs):
if command[1] == 'inspect':
return SimpleNamespace(returncode=0, stdout=json.dumps([
{'Id': name, 'State': {'Running': running[name]}} for name in command[2:]]))
name = command[2]
if name == 'komf' and not running['komga']:
return SimpleNamespace(returncode=1)
running[name] = True
return SimpleNamespace(returncode=0)
ctx = SimpleNamespace(sys=SimpleNamespace(run=docker), log=Mock())
backup.restore_containers(ctx, ['komf', 'komga'])
self.assertTrue(running['komf'])
self.assertTrue(running['komga'])
self.assertFalse(running['intentionally-stopped'])
def test_unrecoverable_container_prevents_success(self):
def docker(command, **kwargs):
return SimpleNamespace(returncode=0 if command[1] == 'inspect' else 1,
stdout='[{"Id":"broken","State":{"Running":false}}]')
ctx = SimpleNamespace(sys=SimpleNamespace(run=docker), log=Mock())
with self.assertRaisesRegex(RuntimeError, 'broken'):
backup.restore_containers(ctx, ['broken'])
+37
View File
@@ -0,0 +1,37 @@
import importlib.util
from pathlib import Path
import subprocess
import tempfile
from types import SimpleNamespace
import unittest
from unittest.mock import Mock, patch
from engine.lib.sys import Sys
spec = importlib.util.spec_from_file_location('compose_lint', Path(__file__).resolve().parents[1] / 'actions/utils/lint-docker-compose.py')
lint = importlib.util.module_from_spec(spec)
spec.loader.exec_module(lint)
class HookTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.repo = Path(self.tmp.name)
subprocess.run(['git', 'init', '-q', str(self.repo)], check=True)
self.log = Mock()
self.ctx = SimpleNamespace(log=self.log, sys=Sys(self.log), node=None,
paths=SimpleNamespace(repo_root=str(self.repo), compositions=str(self.repo / 'compositions')))
def stage(self, name, content):
(self.repo / name).write_text(content)
subprocess.run(['git', '-C', str(self.repo), 'add', name], check=True)
def test_documentation_commit_does_not_require_docker(self):
self.stage('README.md', 'Documentation\n')
with patch.object(lint.shutil, 'which', return_value=None):
self.assertEqual(lint.run(self.ctx, SimpleNamespace(hook=True)), 0)
def test_compose_change_still_requires_docker(self):
self.stage('docker-compose.yml', 'services: {}\n')
with patch.object(lint.shutil, 'which', return_value=None):
self.assertNotEqual(lint.run(self.ctx, SimpleNamespace(hook=True)), 0)
+52
View File
@@ -0,0 +1,52 @@
"""Native Alloy opt-in and validation boundaries."""
import json
from pathlib import Path
import tempfile
from types import SimpleNamespace
import unittest
from unittest.mock import Mock
from engine import overlay
ROOT = Path(__file__).resolve().parents[1]
alloy = overlay.load_module(str(ROOT / 'actions/configure/alloy.py'), 'test_alloy_action')
class NativeAlloy(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.configs = Path(self.temp.name)
self.ctx = SimpleNamespace(paths=SimpleNamespace(configs=str(self.configs), commons=str(ROOT)),
sys=Mock(), log=Mock())
def configure(self, **settings):
(self.configs / 'alloy').mkdir(exist_ok=True)
(self.configs / 'alloy/config.json').write_text(json.dumps(settings))
def test_unconfigured_node_is_untouched(self):
alloy.run(self.ctx, SimpleNamespace(dry_run=False))
self.ctx.sys.sudo.assert_not_called()
def test_invalid_role_or_environment_path_cannot_change_service(self):
for role, path in [('unknown', '/etc/alloy/secrets'), ('host', '/etc/alloy/invalid\nUser=root')]:
self.configure(role=role, environment_file=path)
with self.assertRaises(ValueError):
alloy.run(self.ctx, SimpleNamespace(dry_run=False))
self.ctx.sys.sudo.assert_not_called()
def test_dry_run_never_installs_or_starts(self):
self.configure(role='guest', environment_file='/etc/alloy/secrets')
alloy.run(self.ctx, SimpleNamespace(dry_run=True))
self.ctx.sys.sudo.assert_not_called()
def test_failed_config_validation_leaves_existing_service_untouched(self):
self.configure(role='host', environment_file='/etc/alloy/secrets')
def execute(command):
if command[0] == 'python3':
raise RuntimeError('invalid Alloy configuration')
self.assertEqual(command, ['test', '-f', '/etc/alloy/secrets'])
self.ctx.sys.sudo.side_effect = execute
with self.assertRaisesRegex(RuntimeError, 'invalid Alloy'):
alloy.run(self.ctx, SimpleNamespace(dry_run=False))
self.ctx.sys.daemon_reload.assert_not_called()
+133
View File
@@ -0,0 +1,133 @@
import os
import subprocess
import tempfile
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import Mock
from actions.sync import repository
from engine.lib.sys import Sys
class RepositorySyncTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name)
self.remote = self.root / "remote.git"
self.repo = self.root / "node"
self.env = dict(os.environ, GIT_AUTHOR_NAME="Test", GIT_AUTHOR_EMAIL="test@example.invalid",
GIT_COMMITTER_NAME="Test", GIT_COMMITTER_EMAIL="test@example.invalid",
GIT_ALLOW_PROTOCOL="file")
self.git("init", "--bare", str(self.remote))
self.git("clone", str(self.remote), str(self.repo))
self.git("checkout", "-b", "main", cwd=self.repo)
(self.repo / "compositions").mkdir()
(self.repo / "compositions/service").write_text("base\n")
self.commons_source = self.root / "shared"
self.git("init", str(self.commons_source))
(self.commons_source / "file").write_text("shared\n")
self.git("add", ".", cwd=self.commons_source)
self.git("commit", "-m", "shared", cwd=self.commons_source)
self.git("submodule", "add", str(self.commons_source), "commons", cwd=self.repo)
self.git("add", ".", cwd=self.repo)
self.git("commit", "-m", "base", cwd=self.repo)
self.git("push", "origin", "main", cwd=self.repo)
self.log = Mock()
class Commands(Sys):
def run(inner, cmd, **kw):
kw["env"] = dict(self.env, **kw.get("env", {}))
kw["env"]["GIT_ALLOW_PROTOCOL"] = "file"
return super().run(cmd, **kw)
self.ctx = SimpleNamespace(log=self.log, sys=Commands(self.log), notify=Mock(),
node=SimpleNamespace(name="node"),
paths=SimpleNamespace(repo_root=str(self.repo), commons=str(self.repo / "commons"),
compositions=str(self.repo / "compositions")))
self.args = SimpleNamespace(telegram_bot_url=None)
def git(self, *args, cwd=None):
return subprocess.check_output(["git", *args], cwd=cwd, env=self.env, stderr=subprocess.PIPE, text=True).strip()
def sync(self):
repository.run(self.ctx, self.args)
def test_local_change_publishes_main_without_sync_branch_or_notification(self):
(self.repo / "compositions/service").write_text("changed\n")
self.sync()
self.assertEqual(self.git("--git-dir=" + str(self.remote), "show", "main:compositions/service"), "changed")
self.assertEqual(self.git("--git-dir=" + str(self.remote), "for-each-ref", "--format=%(refname)", "refs/heads"), "refs/heads/main")
self.ctx.notify.success.assert_not_called()
self.ctx.notify.error.assert_not_called()
def test_noop_keeps_head(self):
before = self.git("rev-parse", "HEAD", cwd=self.repo)
self.sync()
self.assertEqual(before, self.git("rev-parse", "HEAD", cwd=self.repo))
def test_other_branch_does_not_stage_changes(self):
self.git("checkout", "-b", "work", cwd=self.repo)
(self.repo / "compositions/service").write_text("changed\n")
with self.assertRaises(SystemExit):
self.sync()
self.assertEqual(self.git("diff", "--cached", cwd=self.repo), "")
def test_dirty_commons_blocks_parent_commit(self):
before = self.git("rev-parse", "HEAD", cwd=self.repo)
(self.repo / "commons/file").write_text("dirty\n")
with self.assertRaises(SystemExit):
self.sync()
self.assertEqual(before, self.git("rev-parse", "HEAD", cwd=self.repo))
def test_stale_submodule_after_failed_update_cannot_rollback_pin(self):
old = self.git("rev-parse", "HEAD", cwd=self.repo / "commons")
(self.commons_source / "file").write_text("new shared\n")
self.git("add", ".", cwd=self.commons_source)
self.git("commit", "-m", "new shared", cwd=self.commons_source)
new = self.git("rev-parse", "HEAD", cwd=self.commons_source)
self.git("-C", str(self.repo / "commons"), "fetch", "origin")
self.git("update-index", "--cacheinfo", "160000," + new + ",commons", cwd=self.repo)
self.git("commit", "-m", "advance pin", cwd=self.repo)
before = self.git("rev-parse", "HEAD", cwd=self.repo)
with self.assertRaises(SystemExit):
self.sync()
self.assertEqual(before, self.git("rev-parse", "HEAD", cwd=self.repo))
self.assertEqual(old, self.git("rev-parse", "HEAD", cwd=self.repo / "commons"))
self.git("submodule", "update", "--init", cwd=self.repo)
self.sync()
self.assertEqual(new, self.git("--git-dir=" + str(self.remote), "rev-parse", "main:commons"))
def test_concurrent_main_advance_rejects_push_without_overwriting(self):
original = self.ctx.sys.run
other = self.root / "other"
self.git("clone", "--branch", "main", str(self.remote), str(other))
def race(cmd, **kw):
if "push" in cmd:
(other / "compositions/service").write_text("concurrent\n")
self.git("add", ".", cwd=other)
self.git("commit", "-m", "concurrent", cwd=other)
self.git("push", "origin", "main", cwd=other)
return original(cmd, **kw)
self.ctx.sys.run = race
(self.repo / "compositions/service").write_text("local\n")
with self.assertRaises(SystemExit):
self.sync()
self.assertEqual("concurrent", self.git("--git-dir=" + str(self.remote), "show", "main:compositions/service"))
self.assertEqual("local", self.git("show", "HEAD:compositions/service", cwd=self.repo))
def test_conflict_preserves_local_commit_and_remote(self):
other = self.root / "other"
self.git("clone", "--branch", "main", str(self.remote), str(other))
(other / "compositions/service").write_text("remote\n")
self.git("add", ".", cwd=other)
self.git("commit", "-m", "remote", cwd=other)
self.git("push", "origin", "main", cwd=other)
(self.repo / "compositions/service").write_text("local\n")
with self.assertRaises(SystemExit):
self.sync()
self.assertEqual(self.git("show", "HEAD:compositions/service", cwd=self.repo), "local")
self.assertEqual(self.git("--git-dir=" + str(self.remote), "show", "main:compositions/service"), "remote")
self.assertFalse((self.repo / ".git/rebase-merge").exists())
if __name__ == "__main__":
unittest.main()