25666a76cf
- Add build-container.yml: Main build pipeline with multi-arch support - Add pr-check.yml: Pull request validation with comprehensive testing - Add release.yml: Automated release pipeline with security scanning - Add nightly.yml: Daily builds with performance testing - Add health_check.sh: Container health validation script - Add setup-ci.sh: Local CI/CD environment setup script - Add comprehensive CI/CD documentation Features: - Multi-architecture builds (linux/amd64, linux/arm64) - Security scanning with Trivy - Automated PyPI publishing for releases - Container registry integration - Performance testing and validation - Artifact management and cleanup - Build caching and optimization Supports full development workflow from PR to production deployment.
297 lines
10 KiB
YAML
297 lines
10 KiB
YAML
name: Nightly Build
|
|
|
|
on:
|
|
schedule:
|
|
# Run every night at 2 AM UTC
|
|
- cron: "0 2 * * *"
|
|
workflow_dispatch:
|
|
inputs:
|
|
force_build:
|
|
description: "Force build even if no changes"
|
|
required: false
|
|
default: "false"
|
|
type: boolean
|
|
|
|
env:
|
|
REGISTRY: gitea-http.taildb3494.ts.net
|
|
IMAGE_NAME: will/unitforge
|
|
|
|
jobs:
|
|
check-changes:
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
should_build: ${{ steps.changes.outputs.should_build }}
|
|
commit_sha: ${{ steps.changes.outputs.commit_sha }}
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 2
|
|
|
|
- name: Check for changes
|
|
id: changes
|
|
run: |
|
|
# Get the latest commit from the last 24 hours
|
|
YESTERDAY=$(date -d "24 hours ago" --iso-8601)
|
|
RECENT_COMMITS=$(git log --since="$YESTERDAY" --format="%H" | wc -l)
|
|
|
|
FORCE_BUILD="${{ github.event.inputs.force_build }}"
|
|
|
|
if [[ "$FORCE_BUILD" == "true" ]] || [[ $RECENT_COMMITS -gt 0 ]]; then
|
|
echo "should_build=true" >> $GITHUB_OUTPUT
|
|
echo "Found $RECENT_COMMITS commits in the last 24 hours or force build requested"
|
|
else
|
|
echo "should_build=false" >> $GITHUB_OUTPUT
|
|
echo "No changes in the last 24 hours, skipping build"
|
|
fi
|
|
|
|
echo "commit_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
|
|
|
|
nightly-tests:
|
|
needs: check-changes
|
|
runs-on: ubuntu-latest
|
|
if: needs.check-changes.outputs.should_build == 'true'
|
|
|
|
strategy:
|
|
matrix:
|
|
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@v3
|
|
with:
|
|
version: "latest"
|
|
|
|
- name: Set up Python ${{ matrix.python-version }}
|
|
run: uv python install ${{ matrix.python-version }}
|
|
|
|
- name: Install dependencies
|
|
run: |
|
|
uv venv --python ${{ matrix.python-version }}
|
|
uv pip install -e ".[dev]"
|
|
|
|
- name: Run comprehensive tests
|
|
run: |
|
|
source .venv/bin/activate
|
|
|
|
# Run all checks
|
|
make lint
|
|
make type-check
|
|
make security-check
|
|
make test-cov
|
|
|
|
# Additional nightly-specific tests
|
|
echo "Running extended test suite..."
|
|
python -m pytest tests/ -v --durations=10 --tb=short
|
|
|
|
- name: Upload coverage for Python ${{ matrix.python-version }}
|
|
uses: codecov/codecov-action@v3
|
|
if: matrix.python-version == '3.11'
|
|
with:
|
|
file: ./htmlcov/coverage.xml
|
|
flags: nightly-${{ matrix.python-version }}
|
|
|
|
build-nightly:
|
|
needs: [check-changes, nightly-tests]
|
|
runs-on: ubuntu-latest
|
|
if: needs.check-changes.outputs.should_build == 'true'
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Set up Docker Buildx
|
|
uses: docker/setup-buildx-action@v3
|
|
with:
|
|
driver-opts: network=host
|
|
|
|
- name: Log in to Container Registry
|
|
uses: docker/login-action@v3
|
|
with:
|
|
registry: ${{ env.REGISTRY }}
|
|
username: ${{ secrets.CONTAINER_REGISTRY_USERNAME }}
|
|
password: ${{ secrets.CONTAINER_REGISTRY_PASSWORD }}
|
|
|
|
- name: Generate nightly tags
|
|
id: tags
|
|
run: |
|
|
COMMIT_SHA="${{ needs.check-changes.outputs.commit_sha }}"
|
|
DATE=$(date +%Y%m%d)
|
|
SHORT_SHA=${COMMIT_SHA:0:7}
|
|
|
|
echo "nightly_tag=nightly-${DATE}-${SHORT_SHA}" >> $GITHUB_OUTPUT
|
|
echo "nightly_latest=nightly-latest" >> $GITHUB_OUTPUT
|
|
|
|
- name: Verify vendor assets
|
|
run: |
|
|
assets=(
|
|
"frontend/static/vendor/bootstrap/css/bootstrap.min.css"
|
|
"frontend/static/vendor/bootstrap/js/bootstrap.bundle.min.js"
|
|
"frontend/static/vendor/fontawesome/css/all.min.css"
|
|
"frontend/static/vendor/fontawesome/webfonts/fa-solid-900.woff2"
|
|
"frontend/static/img/osi-logo.svg"
|
|
)
|
|
|
|
for asset in "${assets[@]}"; do
|
|
if [ ! -f "$asset" ]; then
|
|
echo "Error: Missing required asset: $asset"
|
|
exit 1
|
|
fi
|
|
done
|
|
echo "All vendor assets verified"
|
|
|
|
- name: Build and push nightly image
|
|
uses: docker/build-push-action@v5
|
|
with:
|
|
context: .
|
|
file: ./Dockerfile
|
|
platforms: linux/amd64,linux/arm64
|
|
push: true
|
|
tags: |
|
|
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.nightly_tag }}
|
|
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.nightly_latest }}
|
|
labels: |
|
|
org.opencontainers.image.title=UnitForge Nightly
|
|
org.opencontainers.image.description=Nightly build of UnitForge
|
|
org.opencontainers.image.version=nightly-${{ steps.tags.outputs.nightly_tag }}
|
|
org.opencontainers.image.revision=${{ needs.check-changes.outputs.commit_sha }}
|
|
org.opencontainers.image.created=${{ github.event.repository.pushed_at }}
|
|
cache-from: type=gha
|
|
cache-to: type=gha,mode=max
|
|
build-args: |
|
|
BUILDKIT_INLINE_CACHE=1
|
|
|
|
performance-test:
|
|
needs: [check-changes, build-nightly]
|
|
runs-on: ubuntu-latest
|
|
if: needs.check-changes.outputs.should_build == 'true'
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Run performance tests
|
|
run: |
|
|
# Pull the nightly image
|
|
docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:nightly-latest
|
|
|
|
# Start the container
|
|
docker run -d --name unitforge-perf \
|
|
-p 8000:8000 \
|
|
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:nightly-latest
|
|
|
|
# Wait for startup
|
|
sleep 15
|
|
|
|
# Basic performance test
|
|
echo "Running basic performance test..."
|
|
for i in {1..10}; do
|
|
curl -s -o /dev/null -w "%{http_code} %{time_total}s\n" \
|
|
http://localhost:8000/
|
|
done
|
|
|
|
# Memory usage check
|
|
echo "Checking memory usage..."
|
|
docker stats unitforge-perf --no-stream --format "table {{.Container}}\t{{.CPUPerc}}\t{{.MemUsage}}"
|
|
|
|
# Cleanup
|
|
docker stop unitforge-perf
|
|
docker rm unitforge-perf
|
|
|
|
security-scan-nightly:
|
|
needs: [check-changes, build-nightly]
|
|
runs-on: ubuntu-latest
|
|
if: needs.check-changes.outputs.should_build == 'true'
|
|
|
|
steps:
|
|
- name: Run comprehensive security scan
|
|
uses: aquasecurity/trivy-action@master
|
|
with:
|
|
image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:nightly-latest
|
|
format: "sarif"
|
|
output: "trivy-nightly.sarif"
|
|
|
|
- name: Upload security scan results
|
|
uses: github/codeql-action/upload-sarif@v2
|
|
if: always()
|
|
with:
|
|
sarif_file: "trivy-nightly.sarif"
|
|
|
|
- name: Generate security report
|
|
uses: aquasecurity/trivy-action@master
|
|
with:
|
|
image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:nightly-latest
|
|
format: "json"
|
|
output: "security-report.json"
|
|
|
|
- name: Upload security report
|
|
uses: actions/upload-artifact@v3
|
|
with:
|
|
name: nightly-security-report
|
|
path: security-report.json
|
|
|
|
cleanup-old-nightlies:
|
|
needs: [check-changes, build-nightly]
|
|
runs-on: ubuntu-latest
|
|
if: needs.check-changes.outputs.should_build == 'true'
|
|
|
|
steps:
|
|
- name: Clean up old nightly images
|
|
run: |
|
|
echo "Cleaning up nightly images older than 7 days..."
|
|
|
|
# Note: This would require registry API access or container registry-specific tools
|
|
# For now, we'll just log what would be cleaned
|
|
|
|
CUTOFF_DATE=$(date -d "7 days ago" +%Y%m%d)
|
|
echo "Would clean images tagged before: nightly-${CUTOFF_DATE}"
|
|
|
|
# Add actual cleanup logic here based on your registry
|
|
# Examples:
|
|
# - Use registry API to list and delete old tags
|
|
# - Use container registry CLI tools
|
|
# - Use registry-specific cleanup policies
|
|
|
|
notify-results:
|
|
needs:
|
|
[
|
|
check-changes,
|
|
nightly-tests,
|
|
build-nightly,
|
|
performance-test,
|
|
security-scan-nightly,
|
|
]
|
|
runs-on: ubuntu-latest
|
|
if: always() && needs.check-changes.outputs.should_build == 'true'
|
|
|
|
steps:
|
|
- name: Generate build report
|
|
run: |
|
|
echo "## Nightly Build Report - $(date)" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
echo "| Component | Status |" >> $GITHUB_STEP_SUMMARY
|
|
echo "|-----------|--------|" >> $GITHUB_STEP_SUMMARY
|
|
echo "| Tests | ${{ needs.nightly-tests.result == 'success' && '✅ Passed' || '❌ Failed' }} |" >> $GITHUB_STEP_SUMMARY
|
|
echo "| Build | ${{ needs.build-nightly.result == 'success' && '✅ Passed' || '❌ Failed' }} |" >> $GITHUB_STEP_SUMMARY
|
|
echo "| Performance | ${{ needs.performance-test.result == 'success' && '✅ Passed' || '❌ Failed' }} |" >> $GITHUB_STEP_SUMMARY
|
|
echo "| Security | ${{ needs.security-scan-nightly.result == 'success' && '✅ Passed' || '❌ Failed' }} |" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
|
|
if [[ "${{ needs.nightly-tests.result }}" == "success" && "${{ needs.build-nightly.result }}" == "success" ]]; then
|
|
echo "🌙 Nightly build completed successfully!" >> $GITHUB_STEP_SUMMARY
|
|
echo "📦 Image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:nightly-latest" >> $GITHUB_STEP_SUMMARY
|
|
else
|
|
echo "❌ Nightly build encountered issues. Check failed jobs above." >> $GITHUB_STEP_SUMMARY
|
|
fi
|
|
|
|
- name: Send notification
|
|
if: failure()
|
|
run: |
|
|
echo "🚨 Nightly build failed!"
|
|
echo "Check the workflow run for details: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
|
|
|
# Add notification logic here (webhook, email, Slack, etc.)
|