25666a76cf
- Add build-container.yml: Main build pipeline with multi-arch support - Add pr-check.yml: Pull request validation with comprehensive testing - Add release.yml: Automated release pipeline with security scanning - Add nightly.yml: Daily builds with performance testing - Add health_check.sh: Container health validation script - Add setup-ci.sh: Local CI/CD environment setup script - Add comprehensive CI/CD documentation Features: - Multi-architecture builds (linux/amd64, linux/arm64) - Security scanning with Trivy - Automated PyPI publishing for releases - Container registry integration - Performance testing and validation - Artifact management and cleanup - Build caching and optimization Supports full development workflow from PR to production deployment.
446 lines
11 KiB
Bash
Executable File
446 lines
11 KiB
Bash
Executable File
#!/bin/bash
|
|
# CI/CD Setup Script for UnitForge
|
|
# Sets up local environment for testing CI/CD workflows
|
|
|
|
set -e
|
|
|
|
# Colors for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# Helper functions
|
|
log_info() {
|
|
echo -e "${GREEN}[INFO]${NC} $1"
|
|
}
|
|
|
|
log_warn() {
|
|
echo -e "${YELLOW}[WARN]${NC} $1"
|
|
}
|
|
|
|
log_error() {
|
|
echo -e "${RED}[ERROR]${NC} $1"
|
|
}
|
|
|
|
log_step() {
|
|
echo -e "${BLUE}[STEP]${NC} $1"
|
|
}
|
|
|
|
# Check if command exists
|
|
command_exists() {
|
|
command -v "$1" >/dev/null 2>&1
|
|
}
|
|
|
|
# Check Docker and Docker Buildx
|
|
check_docker() {
|
|
log_step "Checking Docker installation..."
|
|
|
|
if ! command_exists docker; then
|
|
log_error "Docker is not installed"
|
|
echo "Please install Docker from: https://docs.docker.com/get-docker/"
|
|
return 1
|
|
fi
|
|
|
|
log_info "Docker found: $(docker --version)"
|
|
|
|
# Check if Docker daemon is running
|
|
if ! docker info >/dev/null 2>&1; then
|
|
log_error "Docker daemon is not running"
|
|
echo "Please start Docker daemon"
|
|
return 1
|
|
fi
|
|
|
|
# Check Docker Buildx
|
|
if ! docker buildx version >/dev/null 2>&1; then
|
|
log_warn "Docker Buildx not found, installing..."
|
|
docker buildx install 2>/dev/null || true
|
|
fi
|
|
|
|
log_info "Docker Buildx found: $(docker buildx version)"
|
|
|
|
return 0
|
|
}
|
|
|
|
# Setup Docker Buildx for multi-arch builds
|
|
setup_buildx() {
|
|
log_step "Setting up Docker Buildx for multi-arch builds..."
|
|
|
|
# Create builder if it doesn't exist
|
|
if ! docker buildx ls | grep -q "unitforge-builder"; then
|
|
log_info "Creating unitforge-builder..."
|
|
docker buildx create --name unitforge-builder --use
|
|
else
|
|
log_info "Using existing unitforge-builder"
|
|
docker buildx use unitforge-builder
|
|
fi
|
|
|
|
# Bootstrap the builder
|
|
log_info "Bootstrapping builder..."
|
|
docker buildx inspect --bootstrap
|
|
|
|
log_info "Builder setup complete"
|
|
return 0
|
|
}
|
|
|
|
# Check container registry access
|
|
check_registry() {
|
|
log_step "Checking container registry configuration..."
|
|
|
|
if [ -f ".env" ]; then
|
|
log_info "Found .env file"
|
|
|
|
if grep -q "CONTAINER_REGISTRY_URL" .env; then
|
|
local registry_url
|
|
registry_url=$(grep '^CONTAINER_REGISTRY_URL=' .env | cut -d'=' -f2)
|
|
log_info "Registry URL: $registry_url"
|
|
else
|
|
log_warn "CONTAINER_REGISTRY_URL not found in .env"
|
|
fi
|
|
|
|
if grep -q "CONTAINER_TAG" .env; then
|
|
local container_tag
|
|
container_tag=$(grep '^CONTAINER_TAG=' .env | cut -d'=' -f2)
|
|
log_info "Container tag: $container_tag"
|
|
else
|
|
log_warn "CONTAINER_TAG not found in .env"
|
|
fi
|
|
else
|
|
log_warn ".env file not found"
|
|
log_info "Creating sample .env file..."
|
|
|
|
cat > .env << EOF
|
|
# Container Registry Configuration
|
|
CONTAINER_REGISTRY_URL=gitea-http.taildb3494.ts.net/will/unitforge
|
|
CONTAINER_TAG=latest
|
|
|
|
# Development Configuration
|
|
DEBUG=true
|
|
LOG_LEVEL=debug
|
|
EOF
|
|
|
|
log_info "Sample .env file created. Please update with your registry details."
|
|
fi
|
|
|
|
return 0
|
|
}
|
|
|
|
# Verify vendor assets
|
|
check_vendor_assets() {
|
|
log_step "Checking vendor assets..."
|
|
|
|
local assets=(
|
|
"frontend/static/vendor/bootstrap/css/bootstrap.min.css"
|
|
"frontend/static/vendor/bootstrap/js/bootstrap.bundle.min.js"
|
|
"frontend/static/vendor/fontawesome/css/all.min.css"
|
|
"frontend/static/vendor/fontawesome/webfonts/fa-solid-900.woff2"
|
|
"frontend/static/img/osi-logo.svg"
|
|
)
|
|
|
|
local missing_assets=0
|
|
|
|
for asset in "${assets[@]}"; do
|
|
if [ ! -f "$asset" ]; then
|
|
log_warn "Missing asset: $asset"
|
|
missing_assets=$((missing_assets + 1))
|
|
else
|
|
log_info "Found asset: $asset"
|
|
fi
|
|
done
|
|
|
|
if [ $missing_assets -gt 0 ]; then
|
|
log_error "$missing_assets vendor assets are missing"
|
|
log_info "Please ensure all vendor assets are downloaded and committed"
|
|
log_info "Run: make setup-dev to download missing assets"
|
|
return 1
|
|
fi
|
|
|
|
log_info "All vendor assets found"
|
|
return 0
|
|
}
|
|
|
|
# Test local build
|
|
test_local_build() {
|
|
log_step "Testing local Docker build..."
|
|
|
|
if docker build -t unitforge:test . >/dev/null 2>&1; then
|
|
log_info "Local Docker build successful"
|
|
|
|
# Test container startup
|
|
log_info "Testing container startup..."
|
|
if docker run -d --name unitforge-test -p 8080:8000 unitforge:test >/dev/null 2>&1; then
|
|
sleep 5
|
|
|
|
if curl -s -f http://localhost:8080/ >/dev/null 2>&1; then
|
|
log_info "Container startup test successful"
|
|
else
|
|
log_warn "Container started but not responding on port 8080"
|
|
fi
|
|
|
|
docker stop unitforge-test >/dev/null 2>&1
|
|
docker rm unitforge-test >/dev/null 2>&1
|
|
else
|
|
log_warn "Container startup test failed"
|
|
fi
|
|
|
|
# Clean up test image
|
|
docker rmi unitforge:test >/dev/null 2>&1 || true
|
|
|
|
return 0
|
|
else
|
|
log_error "Local Docker build failed"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Test multi-arch build
|
|
test_multiarch_build() {
|
|
log_step "Testing multi-architecture build..."
|
|
|
|
if docker buildx build --platform linux/amd64,linux/arm64 -t unitforge:multiarch-test . >/dev/null 2>&1; then
|
|
log_info "Multi-architecture build successful"
|
|
|
|
# Clean up
|
|
docker buildx rm --force >/dev/null 2>&1 || true
|
|
|
|
return 0
|
|
else
|
|
log_error "Multi-architecture build failed"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Check development environment
|
|
check_dev_environment() {
|
|
log_step "Checking development environment..."
|
|
|
|
# Check uv
|
|
if ! command_exists uv; then
|
|
log_error "uv is not installed"
|
|
echo "Install with: curl -LsSf https://astral.sh/uv/install.sh | sh"
|
|
return 1
|
|
fi
|
|
|
|
log_info "uv found: $(uv --version)"
|
|
|
|
# Check Python
|
|
if ! command_exists python3; then
|
|
log_error "Python 3 is not installed"
|
|
return 1
|
|
fi
|
|
|
|
log_info "Python found: $(python3 --version)"
|
|
|
|
# Check if virtual environment exists
|
|
if [ -d ".venv" ]; then
|
|
log_info "Virtual environment exists"
|
|
else
|
|
log_warn "Virtual environment not found"
|
|
log_info "Run: make setup-dev to create it"
|
|
fi
|
|
|
|
return 0
|
|
}
|
|
|
|
# Test CI/CD workflow syntax
|
|
check_workflow_syntax() {
|
|
log_step "Checking workflow syntax..."
|
|
|
|
local workflows_dir=".gitea/workflows"
|
|
|
|
if [ ! -d "$workflows_dir" ]; then
|
|
log_error "Workflows directory not found: $workflows_dir"
|
|
return 1
|
|
fi
|
|
|
|
local yaml_files=("$workflows_dir"/*.yml)
|
|
|
|
if [ ${#yaml_files[@]} -eq 0 ]; then
|
|
log_warn "No YAML workflow files found"
|
|
return 0
|
|
fi
|
|
|
|
local syntax_errors=0
|
|
|
|
for file in "${yaml_files[@]}"; do
|
|
if [ -f "$file" ]; then
|
|
log_info "Checking syntax: $(basename "$file")"
|
|
|
|
# Basic YAML syntax check (if python3 is available)
|
|
if command_exists python3; then
|
|
if python3 -c "import yaml; yaml.safe_load(open('$file'))" 2>/dev/null; then
|
|
log_info "✓ $(basename "$file") syntax OK"
|
|
else
|
|
log_error "✗ $(basename "$file") has syntax errors"
|
|
syntax_errors=$((syntax_errors + 1))
|
|
fi
|
|
else
|
|
log_warn "Python3 not available, skipping YAML syntax check"
|
|
fi
|
|
fi
|
|
done
|
|
|
|
if [ $syntax_errors -eq 0 ]; then
|
|
log_info "All workflow files have valid syntax"
|
|
return 0
|
|
else
|
|
log_error "$syntax_errors workflow files have syntax errors"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Generate CI/CD documentation
|
|
generate_docs() {
|
|
log_step "Generating CI/CD documentation..."
|
|
|
|
local docs_dir="docs/ci-cd"
|
|
mkdir -p "$docs_dir"
|
|
|
|
cat > "$docs_dir/local-testing.md" << 'EOF'
|
|
# Local CI/CD Testing
|
|
|
|
This guide helps you test CI/CD workflows locally before pushing to the repository.
|
|
|
|
## Prerequisites
|
|
|
|
- Docker with Buildx support
|
|
- uv package manager
|
|
- Python 3.8+
|
|
|
|
## Local Testing Commands
|
|
|
|
```bash
|
|
# Test local build
|
|
make docker-build
|
|
|
|
# Test multi-arch build
|
|
make docker-buildx-local
|
|
|
|
# Test full development workflow
|
|
make dev
|
|
|
|
# Run health checks
|
|
./scripts/health_check.sh
|
|
```
|
|
|
|
## Workflow Testing
|
|
|
|
Use `act` to test GitHub/Gitea workflows locally:
|
|
|
|
```bash
|
|
# Install act
|
|
curl https://raw.githubusercontent.com/nektos/act/master/install.sh | sudo bash
|
|
|
|
# Test PR workflow
|
|
act pull_request -s CONTAINER_REGISTRY_USERNAME=test -s CONTAINER_REGISTRY_PASSWORD=test
|
|
|
|
# Test release workflow
|
|
act push -e tests/fixtures/release-event.json
|
|
```
|
|
|
|
## Troubleshooting
|
|
|
|
### Build Issues
|
|
- Ensure all vendor assets are committed
|
|
- Check Docker daemon is running
|
|
- Verify buildx is properly configured
|
|
|
|
### Registry Issues
|
|
- Check .env file configuration
|
|
- Verify registry credentials
|
|
- Test registry connectivity
|
|
|
|
### Performance Issues
|
|
- Use build cache: `--cache-from type=gha`
|
|
- Optimize Docker layers
|
|
- Use multi-stage builds
|
|
```
|
|
EOF
|
|
|
|
log_info "Local testing documentation generated: $docs_dir/local-testing.md"
|
|
|
|
return 0
|
|
}
|
|
|
|
# Main setup function
|
|
main() {
|
|
echo -e "${BLUE}========================================${NC}"
|
|
echo -e "${BLUE} UnitForge CI/CD Setup${NC}"
|
|
echo -e "${BLUE}========================================${NC}"
|
|
echo ""
|
|
|
|
local failed_checks=0
|
|
|
|
# Run all checks
|
|
check_docker || failed_checks=$((failed_checks + 1))
|
|
setup_buildx || failed_checks=$((failed_checks + 1))
|
|
check_registry || true # Don't fail on registry issues
|
|
check_vendor_assets || failed_checks=$((failed_checks + 1))
|
|
check_dev_environment || failed_checks=$((failed_checks + 1))
|
|
check_workflow_syntax || failed_checks=$((failed_checks + 1))
|
|
|
|
# Optional tests
|
|
if [ "$1" = "--test-build" ]; then
|
|
test_local_build || failed_checks=$((failed_checks + 1))
|
|
test_multiarch_build || failed_checks=$((failed_checks + 1))
|
|
fi
|
|
|
|
# Generate documentation
|
|
generate_docs || true
|
|
|
|
echo ""
|
|
echo -e "${BLUE}========================================${NC}"
|
|
|
|
if [ $failed_checks -eq 0 ]; then
|
|
log_info "✅ CI/CD setup completed successfully!"
|
|
echo ""
|
|
echo "Next steps:"
|
|
echo "1. Update .env with your registry details"
|
|
echo "2. Test local build: make docker-buildx-local"
|
|
echo "3. Run full test suite: make dev"
|
|
echo "4. Check workflow syntax: ./scripts/setup-ci.sh"
|
|
echo ""
|
|
echo "For testing builds:"
|
|
echo " ./scripts/setup-ci.sh --test-build"
|
|
else
|
|
log_error "❌ CI/CD setup completed with $failed_checks issues"
|
|
echo ""
|
|
echo "Please fix the issues above before proceeding."
|
|
exit 1
|
|
fi
|
|
|
|
echo -e "${BLUE}========================================${NC}"
|
|
}
|
|
|
|
# Usage information
|
|
usage() {
|
|
echo "Usage: $0 [OPTIONS]"
|
|
echo ""
|
|
echo "Options:"
|
|
echo " --test-build Run local and multi-arch build tests"
|
|
echo " --help Show this help message"
|
|
echo ""
|
|
echo "This script sets up your local environment for CI/CD development."
|
|
echo "It checks Docker, Buildx, dependencies, and workflow syntax."
|
|
}
|
|
|
|
# Parse command line arguments
|
|
case "${1:-}" in
|
|
--help)
|
|
usage
|
|
exit 0
|
|
;;
|
|
--test-build)
|
|
main --test-build
|
|
;;
|
|
"")
|
|
main
|
|
;;
|
|
*)
|
|
echo "Unknown option: $1"
|
|
usage
|
|
exit 1
|
|
;;
|
|
esac
|