138 lines
4.6 KiB
Bash
138 lines
4.6 KiB
Bash
#!/bin/bash
|
|
# pre-upgrade-check.sh - Pre-upgrade cluster validation
|
|
# Usage: ./pre-upgrade-check.sh
|
|
|
|
set -e
|
|
|
|
echo "=== PRE-UPGRADE CLUSTER VALIDATION ===" >&2
|
|
echo "Timestamp: $(date -u +"%Y-%m-%dT%H:%M:%SZ")" >&2
|
|
echo "" >&2
|
|
|
|
WARNINGS=0
|
|
BLOCKERS=0
|
|
|
|
# 1. Cluster Version
|
|
echo "### Cluster Version ###" >&2
|
|
SERVER_VERSION=$(kubectl version -o json 2>/dev/null | jq -r '.serverVersion.gitVersion')
|
|
echo "Server Version: $SERVER_VERSION" >&2
|
|
|
|
# 2. Node Status
|
|
echo -e "\n### Node Status ###" >&2
|
|
kubectl get nodes >&2
|
|
NOT_READY=$(kubectl get nodes --no-headers | grep -cv "Ready" || echo 0)
|
|
if [ "$NOT_READY" -gt 0 ]; then
|
|
BLOCKERS=$((BLOCKERS + 1))
|
|
echo "BLOCKER: $NOT_READY nodes not ready" >&2
|
|
fi
|
|
|
|
# 3. Control Plane Health
|
|
echo -e "\n### Control Plane Health ###" >&2
|
|
kubectl get pods -n kube-system -l tier=control-plane 2>/dev/null || \
|
|
kubectl get pods -n kube-system | grep -E "kube-apiserver|kube-controller|kube-scheduler|etcd" >&2
|
|
|
|
# 4. Pods Not Running
|
|
echo -e "\n### Pods Not Running ###" >&2
|
|
NOT_RUNNING=$(kubectl get pods -A --field-selector=status.phase!=Running,status.phase!=Succeeded --no-headers 2>/dev/null | wc -l | tr -d ' ')
|
|
if [ "$NOT_RUNNING" -gt 0 ]; then
|
|
WARNINGS=$((WARNINGS + 1))
|
|
echo "WARNING: $NOT_RUNNING pods not in Running/Succeeded state" >&2
|
|
kubectl get pods -A --field-selector=status.phase!=Running,status.phase!=Succeeded >&2
|
|
else
|
|
echo "✓ All pods running" >&2
|
|
fi
|
|
|
|
# 5. PodDisruptionBudgets
|
|
echo -e "\n### PodDisruptionBudgets ###" >&2
|
|
PDB_COUNT=$(kubectl get pdb -A --no-headers 2>/dev/null | wc -l | tr -d ' ')
|
|
echo "Found $PDB_COUNT PDBs" >&2
|
|
if [ "$PDB_COUNT" -gt 0 ]; then
|
|
kubectl get pdb -A >&2
|
|
fi
|
|
|
|
# 6. Pending PVCs
|
|
echo -e "\n### Pending PVCs ###" >&2
|
|
PENDING_PVC=$(kubectl get pvc -A --field-selector=status.phase=Pending --no-headers 2>/dev/null | wc -l | tr -d ' ')
|
|
if [ "$PENDING_PVC" -gt 0 ]; then
|
|
WARNINGS=$((WARNINGS + 1))
|
|
echo "WARNING: $PENDING_PVC PVCs pending" >&2
|
|
kubectl get pvc -A --field-selector=status.phase=Pending >&2
|
|
else
|
|
echo "✓ No pending PVCs" >&2
|
|
fi
|
|
|
|
# 7. Deprecated APIs
|
|
echo -e "\n### Deprecated API Usage ###" >&2
|
|
DEPRECATED=$(kubectl get --raw /metrics 2>/dev/null | grep -c "apiserver_requested_deprecated_apis" || echo 0)
|
|
if [ "$DEPRECATED" -gt 0 ]; then
|
|
WARNINGS=$((WARNINGS + 1))
|
|
echo "WARNING: Deprecated APIs may be in use" >&2
|
|
echo "Check: kubectl get --raw /metrics | grep apiserver_requested_deprecated_apis" >&2
|
|
else
|
|
echo "✓ No deprecated API metrics found" >&2
|
|
fi
|
|
|
|
# 8. etcd Health (if accessible)
|
|
echo -e "\n### etcd Health ###" >&2
|
|
ETCD_PODS=$(kubectl get pods -n kube-system -l component=etcd --no-headers 2>/dev/null | wc -l | tr -d ' ')
|
|
if [ "$ETCD_PODS" -gt 0 ]; then
|
|
kubectl get pods -n kube-system -l component=etcd >&2
|
|
else
|
|
echo "etcd pods not directly visible (managed platform)" >&2
|
|
fi
|
|
|
|
# 9. Resource Pressure
|
|
echo -e "\n### Node Resource Pressure ###" >&2
|
|
PRESSURE=$(kubectl get nodes -o json 2>/dev/null | jq -r '.items[] | select(.status.conditions[] | select(.type | contains("Pressure")) | .status == "True") | .metadata.name')
|
|
if [ -n "$PRESSURE" ]; then
|
|
WARNINGS=$((WARNINGS + 1))
|
|
echo "WARNING: Nodes under pressure:" >&2
|
|
echo "$PRESSURE" >&2
|
|
else
|
|
echo "✓ No resource pressure detected" >&2
|
|
fi
|
|
|
|
# OpenShift-specific checks
|
|
if command -v oc &> /dev/null && oc whoami &> /dev/null; then
|
|
echo -e "\n### OpenShift Cluster Operators ###" >&2
|
|
DEGRADED=$(oc get clusteroperators --no-headers 2>/dev/null | grep -c -E "False.*True|False.*False" || echo 0)
|
|
if [ "$DEGRADED" -gt 0 ]; then
|
|
BLOCKERS=$((BLOCKERS + 1))
|
|
echo "BLOCKER: $DEGRADED cluster operators degraded" >&2
|
|
oc get clusteroperators | grep -E "False.*True|False.*False" >&2
|
|
else
|
|
echo "✓ All cluster operators healthy" >&2
|
|
fi
|
|
fi
|
|
|
|
# Summary
|
|
echo "" >&2
|
|
echo "========================================" >&2
|
|
echo "PRE-UPGRADE CHECK SUMMARY" >&2
|
|
echo "========================================" >&2
|
|
echo "Blockers: $BLOCKERS" >&2
|
|
echo "Warnings: $WARNINGS" >&2
|
|
|
|
if [ "$BLOCKERS" -gt 0 ]; then
|
|
echo "" >&2
|
|
echo "❌ DO NOT PROCEED WITH UPGRADE" >&2
|
|
echo " Resolve blockers before upgrading" >&2
|
|
elif [ "$WARNINGS" -gt 0 ]; then
|
|
echo "" >&2
|
|
echo "⚠️ PROCEED WITH CAUTION" >&2
|
|
echo " Review warnings before upgrading" >&2
|
|
else
|
|
echo "" >&2
|
|
echo "✅ CLUSTER READY FOR UPGRADE" >&2
|
|
fi
|
|
|
|
# Output JSON
|
|
cat << EOF
|
|
{
|
|
"timestamp": "$(date -u +"%Y-%m-%dT%H:%M:%SZ")",
|
|
"server_version": "$SERVER_VERSION",
|
|
"blockers": $BLOCKERS,
|
|
"warnings": $WARNINGS,
|
|
"ready_for_upgrade": $([ $BLOCKERS -eq 0 ] && echo "true" || echo "false")
|
|
}
|
|
EOF
|