#!/bin/bash # 🔍 Infrastructure Status Verification Script # Comprehensive health check for homelab infrastructure set -euo pipefail # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # No Color # Configuration ATLANTIS_IP="192.168.0.200" ATLANTIS_SSH_PORT="60000" PORTAINER_URL="https://${ATLANTIS_IP}:9443" DOKUWIKI_URL="http://${ATLANTIS_IP}:8399" GITEA_URL="https://git.vish.gg" echo -e "${BLUE}🏠 Homelab Infrastructure Status Verification${NC}" echo -e "${BLUE}================================================${NC}" echo "" # Function to check service status check_service() { local service_name="$1" local check_command="$2" local expected_result="$3" echo -n "Checking $service_name... " if eval "$check_command" | grep -q "$expected_result"; then echo -e "${GREEN}✅ OK${NC}" return 0 else echo -e "${RED}❌ FAILED${NC}" return 1 fi } # Function to check HTTP service check_http_service() { local service_name="$1" local url="$2" local expected_code="${3:-200}" echo -n "Checking $service_name... " local response_code response_code=$(curl -s -o /dev/null -w "%{http_code}" "$url" 2>/dev/null || echo "000") if [[ "$response_code" == "$expected_code" ]]; then echo -e "${GREEN}✅ OK (HTTP $response_code)${NC}" return 0 else echo -e "${RED}❌ FAILED (HTTP $response_code)${NC}" return 1 fi } # Function to check SSH connectivity check_ssh() { local host="$1" local port="$2" local service_name="$3" echo -n "Checking $service_name SSH... " if ssh -p "$port" -o ConnectTimeout=5 -o BatchMode=yes "$host" "echo 'SSH OK'" 2>/dev/null | grep -q "SSH OK"; then echo -e "${GREEN}✅ OK${NC}" return 0 else echo -e "${RED}❌ FAILED${NC}" return 1 fi } # Initialize counters total_checks=0 passed_checks=0 echo -e "${YELLOW}🌐 Network Connectivity${NC}" echo "------------------------" # Check basic network connectivity ((total_checks++)) if ping -c 1 -W 2 "$ATLANTIS_IP" >/dev/null 2>&1; then echo -e "Atlantis IP connectivity... ${GREEN}✅ OK${NC}" ((passed_checks++)) else echo -e "Atlantis IP connectivity... ${RED}❌ FAILED${NC}" fi # Check SSH connectivity ((total_checks++)) if check_ssh "vish@$ATLANTIS_IP" "$ATLANTIS_SSH_PORT" "Atlantis"; then ((passed_checks++)) fi echo "" echo -e "${YELLOW}🐳 Container Management${NC}" echo "-------------------------" # Check Portainer API ((total_checks++)) if check_http_service "Portainer API" "$PORTAINER_URL/api/status"; then ((passed_checks++)) # Get Portainer version if accessible portainer_version=$(curl -k -s "$PORTAINER_URL/api/status" 2>/dev/null | jq -r '.Version' 2>/dev/null || echo "Unknown") echo " └─ Version: $portainer_version" fi # Check container count via SSH ((total_checks++)) echo -n "Checking container count... " container_count=$(ssh -p "$ATLANTIS_SSH_PORT" "vish@$ATLANTIS_IP" "docker ps -q 2>/dev/null | wc -l" 2>/dev/null || echo "0") if [[ "$container_count" -gt 0 ]]; then echo -e "${GREEN}✅ OK ($container_count containers)${NC}" ((passed_checks++)) else echo -e "${RED}❌ FAILED (No containers or access denied)${NC}" fi echo "" echo -e "${YELLOW}📚 Documentation Systems${NC}" echo "----------------------------" # Check DokuWiki ((total_checks++)) if check_http_service "DokuWiki" "$DOKUWIKI_URL/doku.php?id=homelab:start"; then ((passed_checks++)) # Check if homelab documentation is accessible if curl -s "$DOKUWIKI_URL/doku.php?id=homelab:start" 2>/dev/null | grep -q "homelab:start"; then echo " └─ Homelab documentation: ✅ Available" else echo " └─ Homelab documentation: ⚠️ May not be synced" fi fi # Check Gitea ((total_checks++)) if check_http_service "Gitea" "$GITEA_URL"; then ((passed_checks++)) # Check repository accessibility if curl -s "$GITEA_URL/Vish/homelab" 2>/dev/null | grep -q "homelab"; then echo " └─ Repository access: ✅ Available" else echo " └─ Repository access: ⚠️ May require authentication" fi fi echo "" echo -e "${YELLOW}🔧 GitOps Deployment${NC}" echo "----------------------" # Check if we can access Portainer stacks ((total_checks++)) echo -n "Checking GitOps stacks... " if command -v jq >/dev/null 2>&1; then # This would require authentication, so we'll just check if the endpoint responds if curl -k -s -o /dev/null -w "%{http_code}" "$PORTAINER_URL/api/stacks" 2>/dev/null | grep -q "401\|200"; then echo -e "${GREEN}✅ OK (API accessible)${NC}" ((passed_checks++)) echo " └─ Note: Authentication required for detailed stack info" else echo -e "${RED}❌ FAILED${NC}" fi else echo -e "${YELLOW}⚠️ SKIPPED (jq not available)${NC}" ((passed_checks++)) # Don't count as failure fi echo "" echo -e "${YELLOW}📊 System Resources${NC}" echo "---------------------" # Check disk space on Atlantis ((total_checks++)) echo -n "Checking Atlantis disk space... " disk_usage=$(ssh -p "$ATLANTIS_SSH_PORT" "vish@$ATLANTIS_IP" "df -h / | tail -1 | awk '{print \$5}' | sed 's/%//'" 2>/dev/null || echo "100") if [[ "$disk_usage" -lt 90 ]]; then echo -e "${GREEN}✅ OK (${disk_usage}% used)${NC}" ((passed_checks++)) elif [[ "$disk_usage" -lt 95 ]]; then echo -e "${YELLOW}⚠️ WARNING (${disk_usage}% used)${NC}" ((passed_checks++)) else echo -e "${RED}❌ CRITICAL (${disk_usage}% used)${NC}" fi # Check memory usage ((total_checks++)) echo -n "Checking Atlantis memory... " memory_usage=$(ssh -p "$ATLANTIS_SSH_PORT" "vish@$ATLANTIS_IP" "free | grep Mem | awk '{printf \"%.0f\", \$3/\$2 * 100}'" 2>/dev/null || echo "100") if [[ "$memory_usage" -lt 85 ]]; then echo -e "${GREEN}✅ OK (${memory_usage}% used)${NC}" ((passed_checks++)) elif [[ "$memory_usage" -lt 95 ]]; then echo -e "${YELLOW}⚠️ WARNING (${memory_usage}% used)${NC}" ((passed_checks++)) else echo -e "${RED}❌ CRITICAL (${memory_usage}% used)${NC}" fi echo "" echo -e "${YELLOW}🔍 Service Discovery${NC}" echo "---------------------" # Check common service ports common_services=( "8080:Portainer Agent" "9443:Portainer Server" "8399:DokuWiki" "3000:Grafana" "9090:Prometheus" "8096:Jellyfin" "32400:Plex" ) for service in "${common_services[@]}"; do port=$(echo "$service" | cut -d: -f1) name=$(echo "$service" | cut -d: -f2) ((total_checks++)) echo -n "Checking $name (port $port)... " if ssh -p "$ATLANTIS_SSH_PORT" "vish@$ATLANTIS_IP" "netstat -tlnp 2>/dev/null | grep -q :$port" 2>/dev/null; then echo -e "${GREEN}✅ LISTENING${NC}" ((passed_checks++)) else echo -e "${YELLOW}⚠️ NOT LISTENING${NC}" fi done echo "" echo -e "${BLUE}📊 Summary${NC}" echo "============" # Calculate success rate success_rate=$((passed_checks * 100 / total_checks)) echo "Total checks: $total_checks" echo "Passed: $passed_checks" echo "Failed: $((total_checks - passed_checks))" echo -n "Success rate: " if [[ $success_rate -ge 90 ]]; then echo -e "${GREEN}$success_rate% ✅ EXCELLENT${NC}" elif [[ $success_rate -ge 75 ]]; then echo -e "${YELLOW}$success_rate% ⚠️ GOOD${NC}" elif [[ $success_rate -ge 50 ]]; then echo -e "${YELLOW}$success_rate% ⚠️ NEEDS ATTENTION${NC}" else echo -e "${RED}$success_rate% ❌ CRITICAL${NC}" fi echo "" echo -e "${BLUE}🔗 Quick Access Links${NC}" echo "======================" echo "• Portainer: https://$ATLANTIS_IP:9443" echo "• DokuWiki: http://$ATLANTIS_IP:8399/doku.php?id=homelab:start" echo "• Gitea: $GITEA_URL/Vish/homelab" echo "• SSH: ssh -p $ATLANTIS_SSH_PORT vish@$ATLANTIS_IP" echo "" echo -e "${BLUE}📅 Report Generated: $(date)${NC}" # Exit with appropriate code if [[ $success_rate -ge 75 ]]; then exit 0 else exit 1 fi