homelab-optimized/scripts/verify-infrastructure-status.sh

#!/bin/bash

# 🔍 Infrastructure Status Verification Script
# Comprehensive health check for homelab infrastructure

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Configuration
ATLANTIS_IP="192.168.0.200"
ATLANTIS_SSH_PORT="60000"
PORTAINER_URL="https://${ATLANTIS_IP}:9443"
DOKUWIKI_URL="http://${ATLANTIS_IP}:8399"
GITEA_URL="https://git.vish.gg"

echo -e "${BLUE}🏠 Homelab Infrastructure Status Verification${NC}"
echo -e "${BLUE}================================================${NC}"
echo ""

# Function to check service status
check_service() {
    local service_name="$1"
    local check_command="$2"
    local expected_result="$3"

    echo -n "Checking $service_name... "

    if eval "$check_command" | grep -q "$expected_result"; then
        echo -e "${GREEN}✅ OK${NC}"
        return 0
    else
        echo -e "${RED}❌ FAILED${NC}"
        return 1
    fi
}

# Function to check HTTP service
check_http_service() {
    local service_name="$1"
    local url="$2"
    local expected_code="${3:-200}"

    echo -n "Checking $service_name... "

    local response_code
    response_code=$(curl -s -o /dev/null -w "%{http_code}" "$url" 2>/dev/null || echo "000")

    if [[ "$response_code" == "$expected_code" ]]; then
        echo -e "${GREEN}✅ OK (HTTP $response_code)${NC}"
        return 0
    else
        echo -e "${RED}❌ FAILED (HTTP $response_code)${NC}"
        return 1
    fi
}

# Function to check SSH connectivity
check_ssh() {
    local host="$1"
    local port="$2"
    local service_name="$3"

    echo -n "Checking $service_name SSH... "

    if ssh -p "$port" -o ConnectTimeout=5 -o BatchMode=yes "$host" "echo 'SSH OK'" 2>/dev/null | grep -q "SSH OK"; then
        echo -e "${GREEN}✅ OK${NC}"
        return 0
    else
        echo -e "${RED}❌ FAILED${NC}"
        return 1
    fi
}

# Initialize counters
total_checks=0
passed_checks=0

echo -e "${YELLOW}🌐 Network Connectivity${NC}"
echo "------------------------"

# Check basic network connectivity
((total_checks++))
if ping -c 1 -W 2 "$ATLANTIS_IP" >/dev/null 2>&1; then
    echo -e "Atlantis IP connectivity... ${GREEN}✅ OK${NC}"
    ((passed_checks++))
else
    echo -e "Atlantis IP connectivity... ${RED}❌ FAILED${NC}"
fi

# Check SSH connectivity
((total_checks++))
if check_ssh "vish@$ATLANTIS_IP" "$ATLANTIS_SSH_PORT" "Atlantis"; then
    ((passed_checks++))
fi

echo ""
echo -e "${YELLOW}🐳 Container Management${NC}"
echo "-------------------------"

# Check Portainer API
((total_checks++))
if check_http_service "Portainer API" "$PORTAINER_URL/api/status"; then
    ((passed_checks++))

    # Get Portainer version if accessible
    portainer_version=$(curl -k -s "$PORTAINER_URL/api/status" 2>/dev/null | jq -r '.Version' 2>/dev/null || echo "Unknown")
    echo "  └─ Version: $portainer_version"
fi

# Check container count via SSH
((total_checks++))
echo -n "Checking container count... "
container_count=$(ssh -p "$ATLANTIS_SSH_PORT" "vish@$ATLANTIS_IP" "docker ps -q 2>/dev/null | wc -l" 2>/dev/null || echo "0")
if [[ "$container_count" -gt 0 ]]; then
    echo -e "${GREEN}✅ OK ($container_count containers)${NC}"
    ((passed_checks++))
else
    echo -e "${RED}❌ FAILED (No containers or access denied)${NC}"
fi

echo ""
echo -e "${YELLOW}📚 Documentation Systems${NC}"
echo "----------------------------"

# Check DokuWiki
((total_checks++))
if check_http_service "DokuWiki" "$DOKUWIKI_URL/doku.php?id=homelab:start"; then
    ((passed_checks++))

    # Check if homelab documentation is accessible
    if curl -s "$DOKUWIKI_URL/doku.php?id=homelab:start" 2>/dev/null | grep -q "homelab:start"; then
        echo "  └─ Homelab documentation: ✅ Available"
    else
        echo "  └─ Homelab documentation: ⚠️ May not be synced"
    fi
fi

# Check Gitea
((total_checks++))
if check_http_service "Gitea" "$GITEA_URL"; then
    ((passed_checks++))

    # Check repository accessibility
    if curl -s "$GITEA_URL/Vish/homelab" 2>/dev/null | grep -q "homelab"; then
        echo "  └─ Repository access: ✅ Available"
    else
        echo "  └─ Repository access: ⚠️ May require authentication"
    fi
fi

echo ""
echo -e "${YELLOW}🔧 GitOps Deployment${NC}"
echo "----------------------"

# Check if we can access Portainer stacks
((total_checks++))
echo -n "Checking GitOps stacks... "
if command -v jq >/dev/null 2>&1; then
    # This would require authentication, so we'll just check if the endpoint responds
    if curl -k -s -o /dev/null -w "%{http_code}" "$PORTAINER_URL/api/stacks" 2>/dev/null | grep -q "401\|200"; then
        echo -e "${GREEN}✅ OK (API accessible)${NC}"
        ((passed_checks++))
        echo "  └─ Note: Authentication required for detailed stack info"
    else
        echo -e "${RED}❌ FAILED${NC}"
    fi
else
    echo -e "${YELLOW}⚠️ SKIPPED (jq not available)${NC}"
    ((passed_checks++)) # Don't count as failure
fi

echo ""
echo -e "${YELLOW}📊 System Resources${NC}"
echo "---------------------"

# Check disk space on Atlantis
((total_checks++))
echo -n "Checking Atlantis disk space... "
disk_usage=$(ssh -p "$ATLANTIS_SSH_PORT" "vish@$ATLANTIS_IP" "df -h / | tail -1 | awk '{print \$5}' | sed 's/%//'" 2>/dev/null || echo "100")
if [[ "$disk_usage" -lt 90 ]]; then
    echo -e "${GREEN}✅ OK (${disk_usage}% used)${NC}"
    ((passed_checks++))
elif [[ "$disk_usage" -lt 95 ]]; then
    echo -e "${YELLOW}⚠️ WARNING (${disk_usage}% used)${NC}"
    ((passed_checks++))
else
    echo -e "${RED}❌ CRITICAL (${disk_usage}% used)${NC}"
fi

# Check memory usage
((total_checks++))
echo -n "Checking Atlantis memory... "
memory_usage=$(ssh -p "$ATLANTIS_SSH_PORT" "vish@$ATLANTIS_IP" "free | grep Mem | awk '{printf \"%.0f\", \$3/\$2 * 100}'" 2>/dev/null || echo "100")
if [[ "$memory_usage" -lt 85 ]]; then
    echo -e "${GREEN}✅ OK (${memory_usage}% used)${NC}"
    ((passed_checks++))
elif [[ "$memory_usage" -lt 95 ]]; then
    echo -e "${YELLOW}⚠️ WARNING (${memory_usage}% used)${NC}"
    ((passed_checks++))
else
    echo -e "${RED}❌ CRITICAL (${memory_usage}% used)${NC}"
fi

echo ""
echo -e "${YELLOW}🔍 Service Discovery${NC}"
echo "---------------------"

# Check common service ports
common_services=(
    "8080:Portainer Agent"
    "9443:Portainer Server"
    "8399:DokuWiki"
    "3000:Grafana"
    "9090:Prometheus"
    "8096:Jellyfin"
    "32400:Plex"
)

for service in "${common_services[@]}"; do
    port=$(echo "$service" | cut -d: -f1)
    name=$(echo "$service" | cut -d: -f2)

    ((total_checks++))
    echo -n "Checking $name (port $port)... "

    if ssh -p "$ATLANTIS_SSH_PORT" "vish@$ATLANTIS_IP" "netstat -tlnp 2>/dev/null | grep -q :$port" 2>/dev/null; then
        echo -e "${GREEN}✅ LISTENING${NC}"
        ((passed_checks++))
    else
        echo -e "${YELLOW}⚠️ NOT LISTENING${NC}"
    fi
done

echo ""
echo -e "${BLUE}📊 Summary${NC}"
echo "============"

# Calculate success rate
success_rate=$((passed_checks * 100 / total_checks))

echo "Total checks: $total_checks"
echo "Passed: $passed_checks"
echo "Failed: $((total_checks - passed_checks))"
echo -n "Success rate: "

if [[ $success_rate -ge 90 ]]; then
    echo -e "${GREEN}$success_rate% ✅ EXCELLENT${NC}"
elif [[ $success_rate -ge 75 ]]; then
    echo -e "${YELLOW}$success_rate% ⚠️ GOOD${NC}"
elif [[ $success_rate -ge 50 ]]; then
    echo -e "${YELLOW}$success_rate% ⚠️ NEEDS ATTENTION${NC}"
else
    echo -e "${RED}$success_rate% ❌ CRITICAL${NC}"
fi

echo ""
echo -e "${BLUE}🔗 Quick Access Links${NC}"
echo "======================"
echo "• Portainer: https://$ATLANTIS_IP:9443"
echo "• DokuWiki: http://$ATLANTIS_IP:8399/doku.php?id=homelab:start"
echo "• Gitea: $GITEA_URL/Vish/homelab"
echo "• SSH: ssh -p $ATLANTIS_SSH_PORT vish@$ATLANTIS_IP"

echo ""
echo -e "${BLUE}📅 Report Generated: $(date)${NC}"

# Exit with appropriate code
if [[ $success_rate -ge 75 ]]; then
    exit 0
else
    exit 1
fi