#!/bin/bash # ============================================================================= # WATCHTOWER ATLANTIS FIX SCRIPT # ============================================================================= # # Purpose: Fix common Watchtower issues on Atlantis server # Created: February 9, 2026 # Based on: Incident resolution for Watchtower container not running # # Usage: ./fix-watchtower-atlantis.sh # Requirements: SSH access to Atlantis, sudo privileges # # ============================================================================= set -e # Exit on any error # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # No Color # Configuration ATLANTIS_HOST="atlantis" CONTAINER_NAME="watchtower" API_PORT="8082" API_TOKEN=REDACTED_TOKEN echo -e "${BLUE}🔧 Watchtower Atlantis Fix Script${NC}" echo -e "${BLUE}===================================${NC}" echo "" # Function to run commands on Atlantis run_on_atlantis() { local cmd="$1" echo -e "${YELLOW}Running on Atlantis:${NC} $cmd" ssh "$ATLANTIS_HOST" "$cmd" } # Function to check if we can connect to Atlantis check_connection() { echo -e "${BLUE}📡 Checking connection to Atlantis...${NC}" if ssh -o ConnectTimeout=5 "$ATLANTIS_HOST" "echo 'Connection successful'" >/dev/null 2>&1; then echo -e "${GREEN}✅ Connected to Atlantis successfully${NC}" return 0 else echo -e "${RED}❌ Cannot connect to Atlantis${NC}" echo "Please ensure:" echo " - SSH access is configured" echo " - Atlantis server is reachable" echo " - SSH keys are properly set up" exit 1 fi } # Function to check Docker permissions check_docker_permissions() { echo -e "${BLUE}🔐 Checking Docker permissions...${NC}" # Try without sudo first if run_on_atlantis "docker ps >/dev/null 2>&1"; then echo -e "${GREEN}✅ Docker access available without sudo${NC}" DOCKER_CMD="docker" else echo -e "${YELLOW}⚠️ Docker requires sudo privileges${NC}" if run_on_atlantis "sudo docker ps >/dev/null 2>&1"; then echo -e "${GREEN}✅ Docker access available with sudo${NC}" DOCKER_CMD="sudo docker" else echo -e "${RED}❌ Cannot access Docker even with sudo${NC}" exit 1 fi fi } # Function to check Watchtower container status check_watchtower_status() { echo -e "${BLUE}🔍 Checking Watchtower container status...${NC}" local container_info container_info=$(run_on_atlantis "$DOCKER_CMD ps -a --filter name=$CONTAINER_NAME --format 'table {{.Names}}\t{{.Status}}\t{{.State}}'") if echo "$container_info" | grep -q "$CONTAINER_NAME"; then echo -e "${GREEN}✅ Watchtower container found${NC}" echo "$container_info" # Check if running if echo "$container_info" | grep -q "Up"; then echo -e "${GREEN}✅ Watchtower is running${NC}" return 0 else echo -e "${YELLOW}⚠️ Watchtower is not running${NC}" return 1 fi else echo -e "${RED}❌ Watchtower container not found${NC}" return 2 fi } # Function to start Watchtower container start_watchtower() { echo -e "${BLUE}🚀 Starting Watchtower container...${NC}" if run_on_atlantis "$DOCKER_CMD start $CONTAINER_NAME"; then echo -e "${GREEN}✅ Watchtower started successfully${NC}" # Wait a moment for startup sleep 3 # Verify it's running if check_watchtower_status >/dev/null; then echo -e "${GREEN}✅ Watchtower is now running and healthy${NC}" return 0 else echo -e "${RED}❌ Watchtower failed to start properly${NC}" return 1 fi else echo -e "${RED}❌ Failed to start Watchtower${NC}" return 1 fi } # Function to check Watchtower logs check_watchtower_logs() { echo -e "${BLUE}📋 Checking Watchtower logs...${NC}" local logs logs=$(run_on_atlantis "$DOCKER_CMD logs $CONTAINER_NAME --tail 10 2>/dev/null" || echo "No logs available") if [ "$logs" != "No logs available" ] && [ -n "$logs" ]; then echo -e "${GREEN}✅ Recent logs:${NC}" echo "$logs" | sed 's/^/ /' else echo -e "${YELLOW}⚠️ No logs available (container may not have started yet)${NC}" fi } # Function to test Watchtower API test_watchtower_api() { echo -e "${BLUE}🌐 Testing Watchtower API...${NC}" local api_response api_response=$(run_on_atlantis "curl -s -w 'HTTP_STATUS:%{http_code}' http://localhost:$API_PORT/v1/update" 2>/dev/null || echo "API_ERROR") if echo "$api_response" | grep -q "HTTP_STATUS:401"; then echo -e "${GREEN}✅ API is responding (401 = authentication required, which is correct)${NC}" echo -e "${BLUE}💡 API URL: http://atlantis:$API_PORT/v1/update${NC}" echo -e "${BLUE}💡 API Token: $API_TOKEN${NC}" return 0 elif echo "$api_response" | grep -q "HTTP_STATUS:200"; then echo -e "${GREEN}✅ API is responding and accessible${NC}" return 0 else echo -e "${YELLOW}⚠️ API test failed or unexpected response${NC}" echo "Response: $api_response" return 1 fi } # Function to verify container configuration verify_configuration() { echo -e "${BLUE}⚙️ Verifying container configuration...${NC}" local restart_policy restart_policy=$(run_on_atlantis "$DOCKER_CMD inspect $CONTAINER_NAME --format '{{.HostConfig.RestartPolicy.Name}}'" 2>/dev/null || echo "unknown") if [ "$restart_policy" = "always" ]; then echo -e "${GREEN}✅ Restart policy: always (will auto-start on reboot)${NC}" else echo -e "${YELLOW}⚠️ Restart policy: $restart_policy (may not auto-start on reboot)${NC}" fi # Check port mapping local port_mapping port_mapping=$(run_on_atlantis "$DOCKER_CMD port $CONTAINER_NAME 2>/dev/null" || echo "No ports mapped") if echo "$port_mapping" | grep -q "$API_PORT"; then echo -e "${GREEN}✅ Port mapping: $port_mapping${NC}" else echo -e "${YELLOW}⚠️ Port mapping: $port_mapping${NC}" fi } # Main execution main() { echo -e "${BLUE}Starting Watchtower diagnostics and fix...${NC}" echo "" # Step 1: Check connection check_connection echo "" # Step 2: Check Docker permissions check_docker_permissions echo "" # Step 3: Check Watchtower status local watchtower_status check_watchtower_status watchtower_status=$? echo "" # Step 4: Start Watchtower if needed if [ $watchtower_status -eq 1 ]; then echo -e "${YELLOW}🔧 Watchtower needs to be started...${NC}" start_watchtower echo "" elif [ $watchtower_status -eq 2 ]; then echo -e "${RED}❌ Watchtower container not found. Please check deployment.${NC}" exit 1 fi # Step 5: Check logs check_watchtower_logs echo "" # Step 6: Test API test_watchtower_api echo "" # Step 7: Verify configuration verify_configuration echo "" # Final status echo -e "${GREEN}🎉 Watchtower fix script completed!${NC}" echo "" echo -e "${BLUE}📋 Summary:${NC}" echo " • Watchtower container: Running" echo " • HTTP API: Available on port $API_PORT" echo " • Authentication: Required (token: $API_TOKEN)" echo " • Auto-restart: Configured" echo "" echo -e "${BLUE}💡 Next steps:${NC}" echo " • Monitor container health" echo " • Check automatic updates are working" echo " • Review logs periodically" echo "" echo -e "${GREEN}✅ All checks completed successfully!${NC}" } # Run main function main "$@"