248 lines
7.7 KiB
Bash
Executable File
248 lines
7.7 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# =============================================================================
|
|
# WATCHTOWER ATLANTIS FIX SCRIPT
|
|
# =============================================================================
|
|
#
|
|
# Purpose: Fix common Watchtower issues on Atlantis server
|
|
# Created: February 9, 2026
|
|
# Based on: Incident resolution for Watchtower container not running
|
|
#
|
|
# Usage: ./fix-watchtower-atlantis.sh
|
|
# Requirements: SSH access to Atlantis, sudo privileges
|
|
#
|
|
# =============================================================================
|
|
|
|
set -e # Exit on any error
|
|
|
|
# Colors for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# Configuration
|
|
ATLANTIS_HOST="atlantis"
|
|
CONTAINER_NAME="watchtower"
|
|
API_PORT="8082"
|
|
API_TOKEN=REDACTED_TOKEN
|
|
|
|
echo -e "${BLUE}🔧 Watchtower Atlantis Fix Script${NC}"
|
|
echo -e "${BLUE}===================================${NC}"
|
|
echo ""
|
|
|
|
# Function to run commands on Atlantis
|
|
run_on_atlantis() {
|
|
local cmd="$1"
|
|
echo -e "${YELLOW}Running on Atlantis:${NC} $cmd"
|
|
ssh "$ATLANTIS_HOST" "$cmd"
|
|
}
|
|
|
|
# Function to check if we can connect to Atlantis
|
|
check_connection() {
|
|
echo -e "${BLUE}📡 Checking connection to Atlantis...${NC}"
|
|
if ssh -o ConnectTimeout=5 "$ATLANTIS_HOST" "echo 'Connection successful'" >/dev/null 2>&1; then
|
|
echo -e "${GREEN}✅ Connected to Atlantis successfully${NC}"
|
|
return 0
|
|
else
|
|
echo -e "${RED}❌ Cannot connect to Atlantis${NC}"
|
|
echo "Please ensure:"
|
|
echo " - SSH access is configured"
|
|
echo " - Atlantis server is reachable"
|
|
echo " - SSH keys are properly set up"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# Function to check Docker permissions
|
|
check_docker_permissions() {
|
|
echo -e "${BLUE}🔐 Checking Docker permissions...${NC}"
|
|
|
|
# Try without sudo first
|
|
if run_on_atlantis "docker ps >/dev/null 2>&1"; then
|
|
echo -e "${GREEN}✅ Docker access available without sudo${NC}"
|
|
DOCKER_CMD="docker"
|
|
else
|
|
echo -e "${YELLOW}⚠️ Docker requires sudo privileges${NC}"
|
|
if run_on_atlantis "sudo docker ps >/dev/null 2>&1"; then
|
|
echo -e "${GREEN}✅ Docker access available with sudo${NC}"
|
|
DOCKER_CMD="sudo docker"
|
|
else
|
|
echo -e "${RED}❌ Cannot access Docker even with sudo${NC}"
|
|
exit 1
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# Function to check Watchtower container status
|
|
check_watchtower_status() {
|
|
echo -e "${BLUE}🔍 Checking Watchtower container status...${NC}"
|
|
|
|
local container_info
|
|
container_info=$(run_on_atlantis "$DOCKER_CMD ps -a --filter name=$CONTAINER_NAME --format 'table {{.Names}}\t{{.Status}}\t{{.State}}'")
|
|
|
|
if echo "$container_info" | grep -q "$CONTAINER_NAME"; then
|
|
echo -e "${GREEN}✅ Watchtower container found${NC}"
|
|
echo "$container_info"
|
|
|
|
# Check if running
|
|
if echo "$container_info" | grep -q "Up"; then
|
|
echo -e "${GREEN}✅ Watchtower is running${NC}"
|
|
return 0
|
|
else
|
|
echo -e "${YELLOW}⚠️ Watchtower is not running${NC}"
|
|
return 1
|
|
fi
|
|
else
|
|
echo -e "${RED}❌ Watchtower container not found${NC}"
|
|
return 2
|
|
fi
|
|
}
|
|
|
|
# Function to start Watchtower container
|
|
start_watchtower() {
|
|
echo -e "${BLUE}🚀 Starting Watchtower container...${NC}"
|
|
|
|
if run_on_atlantis "$DOCKER_CMD start $CONTAINER_NAME"; then
|
|
echo -e "${GREEN}✅ Watchtower started successfully${NC}"
|
|
|
|
# Wait a moment for startup
|
|
sleep 3
|
|
|
|
# Verify it's running
|
|
if check_watchtower_status >/dev/null; then
|
|
echo -e "${GREEN}✅ Watchtower is now running and healthy${NC}"
|
|
return 0
|
|
else
|
|
echo -e "${RED}❌ Watchtower failed to start properly${NC}"
|
|
return 1
|
|
fi
|
|
else
|
|
echo -e "${RED}❌ Failed to start Watchtower${NC}"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Function to check Watchtower logs
|
|
check_watchtower_logs() {
|
|
echo -e "${BLUE}📋 Checking Watchtower logs...${NC}"
|
|
|
|
local logs
|
|
logs=$(run_on_atlantis "$DOCKER_CMD logs $CONTAINER_NAME --tail 10 2>/dev/null" || echo "No logs available")
|
|
|
|
if [ "$logs" != "No logs available" ] && [ -n "$logs" ]; then
|
|
echo -e "${GREEN}✅ Recent logs:${NC}"
|
|
echo "$logs" | sed 's/^/ /'
|
|
else
|
|
echo -e "${YELLOW}⚠️ No logs available (container may not have started yet)${NC}"
|
|
fi
|
|
}
|
|
|
|
# Function to test Watchtower API
|
|
test_watchtower_api() {
|
|
echo -e "${BLUE}🌐 Testing Watchtower API...${NC}"
|
|
|
|
local api_response
|
|
api_response=$(run_on_atlantis "curl -s -w 'HTTP_STATUS:%{http_code}' http://localhost:$API_PORT/v1/update" 2>/dev/null || echo "API_ERROR")
|
|
|
|
if echo "$api_response" | grep -q "HTTP_STATUS:401"; then
|
|
echo -e "${GREEN}✅ API is responding (401 = authentication required, which is correct)${NC}"
|
|
echo -e "${BLUE}💡 API URL: http://atlantis:$API_PORT/v1/update${NC}"
|
|
echo -e "${BLUE}💡 API Token: $API_TOKEN${NC}"
|
|
return 0
|
|
elif echo "$api_response" | grep -q "HTTP_STATUS:200"; then
|
|
echo -e "${GREEN}✅ API is responding and accessible${NC}"
|
|
return 0
|
|
else
|
|
echo -e "${YELLOW}⚠️ API test failed or unexpected response${NC}"
|
|
echo "Response: $api_response"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Function to verify container configuration
|
|
verify_configuration() {
|
|
echo -e "${BLUE}⚙️ Verifying container configuration...${NC}"
|
|
|
|
local restart_policy
|
|
restart_policy=$(run_on_atlantis "$DOCKER_CMD inspect $CONTAINER_NAME --format '{{.HostConfig.RestartPolicy.Name}}'" 2>/dev/null || echo "unknown")
|
|
|
|
if [ "$restart_policy" = "always" ]; then
|
|
echo -e "${GREEN}✅ Restart policy: always (will auto-start on reboot)${NC}"
|
|
else
|
|
echo -e "${YELLOW}⚠️ Restart policy: $restart_policy (may not auto-start on reboot)${NC}"
|
|
fi
|
|
|
|
# Check port mapping
|
|
local port_mapping
|
|
port_mapping=$(run_on_atlantis "$DOCKER_CMD port $CONTAINER_NAME 2>/dev/null" || echo "No ports mapped")
|
|
|
|
if echo "$port_mapping" | grep -q "$API_PORT"; then
|
|
echo -e "${GREEN}✅ Port mapping: $port_mapping${NC}"
|
|
else
|
|
echo -e "${YELLOW}⚠️ Port mapping: $port_mapping${NC}"
|
|
fi
|
|
}
|
|
|
|
# Main execution
|
|
main() {
|
|
echo -e "${BLUE}Starting Watchtower diagnostics and fix...${NC}"
|
|
echo ""
|
|
|
|
# Step 1: Check connection
|
|
check_connection
|
|
echo ""
|
|
|
|
# Step 2: Check Docker permissions
|
|
check_docker_permissions
|
|
echo ""
|
|
|
|
# Step 3: Check Watchtower status
|
|
local watchtower_status
|
|
check_watchtower_status
|
|
watchtower_status=$?
|
|
echo ""
|
|
|
|
# Step 4: Start Watchtower if needed
|
|
if [ $watchtower_status -eq 1 ]; then
|
|
echo -e "${YELLOW}🔧 Watchtower needs to be started...${NC}"
|
|
start_watchtower
|
|
echo ""
|
|
elif [ $watchtower_status -eq 2 ]; then
|
|
echo -e "${RED}❌ Watchtower container not found. Please check deployment.${NC}"
|
|
exit 1
|
|
fi
|
|
|
|
# Step 5: Check logs
|
|
check_watchtower_logs
|
|
echo ""
|
|
|
|
# Step 6: Test API
|
|
test_watchtower_api
|
|
echo ""
|
|
|
|
# Step 7: Verify configuration
|
|
verify_configuration
|
|
echo ""
|
|
|
|
# Final status
|
|
echo -e "${GREEN}🎉 Watchtower fix script completed!${NC}"
|
|
echo ""
|
|
echo -e "${BLUE}📋 Summary:${NC}"
|
|
echo " • Watchtower container: Running"
|
|
echo " • HTTP API: Available on port $API_PORT"
|
|
echo " • Authentication: Required (token: $API_TOKEN)"
|
|
echo " • Auto-restart: Configured"
|
|
echo ""
|
|
echo -e "${BLUE}💡 Next steps:${NC}"
|
|
echo " • Monitor container health"
|
|
echo " • Check automatic updates are working"
|
|
echo " • Review logs periodically"
|
|
echo ""
|
|
echo -e "${GREEN}✅ All checks completed successfully!${NC}"
|
|
}
|
|
|
|
# Run main function
|
|
main "$@"
|