#!/bin/bash # Emergency Watchtower Fix via Portainer API # Stops crash looping containers and recreates them with correct config API_KEY=REDACTED_API_KEY BASE_URL="http://vishinator.synology.me:10000" echo "🚨 EMERGENCY FIX: Stopping Watchtower crash loops via Portainer API" echo "==================================================================" # Function to fix Watchtower on an endpoint fix_watchtower() { local endpoint_id=$1 local endpoint_name=$2 echo "" echo "🔧 Fixing Watchtower on: $endpoint_name (ID: $endpoint_id)" echo "--------------------------------------------------------" # Get Watchtower container ID container_info=$(curl -s -H "X-API-Key: $API_KEY" \ "$BASE_URL/api/endpoints/$endpoint_id/docker/containers/json?all=true" | \ jq -r '.[] | select(.Names[]? | contains("watchtower")) | "\(.Id) \(.Names[0])"') if [ -z "$container_info" ]; then echo "❌ No Watchtower container found" return 1 fi read container_id container_name <<< "$container_info" echo "📍 Found container: $container_name ($container_id)" # Stop the container echo "🛑 Stopping crash looping container..." curl -s -X POST -H "X-API-Key: $API_KEY" \ "$BASE_URL/api/endpoints/$endpoint_id/docker/containers/$container_id/stop" sleep 2 # Remove the container echo "🗑️ Removing broken container..." curl -s -X DELETE -H "X-API-Key: $API_KEY" \ "$BASE_URL/api/endpoints/$endpoint_id/docker/containers/$container_id?force=true" sleep 2 # Create new container with correct notification URL echo "🔄 Creating new Watchtower with fixed notification URL..." # Determine the correct notification URL based on endpoint if [ "$endpoint_name" = "Atlantis" ]; then NOTIFICATION_URL="ntfy://localhost:8081/updates?insecure=yes" elif [ "$endpoint_name" = "Calypso" ]; then NOTIFICATION_URL="ntfy://localhost:8081/updates?insecure=yes" else NOTIFICATION_URL="ntfy://ntfy.vish.gg/REDACTED_NTFY_TOPIC" fi # Create container via API create_response=$(curl -s -X POST -H "X-API-Key: $API_KEY" \ -H "Content-Type: application/json" \ "$BASE_URL/api/endpoints/$endpoint_id/docker/containers/create?name=watchtower" \ -d "{ \"Image\": \"containrrr/watchtower:latest\", \"Env\": [ \"WATCHTOWER_CLEANUP=true\", \"WATCHTOWER_INCLUDE_RESTARTING=true\", \"WATCHTOWER_INCLUDE_STOPPED=true\", \"WATCHTOWER_REVIVE_STOPPED=false\", \"WATCHTOWER_POLL_INTERVAL=3600\", \"WATCHTOWER_TIMEOUT=10s\", \"WATCHTOWER_HTTP_API_UPDATE=true\", \"WATCHTOWER_HTTP_API_TOKEN="REDACTED_HTTP_TOKEN"\", \"WATCHTOWER_NOTIFICATIONS=shoutrrr\", \"WATCHTOWER_NOTIFICATION_URL=$NOTIFICATION_URL\", \"TZ=America/Los_Angeles\" ], \"HostConfig\": { \"Binds\": [\"/var/run/docker.sock:/var/run/docker.sock\"], \"RestartPolicy\": {\"Name\": \"always\"}, \"PortBindings\": {\"8080/tcp\": [{\"HostPort\": \"8080\"}]} } }") new_container_id=$(echo "$create_response" | jq -r '.Id') if [ "$new_container_id" != "null" ] && [ -n "$new_container_id" ]; then echo "✅ Created new container: $new_container_id" # Start the container echo "▶️ Starting new Watchtower container..." curl -s -X POST -H "X-API-Key: $API_KEY" \ "$BASE_URL/api/endpoints/$endpoint_id/docker/containers/$new_container_id/start" sleep 3 # Check if it's running status=$(curl -s -H "X-API-Key: $API_KEY" \ "$BASE_URL/api/endpoints/$endpoint_id/docker/containers/$new_container_id/json" | \ jq -r '.State.Status') if [ "$status" = "running" ]; then echo "🟢 SUCCESS: Watchtower is now running!" # Get recent logs to verify no errors echo "📋 Checking logs for errors..." curl -s -H "X-API-Key: $API_KEY" \ "$BASE_URL/api/endpoints/$endpoint_id/docker/containers/$new_container_id/logs?stdout=true&stderr=true&tail=5" | \ sed 's/^.......//g' | sed 's/^/ /' else echo "🔴 FAILED: Container status is $status" fi else echo "🔴 FAILED: Could not create new container" echo "Response: $create_response" fi } # Fix Atlantis (ID: 2) fix_watchtower 2 "Atlantis" # Fix Calypso (ID: 443397) fix_watchtower 443397 "Calypso" echo "" echo "==================================================================" echo "🎯 Emergency fix complete! Run status check to verify:" echo " ./scripts/check-watchtower-status.sh" echo "=================================================================="