131 lines
4.5 KiB
Bash
Executable File
131 lines
4.5 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Emergency Watchtower Fix via Portainer API
|
|
# Stops crash looping containers and recreates them with correct config
|
|
|
|
API_KEY=REDACTED_API_KEY
|
|
BASE_URL="http://vishinator.synology.me:10000"
|
|
|
|
echo "🚨 EMERGENCY FIX: Stopping Watchtower crash loops via Portainer API"
|
|
echo "=================================================================="
|
|
|
|
# Function to fix Watchtower on an endpoint
|
|
fix_watchtower() {
|
|
local endpoint_id=$1
|
|
local endpoint_name=$2
|
|
|
|
echo ""
|
|
echo "🔧 Fixing Watchtower on: $endpoint_name (ID: $endpoint_id)"
|
|
echo "--------------------------------------------------------"
|
|
|
|
# Get Watchtower container ID
|
|
container_info=$(curl -s -H "X-API-Key: $API_KEY" \
|
|
"$BASE_URL/api/endpoints/$endpoint_id/docker/containers/json?all=true" | \
|
|
jq -r '.[] | select(.Names[]? | contains("watchtower")) | "\(.Id) \(.Names[0])"')
|
|
|
|
if [ -z "$container_info" ]; then
|
|
echo "❌ No Watchtower container found"
|
|
return 1
|
|
fi
|
|
|
|
read container_id container_name <<< "$container_info"
|
|
echo "📍 Found container: $container_name ($container_id)"
|
|
|
|
# Stop the container
|
|
echo "🛑 Stopping crash looping container..."
|
|
curl -s -X POST -H "X-API-Key: $API_KEY" \
|
|
"$BASE_URL/api/endpoints/$endpoint_id/docker/containers/$container_id/stop"
|
|
|
|
sleep 2
|
|
|
|
# Remove the container
|
|
echo "🗑️ Removing broken container..."
|
|
curl -s -X DELETE -H "X-API-Key: $API_KEY" \
|
|
"$BASE_URL/api/endpoints/$endpoint_id/docker/containers/$container_id?force=true"
|
|
|
|
sleep 2
|
|
|
|
# Create new container with correct notification URL
|
|
echo "🔄 Creating new Watchtower with fixed notification URL..."
|
|
|
|
# Determine the correct notification URL based on endpoint
|
|
if [ "$endpoint_name" = "Atlantis" ]; then
|
|
NOTIFICATION_URL="ntfy://localhost:8081/updates?insecure=yes"
|
|
elif [ "$endpoint_name" = "Calypso" ]; then
|
|
NOTIFICATION_URL="ntfy://localhost:8081/updates?insecure=yes"
|
|
else
|
|
NOTIFICATION_URL="ntfy://ntfy.vish.gg/REDACTED_NTFY_TOPIC"
|
|
fi
|
|
|
|
# Create container via API
|
|
create_response=$(curl -s -X POST -H "X-API-Key: $API_KEY" \
|
|
-H "Content-Type: application/json" \
|
|
"$BASE_URL/api/endpoints/$endpoint_id/docker/containers/create?name=watchtower" \
|
|
-d "{
|
|
\"Image\": \"containrrr/watchtower:latest\",
|
|
\"Env\": [
|
|
\"WATCHTOWER_CLEANUP=true\",
|
|
\"WATCHTOWER_INCLUDE_RESTARTING=true\",
|
|
\"WATCHTOWER_INCLUDE_STOPPED=true\",
|
|
\"WATCHTOWER_REVIVE_STOPPED=false\",
|
|
\"WATCHTOWER_POLL_INTERVAL=3600\",
|
|
\"WATCHTOWER_TIMEOUT=10s\",
|
|
\"WATCHTOWER_HTTP_API_UPDATE=true\",
|
|
\"WATCHTOWER_HTTP_API_TOKEN="REDACTED_HTTP_TOKEN"\",
|
|
\"WATCHTOWER_NOTIFICATIONS=shoutrrr\",
|
|
\"WATCHTOWER_NOTIFICATION_URL=$NOTIFICATION_URL\",
|
|
\"TZ=America/Los_Angeles\"
|
|
],
|
|
\"HostConfig\": {
|
|
\"Binds\": [\"/var/run/docker.sock:/var/run/docker.sock\"],
|
|
\"RestartPolicy\": {\"Name\": \"always\"},
|
|
\"PortBindings\": {\"8080/tcp\": [{\"HostPort\": \"8080\"}]}
|
|
}
|
|
}")
|
|
|
|
new_container_id=$(echo "$create_response" | jq -r '.Id')
|
|
|
|
if [ "$new_container_id" != "null" ] && [ -n "$new_container_id" ]; then
|
|
echo "✅ Created new container: $new_container_id"
|
|
|
|
# Start the container
|
|
echo "▶️ Starting new Watchtower container..."
|
|
curl -s -X POST -H "X-API-Key: $API_KEY" \
|
|
"$BASE_URL/api/endpoints/$endpoint_id/docker/containers/$new_container_id/start"
|
|
|
|
sleep 3
|
|
|
|
# Check if it's running
|
|
status=$(curl -s -H "X-API-Key: $API_KEY" \
|
|
"$BASE_URL/api/endpoints/$endpoint_id/docker/containers/$new_container_id/json" | \
|
|
jq -r '.State.Status')
|
|
|
|
if [ "$status" = "running" ]; then
|
|
echo "🟢 SUCCESS: Watchtower is now running!"
|
|
|
|
# Get recent logs to verify no errors
|
|
echo "📋 Checking logs for errors..."
|
|
curl -s -H "X-API-Key: $API_KEY" \
|
|
"$BASE_URL/api/endpoints/$endpoint_id/docker/containers/$new_container_id/logs?stdout=true&stderr=true&tail=5" | \
|
|
sed 's/^.......//g' | sed 's/^/ /'
|
|
else
|
|
echo "🔴 FAILED: Container status is $status"
|
|
fi
|
|
else
|
|
echo "🔴 FAILED: Could not create new container"
|
|
echo "Response: $create_response"
|
|
fi
|
|
}
|
|
|
|
# Fix Atlantis (ID: 2)
|
|
fix_watchtower 2 "Atlantis"
|
|
|
|
# Fix Calypso (ID: 443397)
|
|
fix_watchtower 443397 "Calypso"
|
|
|
|
echo ""
|
|
echo "=================================================================="
|
|
echo "🎯 Emergency fix complete! Run status check to verify:"
|
|
echo " ./scripts/check-watchtower-status.sh"
|
|
echo "=================================================================="
|