Sanitized mirror from private repository - 2026-04-18 11:19:59 UTC

scripts/portainer-emergency-fix.sh

@@ -0,0 +1,130 @@
+#!/bin/bash
+
+# Emergency Watchtower Fix via Portainer API
+# Stops crash looping containers and recreates them with correct config
+
+API_KEY=REDACTED_API_KEY
+BASE_URL="http://vishinator.synology.me:10000"
+
+echo "🚨 EMERGENCY FIX: Stopping Watchtower crash loops via Portainer API"
+echo "=================================================================="
+
+# Function to fix Watchtower on an endpoint
+fix_watchtower() {
+  local endpoint_id=$1
+  local endpoint_name=$2
+
+  echo ""
+  echo "🔧 Fixing Watchtower on: $endpoint_name (ID: $endpoint_id)"
+  echo "--------------------------------------------------------"
+
+  # Get Watchtower container ID
+  container_info=$(curl -s -H "X-API-Key: $API_KEY" \
+    "$BASE_URL/api/endpoints/$endpoint_id/docker/containers/json?all=true" | \
+    jq -r '.[] | select(.Names[]? | contains("watchtower")) | "\(.Id) \(.Names[0])"')
+
+  if [ -z "$container_info" ]; then
+    echo "❌ No Watchtower container found"
+    return 1
+  fi
+
+  read container_id container_name <<< "$container_info"
+  echo "📍 Found container: $container_name ($container_id)"
+
+  # Stop the container
+  echo "🛑 Stopping crash looping container..."
+  curl -s -X POST -H "X-API-Key: $API_KEY" \
+    "$BASE_URL/api/endpoints/$endpoint_id/docker/containers/$container_id/stop"
+
+  sleep 2
+
+  # Remove the container
+  echo "🗑️  Removing broken container..."
+  curl -s -X DELETE -H "X-API-Key: $API_KEY" \
+    "$BASE_URL/api/endpoints/$endpoint_id/docker/containers/$container_id?force=true"
+
+  sleep 2
+
+  # Create new container with correct notification URL
+  echo "🔄 Creating new Watchtower with fixed notification URL..."
+
+  # Determine the correct notification URL based on endpoint
+  if [ "$endpoint_name" = "Atlantis" ]; then
+    NOTIFICATION_URL="ntfy://localhost:8081/updates?insecure=yes"
+  elif [ "$endpoint_name" = "Calypso" ]; then
+    NOTIFICATION_URL="ntfy://localhost:8081/updates?insecure=yes"
+  else
+    NOTIFICATION_URL="ntfy://ntfy.vish.gg/REDACTED_NTFY_TOPIC"
+  fi
+
+  # Create container via API
+  create_response=$(curl -s -X POST -H "X-API-Key: $API_KEY" \
+    -H "Content-Type: application/json" \
+    "$BASE_URL/api/endpoints/$endpoint_id/docker/containers/create?name=watchtower" \
+    -d "{
+      \"Image\": \"containrrr/watchtower:latest\",
+      \"Env\": [
+        \"WATCHTOWER_CLEANUP=true\",
+        \"WATCHTOWER_INCLUDE_RESTARTING=true\",
+        \"WATCHTOWER_INCLUDE_STOPPED=true\",
+        \"WATCHTOWER_REVIVE_STOPPED=false\",
+        \"WATCHTOWER_POLL_INTERVAL=3600\",
+        \"WATCHTOWER_TIMEOUT=10s\",
+        \"WATCHTOWER_HTTP_API_UPDATE=true\",
+        \"WATCHTOWER_HTTP_API_TOKEN="REDACTED_HTTP_TOKEN"\",
+        \"WATCHTOWER_NOTIFICATIONS=shoutrrr\",
+        \"WATCHTOWER_NOTIFICATION_URL=$NOTIFICATION_URL\",
+        \"TZ=America/Los_Angeles\"
+      ],
+      \"HostConfig\": {
+        \"Binds\": [\"/var/run/docker.sock:/var/run/docker.sock\"],
+        \"RestartPolicy\": {\"Name\": \"always\"},
+        \"PortBindings\": {\"8080/tcp\": [{\"HostPort\": \"8080\"}]}
+      }
+    }")
+
+  new_container_id=$(echo "$create_response" | jq -r '.Id')
+
+  if [ "$new_container_id" != "null" ] && [ -n "$new_container_id" ]; then
+    echo "✅ Created new container: $new_container_id"
+
+    # Start the container
+    echo "▶️  Starting new Watchtower container..."
+    curl -s -X POST -H "X-API-Key: $API_KEY" \
+      "$BASE_URL/api/endpoints/$endpoint_id/docker/containers/$new_container_id/start"
+
+    sleep 3
+
+    # Check if it's running
+    status=$(curl -s -H "X-API-Key: $API_KEY" \
+      "$BASE_URL/api/endpoints/$endpoint_id/docker/containers/$new_container_id/json" | \
+      jq -r '.State.Status')
+
+    if [ "$status" = "running" ]; then
+      echo "🟢 SUCCESS: Watchtower is now running!"
+
+      # Get recent logs to verify no errors
+      echo "📋 Checking logs for errors..."
+      curl -s -H "X-API-Key: $API_KEY" \
+        "$BASE_URL/api/endpoints/$endpoint_id/docker/containers/$new_container_id/logs?stdout=true&stderr=true&tail=5" | \
+        sed 's/^.......//g' | sed 's/^/   /'
+    else
+      echo "🔴 FAILED: Container status is $status"
+    fi
+  else
+    echo "🔴 FAILED: Could not create new container"
+    echo "Response: $create_response"
+  fi
+}
+
+# Fix Atlantis (ID: 2)
+fix_watchtower 2 "Atlantis"
+
+# Fix Calypso (ID: 443397)
+fix_watchtower 443397 "Calypso"
+
+echo ""
+echo "=================================================================="
+echo "🎯 Emergency fix complete! Run status check to verify:"
+echo "   ./scripts/check-watchtower-status.sh"
+echo "=================================================================="