homelab-optimized/scripts/fix-watchtower-notifications.sh

#!/bin/bash
# Fix Watchtower Notification Issues (CORRECTED VERSION)
# This script ONLY fixes the HTTPS/HTTP notification protocol mismatch
# It does NOT touch Docker socket permissions (which are required for Watchtower to work)

set -e

echo "🔧 Watchtower Notification Fix Script"
echo "====================================="
echo "⚠️  This script ONLY fixes notification issues"
echo "⚠️  It does NOT change Docker socket permissions (those are required!)"
echo

# Check if running as root/sudo
if [[ $EUID -ne 0 ]]; then
   echo "❌ This script must be run as root or with sudo"
   exit 1
fi

# Check if watchtower container exists
if ! docker ps -a --format '{{.Names}}' | grep -q "^watchtower$"; then
    echo "❌ Watchtower container not found"
    exit 1
fi

echo "📋 Current Watchtower Status:"
echo "----------------------------"
echo "Container Status: $(docker ps --format '{{.Status}}' --filter name=watchtower)"
echo "Image: $(docker inspect watchtower | jq -r '.[0].Config.Image')"
echo

echo "🔍 Checking Notification Configuration:"
echo "--------------------------------------"

# Check current notification URL
CURRENT_NOTIFICATION=$(docker inspect watchtower | jq -r '.[0].Config.Env[] | select(contains("NOTIFICATION_URL")) // "Not found"')
echo "Current notification URL: $CURRENT_NOTIFICATION"

# Check recent logs for notification errors
echo
echo "📋 Recent Notification Errors:"
echo "------------------------------"
docker logs watchtower --since 24h 2>/dev/null | grep -i "notification\|ntfy" | tail -5 || echo "No recent notification logs found"

echo
echo "🔍 Issues Identified:"
echo "--------------------"

NEEDS_FIX=false

# Check for HTTPS/HTTP mismatch
if docker logs watchtower --since 24h 2>/dev/null | grep -q "http: server gave HTTP response to HTTPS client"; then
    echo "⚠️  HTTPS/HTTP protocol mismatch detected"
    echo "   Current: https://192.168.0.210:8081/updates"
    echo "   Should be: http://192.168.0.210:8081/updates"
    NEEDS_FIX=true
fi

# Check if notification URL is configured
if [[ "$CURRENT_NOTIFICATION" == "Not found" ]]; then
    echo "ℹ️  No notification URL environment variable found"
    echo "   (URL might be configured via command line arguments)"
fi

echo
if [[ "$NEEDS_FIX" == "true" ]]; then
    echo "🚨 NOTIFICATION ISSUE CONFIRMED"
    echo "The notification system is trying to use HTTPS but the server expects HTTP"
    echo

    # Check if we're in a compose stack
    NETWORK_NAME=$(docker inspect watchtower | jq -r '.[0].NetworkSettings.Networks | keys[0]')
    if [[ "$NETWORK_NAME" == *"stack"* ]]; then
        echo "📝 RECOMMENDED ACTION (Docker Compose Stack):"
        echo "Since Watchtower is part of a Compose stack, you should:"
        echo "1. Find and edit the docker-compose.yml file"
        echo "2. Update the notification URL environment variable:"
        echo "   environment:"
        echo "     - WATCHTOWER_NOTIFICATION_URL=http://192.168.0.210:8081/updates"
        echo "3. Recreate the stack:"
        echo "   docker-compose down && docker-compose up -d"
        echo
        echo "🔍 Looking for compose files..."

        # Try to find the compose file
        find /opt -name "*.yml" -o -name "*.yaml" 2>/dev/null | xargs grep -l "watchtower" 2>/dev/null | head -3 || echo "Compose files not found in /opt"

    else
        echo "🔧 AUTOMATIC FIX AVAILABLE"
        echo "Would you like to fix the notification URL? (y/N)"
        read -r response

        if [[ "$response" =~ ^[Yy]$ ]]; then
            echo "🔄 Stopping Watchtower..."
            docker stop watchtower

            echo "🗑️  Removing old container..."
            docker rm watchtower

            echo "🚀 Creating new Watchtower with corrected notification URL..."
            docker run -d \
                --name watchtower \
                --restart unless-stopped \
                -v /var/run/docker.sock:/var/run/docker.sock \
                -e TZ=America/Los_Angeles \
                -e WATCHTOWER_CLEANUP=true \
                -e WATCHTOWER_HTTP_API_UPDATE=true \
                -e WATCHTOWER_HTTP_API_TOKEN="REDACTED_HTTP_TOKEN" \
                -e WATCHTOWER_HTTP_API_METRICS=true \
                -e WATCHTOWER_SCHEDULE="0 0 4 * * *" \
                -e WATCHTOWER_NOTIFICATION_URL=http://192.168.0.210:8081/updates \
                -e WATCHTOWER_NOTIFICATIONS=shoutrrr \
                -p 8091:8080 \
                containrrr/watchtower:latest

            echo "✅ Watchtower recreated with corrected notification URL"
            echo "🔍 Verifying fix..."

            sleep 3
            if docker ps --format '{{.Names}}' | grep -q watchtower; then
                echo "✅ Watchtower is running"

                # Test notification
                echo "🧪 Testing notification (this may take a moment)..."
                curl -s -H "Authorization: Bearer watchtower-update-token" \
                     -X POST http://localhost:8091/v1/update >/dev/null 2>&1 || echo "API test completed"

                sleep 2
                if docker logs watchtower --since 30s 2>/dev/null | grep -q "HTTP response to HTTPS client"; then
                    echo "❌ Notification issue still present"
                else
                    echo "✅ Notification issue appears to be resolved"
                fi
            else
                echo "❌ Watchtower failed to start"
            fi
        else
            echo "⏭️  Skipping automatic fix"
        fi
    fi
else
    echo "✅ No notification issues detected"
fi

echo
echo "📊 Final Status Check:"
echo "---------------------"
if docker ps --format '{{.Names}}\t{{.Status}}' | grep watchtower; then
    echo "✅ Watchtower is running"

    echo
    echo "🔧 How to manually trigger updates:"
    echo "curl -H \"Authorization: Bearer watchtower-update-token\" \\"
    echo "     -X POST http://localhost:8091/v1/update"
else
    echo "❌ Watchtower is not running"
fi

echo
echo "⚠️  IMPORTANT SECURITY NOTE:"
echo "This script does NOT change Docker socket permissions."
echo "Watchtower REQUIRES read-write access to the Docker socket to:"
echo "- Pull new images"
echo "- Stop and start containers"
echo "- Remove old containers and images"
echo "Making the socket read-only would BREAK Watchtower completely."

echo
echo "🔗 For more information, see:"
echo "   docs/WATCHTOWER_SECURITY_ANALYSIS.md"
echo
echo "✅ Notification fix complete"