fix: graceful HTTP error handling and overlay timing improvements

Backend Changes:
- Updated fetch_website_content to gracefully handle non-200 responses
- Added specific error handling for HTTPStatusError (403, 404, etc.)
- Added timeout handling (30s) and connection error handling
- Returns descriptive error messages instead of crashing agent
- Allows agent to continue with other tools when one URL fails

Frontend Changes:
- Fixed spinner overlay blocking status updates
- Added HTMX handler to hide overlay when first status arrives
- Added MutationObserver in JS to monitor status updates
- Overlay now hides within ~1s, showing real-time progress
- Users can see status timeline while analysis runs

Benefits:
- Partial results when some URLs are blocked (e.g., 403 Forbidden)
- Better UX with visible progress indicators
- No more full-page blocking during analysis
- More resilient to external site failures

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>

src/backend/core/tools.py

@@ -18,14 +18,17 @@ async def fetch_website_content(
     url: str,
 ) -> str:
     """
-    Fetches the HTML content of the given URL via httpx and beautifulsoup
+    Fetches the HTML content of the given URL via httpx and beautifulsoup.
+    Returns error message on failure instead of raising exception.
 
     :param _ctx: RunContext[SwotAgentDeps]
     :param url: str
-    :return: str
+    :return: str - content or error message
     """
     logger.info(f"Fetching website content for: {url}")
-    async with httpx.AsyncClient(follow_redirects=True) as http_client:
+    async with httpx.AsyncClient(
+        follow_redirects=True, timeout=30.0
+    ) as http_client:
         try:
             response = await http_client.get(url)
             response.raise_for_status()
@@ -33,9 +36,29 @@ async def fetch_website_content(
             content = soup(html_content, "html.parser")
             text_content = content.get_text(separator=" ", strip=True)
             return text_content
-        except httpx.HTTPError as e:
-            logger.info(f"Request failed: {e}")
-            raise
+        except httpx.HTTPStatusError as e:
+            # Handle HTTP errors gracefully (403, 404, 500, etc.)
+            error_msg = (
+                f"Unable to fetch {url}: HTTP {e.response.status_code} "
+                f"({e.response.reason_phrase}). "
+                f"The site may be blocking automated requests or require authentication."
+            )
+            logger.warning(error_msg)
+            return error_msg
+        except httpx.TimeoutException:
+            error_msg = f"Request to {url} timed out after 30 seconds."
+            logger.warning(error_msg)
+            return error_msg
+        except httpx.RequestError as e:
+            # Handle connection errors, invalid URLs, etc.
+            error_msg = f"Failed to connect to {url}: {type(e).__name__}"
+            logger.warning(f"{error_msg} - {e}")
+            return error_msg
+        except Exception as e:
+            # Catch any other unexpected errors
+            error_msg = f"Unexpected error fetching {url}: {type(e).__name__}"
+            logger.error(f"{error_msg} - {e}")
+            return error_msg
 
 
 @swot_agent.tool(prepare=report_tool_usage)

src/frontend/static/js/app.js

@@ -188,6 +188,33 @@
         });
     }
 
+    /**
+     * Monitor for status updates arrival
+     */
+    function monitorStatusUpdates() {
+        const statusBox = document.getElementById('status');
+        if (!statusBox) return;
+
+        // Watch for first status update to hide overlay
+        const statusObserver = new MutationObserver(function(mutations) {
+            mutations.forEach(function(mutation) {
+                if (mutation.type === 'childList' && statusBox.innerHTML.trim().length > 0) {
+                    // First status update received - hide spinner overlay
+                    const spinner = document.getElementById('spinner');
+                    if (spinner && !spinner.classList.contains('is-hidden')) {
+                        spinner.classList.add('is-hidden');
+                        stopLoadingMessages();
+                    }
+                }
+            });
+        });
+
+        statusObserver.observe(statusBox, {
+            childList: true,
+            subtree: true
+        });
+    }
+
     /**
      * Monitor for analysis completion
      */
@@ -202,7 +229,7 @@
                     // Analysis complete
                     stopLoadingMessages();
 
-                    // Hide spinner
+                    // Hide spinner (just in case)
                     const spinner = document.getElementById('spinner');
                     if (spinner) {
                         spinner.classList.add('is-hidden');
@@ -277,6 +304,7 @@
      */
     function initialize() {
         initializeForm();
+        monitorStatusUpdates();
         monitorAnalysisCompletion();
         initializeButtonEffects();
         initializeSmoothScrolling();

src/frontend/templates/home.html

@@ -89,6 +89,12 @@
          hx-get={{ url_for('get_status') }}
          hx-trigger="load, every 1s"
          hx-swap="innerHTML"
+         hx-on:after-request="
+            if(this.innerHTML.trim().length > 0) {
+            const spinner = document.querySelector('#spinner');
+            if (spinner) spinner.classList.add('is-hidden');
+            }
+            "
          style="display: none">
     </div>
     <div class="box"