diff --git a/src/feed_handler.py b/src/feed_handler.py index 8f93757..954de35 100644 --- a/src/feed_handler.py +++ b/src/feed_handler.py @@ -1,60 +1,101 @@ import time from typing import Tuple -from urllib.error import URLError import feedparser +import requests from simple_logger_handler import setup_logger logger = setup_logger(__name__) -def grab_latest_chapter_information(id: str, max_retries: int = 3) -> Tuple[int | None, str, str]: +def get_rss_feed_data(rss_feed_url: str, max_retries: int = 3) -> feedparser.FeedParserDict: """ - Fetches the latest chapter information from a Royalroad RSS feed, with retries on network-related errors. + Fetches information from any RSS feed, with retries on network-related errors. Parameters: - id: Royalroad story ID as a string. + rss_feed_url: URL of the RSS feed max_retries: Number of retry attempts if fetching the feed fails due to network issues. Returns: - A tuple: (chapter_number, chapter_link, story_title) + feedparser.FeedParserDict Raises: ValueError: If the feed has no entries. - Other network-related exceptions: If fetching fails after retries. + requests.RequestException: If fetching fails after retries. """ - - rss_feed_url = f"https://www.royalroad.com/fiction/syndication/{id}" - attempt = 1 - - while attempt <= max_retries: - logger.debug(f"[Feed] Parsing feed URL: {rss_feed_url} (attempt {attempt}/{max_retries})") + for attempt in range(1, max_retries + 1): + logger.debug(f"[Feed] Fetching feed URL: {rss_feed_url} (attempt {attempt}/{max_retries})") try: - feed = feedparser.parse(rss_feed_url) + response = requests.get(rss_feed_url, timeout=10) + response.raise_for_status() + break - if not feed.entries: - raise ValueError(f"No entries found for feed {id}") - - latest_chapter_data = feed["entries"][0] - chapter_link = latest_chapter_data["link"] - story_title = feed["feed"]["title"] - chapter_title_with_story_title = latest_chapter_data["title"] - chapter_title = chapter_title_with_story_title.replace(story_title, "")[3:] # " - " is left after replace - chapter_id = int(latest_chapter_data["id"]) - - logger.info(f"[Feed] Latest chapter for story '{story_title}': {chapter_title}") - logger.debug(f"[Feed] Latest chapter for story '{story_title}' (ID {id}): {chapter_title} (ID {chapter_id}) -> {chapter_link}") - return chapter_title, chapter_link, story_title, chapter_id - - except (URLError, OSError) as e: - logger.warning(f"[Feed] Network error on attempt {attempt} for feed {id}: {e}") + except requests.RequestException as e: + logger.warning(f"[Feed] Network error on attempt {attempt} for feed {rss_feed_url}: {e}") if attempt == max_retries: - logger.error(f"[Feed] All {max_retries} attempts failed for feed {id}") + logger.error(f"[Feed] All {max_retries} attempts failed for feed {rss_feed_url}") raise backoff = 2 ** (attempt - 1) logger.debug(f"[Feed] Retrying in {backoff} seconds...") time.sleep(backoff) - attempt += 1 + + feed = feedparser.parse(response.content) + + # Not retried: an empty feed isn't a transient network error + if not feed.entries: + raise ValueError(f"No entries found for feed {rss_feed_url}") + + logger.debug(f"[Feed] Parsed {len(feed.entries)} entries from {rss_feed_url}") + return feed + +def grab_gitea_release_information(rss_feed_url: str) -> Tuple[str, str]: + """ + Fetches the latest release information from a Gitea releases RSS feed. + + Returns: + A tuple: (release_title, release_link) + + Raises: + ValueError: If the feed has no entries. + """ + feed = get_rss_feed_data(rss_feed_url) + entry = feed.entries[0] + title = entry["title"] + link = entry["link"] + + logger.info(f"[Feed] Latest release for feed '{rss_feed_url}': {title}") + logger.debug(f"[Feed] Latest release for feed '{rss_feed_url}': {title} -> {link}") + return title, link + +def grab_royal_road_chapter_information(rss_feed_url: str) -> Tuple[str, str, str, str]: + """ + Fetches the latest chapter information from a Royalroad RSS feed. + + Parameters: + rss_feed_url: URL of the Royalroad story's RSS feed. + + Returns: + A tuple: (chapter_title, chapter_link, story_title, chapter_id) + + Raises: + ValueError: If the feed has no entries. + """ + feed = get_rss_feed_data(rss_feed_url) + latest_chapter_data = feed.entries[0] + chapter_link = latest_chapter_data["link"] + story_title = feed.feed.title + chapter_title_with_story_title = latest_chapter_data["title"] + chapter_title = chapter_title_with_story_title.removeprefix(story_title).removeprefix(" - ") + chapter_id = str(latest_chapter_data["id"]) # str so it compares equal to the DB value + + logger.info(f"[Feed] Latest chapter for story '{story_title}': {chapter_title}") + logger.debug(f"[Feed] Latest chapter for story '{story_title}' (RSS URL {rss_feed_url}): {chapter_title} (ID {chapter_id}) -> {chapter_link}") + return chapter_title, chapter_link, story_title, chapter_id if __name__ == "__main__": - print(grab_latest_chapter_information("118891")) \ No newline at end of file + #rsssss="https://gitea.com/gitea/runner/releases.rss" + #print(get_rss_feed_data(rsssss)) + #print(grab_gitea_release_information(rsssss)) + id="118891" + rss_feed_url = f"https://www.royalroad.com/fiction/syndication/{id}" + print(grab_royal_road_chapter_information(rss_feed_url)) \ No newline at end of file diff --git a/src/main.py b/src/main.py index 6ac3dbc..6aa2106 100644 --- a/src/main.py +++ b/src/main.py @@ -5,8 +5,8 @@ import uvicorn from fastapi import Depends, FastAPI, HTTPException, Request from db_module import close_connection_pool, create_connection_pool, get_db, start_healthcheck_thread -from feed_handler import grab_latest_chapter_information -from metrics_server import REQUEST_COUNTER +from feed_handler import grab_gitea_release_information, grab_royal_road_chapter_information +from metrics_server import REQUEST_COUNTER, metrics_api from secret_manager import cleanup_secret_files from send_notification import send_notification from simple_logger_handler import LOG_LEVEL, setup_logger @@ -60,59 +60,138 @@ async def prometheus_middleware(request: Request, call_next): logger.debug(f"[Metrics] Counter incremented for {request.method} {request.url.path} [{status}]") return response +@api.get("/gitea") +def get_release_information(db = Depends(get_db)): + """ + Checks for new Gitea releases and updates the database accordingly. + + Behaviour: + - Fetches all active Gitea feeds and their last known release link. + - Queries each feed for its latest release. + - Updates `last_entry` if the latest release link differs from the stored one. + - Sends a notification for each changed release. + - A failure on one feed is logged and the remaining feeds are still checked. + + Returns: + A JSON object with checked/new/error counts. If any feed failed, + a 500 is returned with the same counts as the detail. + """ + logger.info("[FastAPI] Checking for new Gitea releases") + results = {"checked": 0, "new": 0, "errors": 0} + with db.cursor() as cursor: + cursor.execute( + "SELECT id, rss_feed, feed_name, last_entry " + "FROM rss_feeds WHERE active = 1 AND type = %s", + ("gitea",), + ) + feeds = cursor.fetchall() + logger.debug(f"[FastAPI] Found {len(feeds)} active Gitea feeds to check.") + + for feed_id, rss_feed, feed_name, last_entry in feeds: + results["checked"] += 1 + try: + title, link = grab_gitea_release_information(rss_feed) + logger.debug(f"[FastAPI] ID:{feed_id},RSS Feed {rss_feed}: db={last_entry}, online={link}") + + if link == last_entry: + continue + + logger.info(f"[FastAPI] Release change for ID {feed_id}: {title}") + send_notification(feed_name, title, link) + cursor.execute( + "UPDATE rss_feeds SET last_entry = %s WHERE id = %s", + (link, feed_id), + ) + db.commit() + logger.debug(f"[FastAPI] Notification sent and database updated for Gitea feed ID {feed_id}") + results["new"] += 1 + + except Exception as e: + db.rollback() + results["errors"] += 1 + logger.error( + f"[FastAPI] Failed for feed ID {feed_id} ({rss_feed}): {e}", + exc_info=True, + ) + + if results["errors"]: + logger.warning(f"[FastAPI] Gitea check finished with errors: {results}") + raise HTTPException(status_code=500, detail=results) + + logger.info(f"[FastAPI] Gitea check complete: {results}") + return results + @api.get("/royalroad") -def get_chapters(request: Request, db = Depends(get_db)): +def get_chapters(db = Depends(get_db)): """ Checks for new Royalroad chapters and updates the database accordingly. Behaviour: - - Fetches all active stories and their last known chapter number. + - Fetches all active stories and their last known chapter ID. - Queries Royalroad for each story's latest chapter. - - Updates `lastChapter` if a new chapter is found. + - Updates `last_entry` if a new chapter is found. - Sends a notification for any newly discovered chapters. + - A failure on one story is logged and the remaining stories are still checked. Returns: - A JSON object indicating the check status. + A JSON object with checked/new/error counts. If any story failed, + a 500 is returned with the same counts as the detail. """ - logger.info("[Royalroad] Checking for new chapters...") - try: - cursor = db.cursor() - cursor.execute("SELECT id, royalroadId, lastChapter FROM stories WHERE active=1") + logger.info("[FastAPI] Checking for new Royalroad chapters") + results = {"checked": 0, "new": 0, "errors": 0} + with db.cursor() as cursor: + cursor.execute( + "SELECT id, rss_feed, last_entry " + "FROM rss_feeds WHERE active = 1 AND type = %s", + ("royalroad",), + ) + feeds = cursor.fetchall() + logger.debug(f"[FastAPI] Found {len(feeds)} active stories to check.") - stories = cursor.fetchall() - logger.debug(f"[Royalroad] Found {len(stories)} active stories to check.") + for feed_id, rss_feed, last_entry in feeds: + results["checked"] += 1 + try: + chapter_title, link, story_title, online_latest_entry = grab_royal_road_chapter_information(rss_feed) + logger.debug(f"[FastAPI] ID:{feed_id},RSS Feed {rss_feed}: db={last_entry}, online={online_latest_entry}") - for id, royalroadId, last_chapter_db in stories: - chapter_title, chapter_link, story_title, chapter_id = grab_latest_chapter_information(royalroadId) - logger.debug(f"[Royalroad] Story {id}: last={last_chapter_db}, latest={chapter_id}") + if online_latest_entry == last_entry: + continue - if chapter_id > last_chapter_db: - logger.info(f"[Royalroad] New chapter detected for story ID {id}: {story_title}") - cursor.execute("UPDATE stories SET lastChapter = %s WHERE id = %s", (chapter_id, id)) + logger.info(f"[FastAPI] New chapter detected for story ID {feed_id}: {story_title}") + send_notification(story_title, chapter_title, link) + cursor.execute( + "UPDATE rss_feeds SET last_entry = %s WHERE id = %s", + (online_latest_entry, feed_id), + ) db.commit() - send_notification(story_title, chapter_title, chapter_link) - logger.debug(f"[Royalroad] Notification sent for story ID {id}") + logger.debug(f"[FastAPI] Notification sent and database updated for story ID {feed_id}") + results["new"] += 1 - logger.info("[Royalroad] Chapter check completed successfully.") - return {"status": "checked"} - except ValueError as e: - logger.error(f"[Royalroad] Failed to fetch feed {royalroadId}: {e}") - raise HTTPException(status_code=404, detail=str(e)) from e + except Exception as e: + db.rollback() + results["errors"] += 1 + logger.error( + f"[Royalroad] Failed for feed ID {feed_id} ({rss_feed}): {e}", + exc_info=True, + ) - except Exception as e: - logger.error(f"[Royalroad] Error during chapter check: {e}", exc_info=True) - raise HTTPException(status_code=500, detail=str(e)) from e + if results["errors"]: + logger.warning(f"[FastAPI] Royalroad check finished with errors: {results}") + raise HTTPException(status_code=500, detail=results) + + logger.info(f"[FastAPI] Royalroad check complete: {results}") + return results async def start_servers(): logger.info("[Server] Starting main API (port 5000) and metrics server (port 9000)...") - config_main = uvicorn.Config("main:api", host="0.0.0.0", port=5000, log_level=LOG_LEVEL.lower(), log_config=LOGGING_CONFIG) - config_metrics = uvicorn.Config("metrics_server:metrics_api", host="0.0.0.0", port=9000, log_level=LOG_LEVEL.lower(), log_config=LOGGING_CONFIG) + config_main = uvicorn.Config(api, host="0.0.0.0", port=5000, log_level=LOG_LEVEL.lower(), log_config=LOGGING_CONFIG) + config_metrics = uvicorn.Config(metrics_api, host="0.0.0.0", port=9000, log_level=LOG_LEVEL.lower(), log_config=LOGGING_CONFIG) server_main = uvicorn.Server(config_main) server_metrics = uvicorn.Server(config_metrics) await asyncio.gather(server_main.serve(), server_metrics.serve()) - logger.info("[Server] Both servers started successfully.") + logger.info("[Server] Both servers have stopped.") if __name__ == "__main__": - asyncio.run(start_servers()) + asyncio.run(start_servers()) \ No newline at end of file