Feature/gitea release (#4)

Reviewed-on: #4
- Changed RSS feed parser to be generic
- Added Gitea RSS feed parser
- Changed source database table

Co-authored-by: Florian <flo@meergans.org>

src/feed_handler.py

@@ -1,60 +1,101 @@
 import time
 from typing import Tuple
-from urllib.error import URLError
 
 import feedparser
+import requests
 
 from simple_logger_handler import setup_logger
 
 logger = setup_logger(__name__)
 
-def grab_latest_chapter_information(id: str, max_retries: int = 3) -> Tuple[int | None, str, str]:
+def get_rss_feed_data(rss_feed_url: str, max_retries: int = 3) -> feedparser.FeedParserDict:
 	"""
-	Fetches the latest chapter information from a Royalroad RSS feed, with retries on network-related errors.
+	Fetches information from any RSS feed, with retries on network-related errors.
 
 	Parameters:
-		id: Royalroad story ID as a string.
+		rss_feed_url: URL of the RSS feed
 		max_retries: Number of retry attempts if fetching the feed fails due to network issues.
 
 	Returns:
-		A tuple: (chapter_number, chapter_link, story_title)
+		feedparser.FeedParserDict
 
 	Raises:
 		ValueError: If the feed has no entries.
-		Other network-related exceptions: If fetching fails after retries.
+		requests.RequestException: If fetching fails after retries.
 	"""
-
+	for attempt in range(1, max_retries + 1):
-	rss_feed_url = f"https://www.royalroad.com/fiction/syndication/{id}"
+		logger.debug(f"[Feed] Fetching feed URL: {rss_feed_url} (attempt {attempt}/{max_retries})")
-	attempt = 1
-
-	while attempt <= max_retries:
-		logger.debug(f"[Feed] Parsing feed URL: {rss_feed_url} (attempt {attempt}/{max_retries})")
 		try:
-			feed = feedparser.parse(rss_feed_url)
+			response = requests.get(rss_feed_url, timeout=10)
+			response.raise_for_status()
+			break
 
-			if not feed.entries:
+		except requests.RequestException as e:
-				raise ValueError(f"No entries found for feed {id}")
+			logger.warning(f"[Feed] Network error on attempt {attempt} for feed {rss_feed_url}: {e}")
-
-			latest_chapter_data = feed["entries"][0]
-			chapter_link = latest_chapter_data["link"]
-			story_title = feed["feed"]["title"]
-			chapter_title_with_story_title = latest_chapter_data["title"]
-			chapter_title = chapter_title_with_story_title.replace(story_title, "")[3:] # " - " is left after replace
-			chapter_id = int(latest_chapter_data["id"])
-
-			logger.info(f"[Feed] Latest chapter for story '{story_title}': {chapter_title}")
-			logger.debug(f"[Feed] Latest chapter for story '{story_title}' (ID {id}): {chapter_title} (ID {chapter_id}) -> {chapter_link}")
-			return chapter_title, chapter_link, story_title, chapter_id
-
-		except (URLError, OSError) as e:
-			logger.warning(f"[Feed] Network error on attempt {attempt} for feed {id}: {e}")
 			if attempt == max_retries:
-				logger.error(f"[Feed] All {max_retries} attempts failed for feed {id}")
+				logger.error(f"[Feed] All {max_retries} attempts failed for feed {rss_feed_url}")
 				raise
 			backoff = 2 ** (attempt - 1)
 			logger.debug(f"[Feed] Retrying in {backoff} seconds...")
 			time.sleep(backoff)
-			attempt += 1
+
+	feed = feedparser.parse(response.content)
+
+	# Not retried: an empty feed isn't a transient network error
+	if not feed.entries:
+		raise ValueError(f"No entries found for feed {rss_feed_url}")
+
+	logger.debug(f"[Feed] Parsed {len(feed.entries)} entries from {rss_feed_url}")
+	return feed
+
+def grab_gitea_release_information(rss_feed_url: str) -> Tuple[str, str]:
+	"""
+	Fetches the latest release information from a Gitea releases RSS feed.
+
+	Returns:
+		A tuple: (release_title, release_link)
+
+	Raises:
+		ValueError: If the feed has no entries.
+	"""
+	feed = get_rss_feed_data(rss_feed_url)
+	entry = feed.entries[0]
+	title = entry["title"]
+	link = entry["link"]
+
+	logger.info(f"[Feed] Latest release for feed '{rss_feed_url}': {title}")
+	logger.debug(f"[Feed] Latest release for feed '{rss_feed_url}': {title} -> {link}")
+	return title, link
+
+def grab_royal_road_chapter_information(rss_feed_url: str) -> Tuple[str, str, str, str]:
+	"""
+	Fetches the latest chapter information from a Royalroad RSS feed.
+
+	Parameters:
+		rss_feed_url: URL of the Royalroad story's RSS feed.
+
+	Returns:
+		A tuple: (chapter_title, chapter_link, story_title, chapter_id)
+
+	Raises:
+		ValueError: If the feed has no entries.
+	"""
+	feed = get_rss_feed_data(rss_feed_url)
+	latest_chapter_data = feed.entries[0]
+	chapter_link = latest_chapter_data["link"]
+	story_title = feed.feed.title
+	chapter_title_with_story_title = latest_chapter_data["title"]
+	chapter_title = chapter_title_with_story_title.removeprefix(story_title).removeprefix(" - ")
+	chapter_id = str(latest_chapter_data["id"])  # str so it compares equal to the DB value
+
+	logger.info(f"[Feed] Latest chapter for story '{story_title}': {chapter_title}")
+	logger.debug(f"[Feed] Latest chapter for story '{story_title}' (RSS URL {rss_feed_url}): {chapter_title} (ID {chapter_id}) -> {chapter_link}")
+	return chapter_title, chapter_link, story_title, chapter_id
 
 if __name__ == "__main__":
-	print(grab_latest_chapter_information("118891"))
+	#rsssss="https://gitea.com/gitea/runner/releases.rss"
+	#print(get_rss_feed_data(rsssss))
+	#print(grab_gitea_release_information(rsssss))
+	id="118891"
+	rss_feed_url = f"https://www.royalroad.com/fiction/syndication/{id}"
+	print(grab_royal_road_chapter_information(rss_feed_url))

src/main.py

@@ -5,8 +5,8 @@ import uvicorn
 from fastapi import Depends, FastAPI, HTTPException, Request
 
 from db_module import close_connection_pool, create_connection_pool, get_db, start_healthcheck_thread
-from feed_handler import grab_latest_chapter_information
+from feed_handler import grab_gitea_release_information, grab_royal_road_chapter_information
-from metrics_server import REQUEST_COUNTER
+from metrics_server import REQUEST_COUNTER, metrics_api
 from secret_manager import cleanup_secret_files
 from send_notification import send_notification
 from simple_logger_handler import LOG_LEVEL, setup_logger
@@ -60,59 +60,138 @@ async def prometheus_middleware(request: Request, call_next):
 		logger.debug(f"[Metrics] Counter incremented for {request.method} {request.url.path} [{status}]")
 	return response
 
+@api.get("/gitea")
+def get_release_information(db = Depends(get_db)):
+	"""
+	Checks for new Gitea releases and updates the database accordingly.
+
+	Behaviour:
+		- Fetches all active Gitea feeds and their last known release link.
+		- Queries each feed for its latest release.
+		- Updates `last_entry` if the latest release link differs from the stored one.
+		- Sends a notification for each changed release.
+		- A failure on one feed is logged and the remaining feeds are still checked.
+
+	Returns:
+		A JSON object with checked/new/error counts. If any feed failed,
+		a 500 is returned with the same counts as the detail.
+	"""
+	logger.info("[FastAPI] Checking for new Gitea releases")
+	results = {"checked": 0, "new": 0, "errors": 0}
+	with db.cursor() as cursor:
+		cursor.execute(
+			"SELECT id, rss_feed, feed_name, last_entry "
+			"FROM rss_feeds WHERE active = 1 AND type = %s",
+			("gitea",),
+		)
+		feeds = cursor.fetchall()
+		logger.debug(f"[FastAPI] Found {len(feeds)} active Gitea feeds to check.")
+
+		for feed_id, rss_feed, feed_name, last_entry in feeds:
+			results["checked"] += 1
+			try:
+				title, link = grab_gitea_release_information(rss_feed)
+				logger.debug(f"[FastAPI] ID:{feed_id},RSS Feed {rss_feed}: db={last_entry}, online={link}")
+
+				if link == last_entry:
+					continue
+
+				logger.info(f"[FastAPI] Release change for ID {feed_id}: {title}")
+				send_notification(feed_name, title, link)
+				cursor.execute(
+					"UPDATE rss_feeds SET last_entry = %s WHERE id = %s",
+					(link, feed_id),
+				)
+				db.commit()
+				logger.debug(f"[FastAPI] Notification sent and database updated for Gitea feed ID {feed_id}")
+				results["new"] += 1
+
+			except Exception as e:
+				db.rollback()
+				results["errors"] += 1
+				logger.error(
+					f"[FastAPI] Failed for feed ID {feed_id} ({rss_feed}): {e}",
+					exc_info=True,
+				)
+
+	if results["errors"]:
+		logger.warning(f"[FastAPI] Gitea check finished with errors: {results}")
+		raise HTTPException(status_code=500, detail=results)
+
+	logger.info(f"[FastAPI] Gitea check complete: {results}")
+	return results
+
 @api.get("/royalroad")
-def get_chapters(request: Request, db = Depends(get_db)):
+def get_chapters(db = Depends(get_db)):
 	"""
 	Checks for new Royalroad chapters and updates the database accordingly.
 
 	Behaviour:
-		- Fetches all active stories and their last known chapter number.
+		- Fetches all active stories and their last known chapter ID.
 		- Queries Royalroad for each story's latest chapter.
-		- Updates `lastChapter` if a new chapter is found.
+		- Updates `last_entry` if a new chapter is found.
 		- Sends a notification for any newly discovered chapters.
+		- A failure on one story is logged and the remaining stories are still checked.
 
 	Returns:
-		A JSON object indicating the check status.
+		A JSON object with checked/new/error counts. If any story failed,
+		a 500 is returned with the same counts as the detail.
 	"""
-	logger.info("[Royalroad] Checking for new chapters...")
+	logger.info("[FastAPI] Checking for new Royalroad chapters")
-	try:
+	results = {"checked": 0, "new": 0, "errors": 0}
-		cursor = db.cursor()
+	with db.cursor() as cursor:
-		cursor.execute("SELECT id, royalroadId, lastChapter FROM stories WHERE active=1")
+		cursor.execute(
+			"SELECT id, rss_feed, last_entry "
+			"FROM rss_feeds WHERE active = 1 AND type = %s",
+			("royalroad",),
+		)
+		feeds = cursor.fetchall()
+		logger.debug(f"[FastAPI] Found {len(feeds)} active stories to check.")
 
-		stories = cursor.fetchall()
+		for feed_id, rss_feed, last_entry in feeds:
-		logger.debug(f"[Royalroad] Found {len(stories)} active stories to check.")
+			results["checked"] += 1
+			try:
+				chapter_title, link, story_title, online_latest_entry = grab_royal_road_chapter_information(rss_feed)
+				logger.debug(f"[FastAPI] ID:{feed_id},RSS Feed {rss_feed}: db={last_entry}, online={online_latest_entry}")
 
-		for id, royalroadId, last_chapter_db in stories:
+				if online_latest_entry == last_entry:
-			chapter_title, chapter_link, story_title, chapter_id = grab_latest_chapter_information(royalroadId)
+					continue
-			logger.debug(f"[Royalroad] Story {id}: last={last_chapter_db}, latest={chapter_id}")
 
-			if chapter_id > last_chapter_db:
+				logger.info(f"[FastAPI] New chapter detected for story ID {feed_id}: {story_title}")
-				logger.info(f"[Royalroad] New chapter detected for story ID {id}: {story_title}")
+				send_notification(story_title, chapter_title, link)
-				cursor.execute("UPDATE stories SET lastChapter = %s WHERE id = %s", (chapter_id, id))
+				cursor.execute(
+					"UPDATE rss_feeds SET last_entry = %s WHERE id = %s",
+					(online_latest_entry, feed_id),
+				)
 				db.commit()
-				send_notification(story_title, chapter_title, chapter_link)
+				logger.debug(f"[FastAPI] Notification sent and database updated for story ID {feed_id}")
-				logger.debug(f"[Royalroad] Notification sent for story ID {id}")
+				results["new"] += 1
 
-		logger.info("[Royalroad] Chapter check completed successfully.")
+			except Exception as e:
-		return {"status": "checked"}
+				db.rollback()
-	except ValueError as e:
+				results["errors"] += 1
-		logger.error(f"[Royalroad] Failed to fetch feed {royalroadId}: {e}")
+				logger.error(
-		raise HTTPException(status_code=404, detail=str(e)) from e
+					f"[Royalroad] Failed for feed ID {feed_id} ({rss_feed}): {e}",
+					exc_info=True,
+				)
 
-	except Exception as e:
+	if results["errors"]:
-		logger.error(f"[Royalroad] Error during chapter check: {e}", exc_info=True)
+		logger.warning(f"[FastAPI] Royalroad check finished with errors: {results}")
-		raise HTTPException(status_code=500, detail=str(e)) from e
+		raise HTTPException(status_code=500, detail=results)
+
+	logger.info(f"[FastAPI] Royalroad check complete: {results}")
+	return results
 
 async def start_servers():
 	logger.info("[Server] Starting main API (port 5000) and metrics server (port 9000)...")
-	config_main = uvicorn.Config("main:api", host="0.0.0.0", port=5000, log_level=LOG_LEVEL.lower(), log_config=LOGGING_CONFIG)
+	config_main = uvicorn.Config(api, host="0.0.0.0", port=5000, log_level=LOG_LEVEL.lower(), log_config=LOGGING_CONFIG)
-	config_metrics = uvicorn.Config("metrics_server:metrics_api", host="0.0.0.0", port=9000, log_level=LOG_LEVEL.lower(), log_config=LOGGING_CONFIG)
+	config_metrics = uvicorn.Config(metrics_api, host="0.0.0.0", port=9000, log_level=LOG_LEVEL.lower(), log_config=LOGGING_CONFIG)
 
 	server_main = uvicorn.Server(config_main)
 	server_metrics = uvicorn.Server(config_metrics)
 
 	await asyncio.gather(server_main.serve(), server_metrics.serve())
-	logger.info("[Server] Both servers started successfully.")
+	logger.info("[Server] Both servers have stopped.")
 
 if __name__ == "__main__":
-	asyncio.run(start_servers())
+	asyncio.run(start_servers())