Add MangaDex handler for manga metadata and latest chapter lookup

Features:
- Fetches manga metadata by MangaDex id
- Fetch the most recent English chapters
- Combine those to track a new manga
This commit is contained in:
florian committed 2026-11-15 23:11:22 +01:00
1 parent 6b4f1e2eb4
commit 0effac114c
5 files changed
+267 -8

No files matched your search

+4 -4
View File
@@ -3,14 +3,14 @@ from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from starlette.middleware.base import BaseHTTPMiddleware
from mdex_errors import MdexError
from metrics_server import REQUEST_COUNTER
from generic_errors import GenericError
from simple_logger_handler import setup_logger
logger = setup_logger(__name__)
async def repository_error_handler(request: Request, exc: GenericError):
async def repository_error_handler(request: Request, exc: MdexError):
log = logger.warning if exc.client_visible else logger.error
log(f"[FastAPI] {request.method} {request.url.path} failed: {exc}")
return JSONResponse(
@@ -25,7 +25,7 @@ async def validation_error_handler(request: Request, exc: RequestValidationError
for err in errors:
original = (err.get("ctx") or {}).get("error")
if isinstance(original, GenericError):
if isinstance(original, MdexError):
return JSONResponse(
status_code=original.status_code,
headers=exc.headers,
@@ -63,6 +63,6 @@ async def prometheus_middleware(request: Request, call_next):
def register_handlers(app: FastAPI) -> None:
"""Attach the error handlers and the metrics middleware to the app."""
app.add_exception_handler(GenericError, repository_error_handler)
app.add_exception_handler(MdexError, repository_error_handler)
app.add_exception_handler(RequestValidationError, validation_error_handler)
app.add_middleware(BaseHTTPMiddleware, dispatch=prometheus_middleware)
+238
View File
@@ -0,0 +1,238 @@
from pprint import pprint
from typing import Optional
import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
from mdex_errors import NoMangaName
from models import MangaEntry, MdexChapter
from simple_logger_handler import setup_logger
logger = setup_logger(__name__)
_BASE_URL = "https://api.mangadex.org"
_TIMEOUT = 15
retry_strategy = Retry(
total=3,
connect=3,
read=3,
status=3,
backoff_factor=1,
status_forcelist=(429,),
allowed_methods=frozenset(["GET"]),
respect_retry_after_header=True,
raise_on_status=False,
)
session = requests.Session()
session.mount("https://", HTTPAdapter(max_retries=retry_strategy))
def _query_mangadex(path: str, parameters=None) -> Optional[dict]:
"""Sends a GET request to the MangaDex manga endpoint.
Retries on 429 are handled by the shared session. All request, HTTP,
and parsing failures are logged and swallowed, so callers only need to
check for None.
Args:
path: Path appended to ``{_BASE_URL}/manga/``, e.g. a manga id or
``"<manga_id>/feed"``.
parameters: Optional query parameters passed to ``requests``.
Returns:
The parsed JSON response as a dict, or None if the request failed,
the body was not valid JSON, was not a JSON object, or contained
``{"result": "error"}``.
"""
url = f"{_BASE_URL}/manga/{path}"
try:
response = session.get(url, params=parameters, timeout=_TIMEOUT)
response.raise_for_status()
try:
data = response.json()
except requests.exceptions.JSONDecodeError:
logger.error(
"Invalid JSON response from MangaDex API: status=%s, path=%s, response=%s",
response.status_code,path,response.text[:500],
)
return None
if not isinstance(data,dict):
logger.error("Unexpected response shape from MangaDex API: path=%s, type=%s", path, type(data).__name__)
return None
if data.get("result") == "error":
logger.error("MangaDex API returned an error result:path=%s, type=%s", path, data.get("errors"))
return None
return data
except requests.exceptions.HTTPError as e:
response = e.response
if response is None:
logger.exception("MangaDex API HTTP error with no response: path=%s", path)
return None
try:
error_data = response.json()
except requests.exceptions.JSONDecodeError:
error_data = response.text[:500]
if response.status_code == 404:
logger.warning("MangaDex resource not found: path=%s", path)
else:
logger.error("MangaDex API HTTP error: status=%s, path=%s, error=%s",response.status_code,path,error_data )
except requests.exceptions.Timeout:
logger.error("MangaDex API request timed out: path=%s", path)
except requests.exceptions.ConnectionError:
logger.error("Failed to connect to MangaDex API: path=%s", path)
except requests.exceptions.RequestException:
logger.exception("MangaDex API request failed: path=%s", path)
return None
def _parse_anilist_id(raw) -> int:
"""Converts the AniList id stored in MangaDex's ``links["al"]`` to an int.
Args:
raw: The raw value of ``links["al"]``. MangaDex stores it as a string,
and it may be missing or empty.
Returns:
The AniList id as an int, or None if the value is missing or cannot be
parsed (a warning is logged in the latter case).
"""
if not raw:
return None
try:
return int(raw)
except (TypeError, ValueError):
logger.warning("Could not parse AniList id from MangaDex links: %r", raw)
return None
def _create_manga_entry(response: dict) -> MangaEntry:
"""Builds a MangaEntry from a MangaDex manga response.
Args:
response: The parsed JSON body of a ``GET /manga/{id}`` request.
Returns:
A MangaEntry with Japanese, romanized Japanese, and English names
(any of which may be None), the MangaDex id, the type, and the
AniList id (None if unknown).
Raises:
NoMangaName: If none of the three name fields could be found.
KeyError: If required fields such as ``data``, ``attributes``, or
``id`` are missing from the response.
TypeError: If a field has an unexpected type.
"""
data = response["data"]
attributes = data["attributes"]
title = attributes.get("title", {})
links = attributes.get("links") or {}
manga = MangaEntry(
name_ja=title.get("ja"),
name_ja_ro=title.get("ja-ro"),
name_en=next(
(
entry["en"]
for entry in attributes.get("altTitles", [])
if "en" in entry
),
None,
),
id=data["id"],
type=data.get("type"),
anilist_link=_parse_anilist_id(links.get("al")),
)
if all(name is None for name in (manga.name_ja, manga.name_ja_ro, manga.name_en)):
raise NoMangaName(manga.id)
return manga
def _query_for_recent_chapters(manga_id: str, limit: int=100) -> list[MdexChapter]:
"""Fetches the most recent English chapters of a manga.
Malformed chapter entries are logged and skipped rather than aborting the
whole list.
Args:
manga_id: The MangaDex UUID of the manga.
limit: Maximum number of chapters to request, newest first. Clamped
to the range 1-500, which is the MangaDex feed maximum.
Returns:
A list of MdexChapter objects ordered by chapter number descending.
The list is empty if the request failed, the response was malformed,
or the manga has no English chapters.
"""
limit = max(1, min(limit, 500))
params={"translatedLanguage[]": ["en"],"order[chapter]":"desc","limit":limit}
response = _query_mangadex(f"{manga_id}/feed",params)
if response is None:
logger.warning("Could not fetch chapter data: manga_id=%s", manga_id)
return []
entries = response.get("data", [])
results = []
for entry in entries:
try:
chapter_id = entry.get("id")
if not chapter_id:
logger.warning("Skipping chapter without id manga_id=%s", manga_id)
continue
attributes = entry.get("attributes", {})
results.append(MdexChapter(
id=chapter_id,
chapter=attributes.get("chapter"),
title=attributes.get("title"),
published=attributes.get("publishAt"),
))
except (AttributeError, TypeError, ValueError):
logger.exception("Skipping malformed chapter entry: manga_id=%s", manga_id)
if not results:
logger.info("No English chapters found for manga_id=%s", manga_id)
return results
def get_manga_and_latest_chapter_data_by_id(manga_id: str):
"""Fetches a manga's metadata together with its latest English chapter.
Args:
manga_id: The MangaDex UUID of the manga.
Returns:
A tuple of ``(MangaEntry, MdexChapter | None)``. The chapter is None
if the manga has no English chapters yet. Returns None if the manga
could not be fetched or its data was malformed.
Raises:
NoMangaName: If the manga has no usable title.
"""
data = _query_mangadex(manga_id, parameters=None)
if data is None:
return None
manga = _create_manga_entry(data)
most_recent_chapter = _query_for_recent_chapters(manga_id,1)[0]
return manga,most_recent_chapter
if __name__ == "__main__":
test_id = "f30a2fd3-cf2c-4f77-9910-51a908459a82"
title = "Tokyo Ghoul"
params={"title": title}
data = _query_mangadex("", params)
# Return short description, chapter counts, finished status, tags/genres/demographic, Clickable AL link?, author/artist
pprint(data)
#pprint(_query_for_recent_chapters(manga_id,10))
#data = get_manga_and_latest_chapter_data_by_id(test_id)
#pprint(data)
+10 -3
View File
@@ -1,7 +1,7 @@
from typing import ClassVar
class GenericError(Exception):
class MdexError(Exception):
"""Base class for every GENERIC error the API can explain to a client."""
status_code: ClassVar[int] = 502
code: ClassVar[str] = "sm_upstream_error"
@@ -15,7 +15,14 @@ class GenericError(Exception):
"""Extra HTTP headers the proxy should send with this error."""
return {}
class NoMangaName(MdexError):
"No manga name found for this entry"
status_code = 422
code = "svm_no_manga_name"
client_visible = True
def __init__(self, manga_id:str | None):
self.manga_id = manga_id
super().__init__(f"Could not find the manga: {manga_id}")
+14
View File
@@ -3,6 +3,20 @@ from datetime import datetime
from pydantic import BaseModel, Field, computed_field
class MdexChapter(BaseModel):
id: str
chapter: float
title : str | None = None
published: datetime | None = None
class MangaEntry(BaseModel):
id: str
name_ja: str | None = None
name_ja_ro: str | None = None
name_en: str | None = None
type: str | None = None
anilist_link: int | None = None
class GenericEntry(BaseModel):
name: str
custom_name: str | None = None
+1 -1
View File
@@ -13,7 +13,7 @@ logger = setup_logger(__name__)
user_router = APIRouter(dependencies=[Depends(auth.require(Caller.GATEWAY))])
@user_router.post("/entry/add")
@user_router.post("/manga/add/mangadex")
def add_entry(
user_id: UserId,
db=Depends(get_db),