Add MangaDex handler for manga metadata and latest chapter lookup
Features: - Fetches manga metadata by MangaDex id - Fetch the most recent English chapters - Combine those to track a new manga
This commit is contained in:
1 parent
6b4f1e2eb4
commit
0effac114c
5 files changed
+267
-8
No files matched your search
+4
-4
@@ -3,14 +3,14 @@ from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
|
||||
from mdex_errors import MdexError
|
||||
from metrics_server import REQUEST_COUNTER
|
||||
from generic_errors import GenericError
|
||||
from simple_logger_handler import setup_logger
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
|
||||
async def repository_error_handler(request: Request, exc: GenericError):
|
||||
async def repository_error_handler(request: Request, exc: MdexError):
|
||||
log = logger.warning if exc.client_visible else logger.error
|
||||
log(f"[FastAPI] {request.method} {request.url.path} failed: {exc}")
|
||||
return JSONResponse(
|
||||
@@ -25,7 +25,7 @@ async def validation_error_handler(request: Request, exc: RequestValidationError
|
||||
|
||||
for err in errors:
|
||||
original = (err.get("ctx") or {}).get("error")
|
||||
if isinstance(original, GenericError):
|
||||
if isinstance(original, MdexError):
|
||||
return JSONResponse(
|
||||
status_code=original.status_code,
|
||||
headers=exc.headers,
|
||||
@@ -63,6 +63,6 @@ async def prometheus_middleware(request: Request, call_next):
|
||||
|
||||
def register_handlers(app: FastAPI) -> None:
|
||||
"""Attach the error handlers and the metrics middleware to the app."""
|
||||
app.add_exception_handler(GenericError, repository_error_handler)
|
||||
app.add_exception_handler(MdexError, repository_error_handler)
|
||||
app.add_exception_handler(RequestValidationError, validation_error_handler)
|
||||
app.add_middleware(BaseHTTPMiddleware, dispatch=prometheus_middleware)
|
||||
@@ -0,0 +1,238 @@
|
||||
from pprint import pprint
|
||||
from typing import Optional
|
||||
|
||||
import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
|
||||
from mdex_errors import NoMangaName
|
||||
from models import MangaEntry, MdexChapter
|
||||
from simple_logger_handler import setup_logger
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
_BASE_URL = "https://api.mangadex.org"
|
||||
_TIMEOUT = 15
|
||||
|
||||
retry_strategy = Retry(
|
||||
total=3,
|
||||
connect=3,
|
||||
read=3,
|
||||
status=3,
|
||||
backoff_factor=1,
|
||||
status_forcelist=(429,),
|
||||
allowed_methods=frozenset(["GET"]),
|
||||
respect_retry_after_header=True,
|
||||
raise_on_status=False,
|
||||
)
|
||||
|
||||
session = requests.Session()
|
||||
session.mount("https://", HTTPAdapter(max_retries=retry_strategy))
|
||||
|
||||
|
||||
def _query_mangadex(path: str, parameters=None) -> Optional[dict]:
|
||||
"""Sends a GET request to the MangaDex manga endpoint.
|
||||
|
||||
Retries on 429 are handled by the shared session. All request, HTTP,
|
||||
and parsing failures are logged and swallowed, so callers only need to
|
||||
check for None.
|
||||
|
||||
Args:
|
||||
path: Path appended to ``{_BASE_URL}/manga/``, e.g. a manga id or
|
||||
``"<manga_id>/feed"``.
|
||||
parameters: Optional query parameters passed to ``requests``.
|
||||
|
||||
Returns:
|
||||
The parsed JSON response as a dict, or None if the request failed,
|
||||
the body was not valid JSON, was not a JSON object, or contained
|
||||
``{"result": "error"}``.
|
||||
"""
|
||||
url = f"{_BASE_URL}/manga/{path}"
|
||||
try:
|
||||
response = session.get(url, params=parameters, timeout=_TIMEOUT)
|
||||
response.raise_for_status()
|
||||
|
||||
try:
|
||||
data = response.json()
|
||||
except requests.exceptions.JSONDecodeError:
|
||||
logger.error(
|
||||
"Invalid JSON response from MangaDex API: status=%s, path=%s, response=%s",
|
||||
response.status_code,path,response.text[:500],
|
||||
)
|
||||
return None
|
||||
|
||||
if not isinstance(data,dict):
|
||||
logger.error("Unexpected response shape from MangaDex API: path=%s, type=%s", path, type(data).__name__)
|
||||
return None
|
||||
|
||||
if data.get("result") == "error":
|
||||
logger.error("MangaDex API returned an error result:path=%s, type=%s", path, data.get("errors"))
|
||||
return None
|
||||
|
||||
return data
|
||||
|
||||
except requests.exceptions.HTTPError as e:
|
||||
response = e.response
|
||||
|
||||
if response is None:
|
||||
logger.exception("MangaDex API HTTP error with no response: path=%s", path)
|
||||
return None
|
||||
|
||||
try:
|
||||
error_data = response.json()
|
||||
except requests.exceptions.JSONDecodeError:
|
||||
error_data = response.text[:500]
|
||||
|
||||
if response.status_code == 404:
|
||||
logger.warning("MangaDex resource not found: path=%s", path)
|
||||
else:
|
||||
logger.error("MangaDex API HTTP error: status=%s, path=%s, error=%s",response.status_code,path,error_data )
|
||||
|
||||
except requests.exceptions.Timeout:
|
||||
logger.error("MangaDex API request timed out: path=%s", path)
|
||||
|
||||
except requests.exceptions.ConnectionError:
|
||||
logger.error("Failed to connect to MangaDex API: path=%s", path)
|
||||
|
||||
except requests.exceptions.RequestException:
|
||||
logger.exception("MangaDex API request failed: path=%s", path)
|
||||
|
||||
return None
|
||||
|
||||
def _parse_anilist_id(raw) -> int:
|
||||
"""Converts the AniList id stored in MangaDex's ``links["al"]`` to an int.
|
||||
|
||||
Args:
|
||||
raw: The raw value of ``links["al"]``. MangaDex stores it as a string,
|
||||
and it may be missing or empty.
|
||||
|
||||
Returns:
|
||||
The AniList id as an int, or None if the value is missing or cannot be
|
||||
parsed (a warning is logged in the latter case).
|
||||
"""
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
return int(raw)
|
||||
except (TypeError, ValueError):
|
||||
logger.warning("Could not parse AniList id from MangaDex links: %r", raw)
|
||||
return None
|
||||
|
||||
def _create_manga_entry(response: dict) -> MangaEntry:
|
||||
"""Builds a MangaEntry from a MangaDex manga response.
|
||||
|
||||
Args:
|
||||
response: The parsed JSON body of a ``GET /manga/{id}`` request.
|
||||
|
||||
Returns:
|
||||
A MangaEntry with Japanese, romanized Japanese, and English names
|
||||
(any of which may be None), the MangaDex id, the type, and the
|
||||
AniList id (None if unknown).
|
||||
|
||||
Raises:
|
||||
NoMangaName: If none of the three name fields could be found.
|
||||
KeyError: If required fields such as ``data``, ``attributes``, or
|
||||
``id`` are missing from the response.
|
||||
TypeError: If a field has an unexpected type.
|
||||
"""
|
||||
data = response["data"]
|
||||
attributes = data["attributes"]
|
||||
title = attributes.get("title", {})
|
||||
links = attributes.get("links") or {}
|
||||
|
||||
manga = MangaEntry(
|
||||
name_ja=title.get("ja"),
|
||||
name_ja_ro=title.get("ja-ro"),
|
||||
name_en=next(
|
||||
(
|
||||
entry["en"]
|
||||
for entry in attributes.get("altTitles", [])
|
||||
if "en" in entry
|
||||
),
|
||||
None,
|
||||
),
|
||||
id=data["id"],
|
||||
type=data.get("type"),
|
||||
anilist_link=_parse_anilist_id(links.get("al")),
|
||||
)
|
||||
if all(name is None for name in (manga.name_ja, manga.name_ja_ro, manga.name_en)):
|
||||
raise NoMangaName(manga.id)
|
||||
return manga
|
||||
|
||||
def _query_for_recent_chapters(manga_id: str, limit: int=100) -> list[MdexChapter]:
|
||||
"""Fetches the most recent English chapters of a manga.
|
||||
|
||||
Malformed chapter entries are logged and skipped rather than aborting the
|
||||
whole list.
|
||||
|
||||
Args:
|
||||
manga_id: The MangaDex UUID of the manga.
|
||||
limit: Maximum number of chapters to request, newest first. Clamped
|
||||
to the range 1-500, which is the MangaDex feed maximum.
|
||||
|
||||
Returns:
|
||||
A list of MdexChapter objects ordered by chapter number descending.
|
||||
The list is empty if the request failed, the response was malformed,
|
||||
or the manga has no English chapters.
|
||||
"""
|
||||
limit = max(1, min(limit, 500))
|
||||
params={"translatedLanguage[]": ["en"],"order[chapter]":"desc","limit":limit}
|
||||
|
||||
response = _query_mangadex(f"{manga_id}/feed",params)
|
||||
if response is None:
|
||||
logger.warning("Could not fetch chapter data: manga_id=%s", manga_id)
|
||||
return []
|
||||
|
||||
entries = response.get("data", [])
|
||||
results = []
|
||||
for entry in entries:
|
||||
try:
|
||||
chapter_id = entry.get("id")
|
||||
if not chapter_id:
|
||||
logger.warning("Skipping chapter without id manga_id=%s", manga_id)
|
||||
continue
|
||||
|
||||
attributes = entry.get("attributes", {})
|
||||
results.append(MdexChapter(
|
||||
id=chapter_id,
|
||||
chapter=attributes.get("chapter"),
|
||||
title=attributes.get("title"),
|
||||
published=attributes.get("publishAt"),
|
||||
))
|
||||
except (AttributeError, TypeError, ValueError):
|
||||
logger.exception("Skipping malformed chapter entry: manga_id=%s", manga_id)
|
||||
if not results:
|
||||
logger.info("No English chapters found for manga_id=%s", manga_id)
|
||||
return results
|
||||
|
||||
def get_manga_and_latest_chapter_data_by_id(manga_id: str):
|
||||
"""Fetches a manga's metadata together with its latest English chapter.
|
||||
|
||||
Args:
|
||||
manga_id: The MangaDex UUID of the manga.
|
||||
|
||||
Returns:
|
||||
A tuple of ``(MangaEntry, MdexChapter | None)``. The chapter is None
|
||||
if the manga has no English chapters yet. Returns None if the manga
|
||||
could not be fetched or its data was malformed.
|
||||
|
||||
Raises:
|
||||
NoMangaName: If the manga has no usable title.
|
||||
"""
|
||||
data = _query_mangadex(manga_id, parameters=None)
|
||||
if data is None:
|
||||
return None
|
||||
manga = _create_manga_entry(data)
|
||||
most_recent_chapter = _query_for_recent_chapters(manga_id,1)[0]
|
||||
return manga,most_recent_chapter
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_id = "f30a2fd3-cf2c-4f77-9910-51a908459a82"
|
||||
title = "Tokyo Ghoul"
|
||||
params={"title": title}
|
||||
data = _query_mangadex("", params)
|
||||
# Return short description, chapter counts, finished status, tags/genres/demographic, Clickable AL link?, author/artist
|
||||
pprint(data)
|
||||
#pprint(_query_for_recent_chapters(manga_id,10))
|
||||
#data = get_manga_and_latest_chapter_data_by_id(test_id)
|
||||
#pprint(data)
|
||||
@@ -1,7 +1,7 @@
|
||||
from typing import ClassVar
|
||||
|
||||
|
||||
class GenericError(Exception):
|
||||
class MdexError(Exception):
|
||||
"""Base class for every GENERIC error the API can explain to a client."""
|
||||
status_code: ClassVar[int] = 502
|
||||
code: ClassVar[str] = "sm_upstream_error"
|
||||
@@ -15,7 +15,14 @@ class GenericError(Exception):
|
||||
"""Extra HTTP headers the proxy should send with this error."""
|
||||
return {}
|
||||
|
||||
|
||||
|
||||
class NoMangaName(MdexError):
|
||||
"No manga name found for this entry"
|
||||
status_code = 422
|
||||
code = "svm_no_manga_name"
|
||||
client_visible = True
|
||||
|
||||
def __init__(self, manga_id:str | None):
|
||||
self.manga_id = manga_id
|
||||
super().__init__(f"Could not find the manga: {manga_id}")
|
||||
|
||||
|
||||
@@ -3,6 +3,20 @@ from datetime import datetime
|
||||
from pydantic import BaseModel, Field, computed_field
|
||||
|
||||
|
||||
class MdexChapter(BaseModel):
|
||||
id: str
|
||||
chapter: float
|
||||
title : str | None = None
|
||||
published: datetime | None = None
|
||||
|
||||
class MangaEntry(BaseModel):
|
||||
id: str
|
||||
name_ja: str | None = None
|
||||
name_ja_ro: str | None = None
|
||||
name_en: str | None = None
|
||||
type: str | None = None
|
||||
anilist_link: int | None = None
|
||||
|
||||
class GenericEntry(BaseModel):
|
||||
name: str
|
||||
custom_name: str | None = None
|
||||
|
||||
+1
-1
@@ -13,7 +13,7 @@ logger = setup_logger(__name__)
|
||||
|
||||
user_router = APIRouter(dependencies=[Depends(auth.require(Caller.GATEWAY))])
|
||||
|
||||
@user_router.post("/entry/add")
|
||||
@user_router.post("/manga/add/mangadex")
|
||||
def add_entry(
|
||||
user_id: UserId,
|
||||
db=Depends(get_db),
|
||||
|
||||
Reference in new issue
Block a user