Add MangaDex handler for manga metadata and latest chapter lookup
Features: - Fetches manga metadata by MangaDex id - Fetch the most recent English chapters - Combine those to track a new manga
This commit is contained in:
1 parent
6b4f1e2eb4
commit
0effac114c
5 files changed
+267
-8
No files matched your search
+4
-4
@@ -3,14 +3,14 @@ from fastapi.exceptions import RequestValidationError
|
|||||||
from fastapi.responses import JSONResponse
|
from fastapi.responses import JSONResponse
|
||||||
from starlette.middleware.base import BaseHTTPMiddleware
|
from starlette.middleware.base import BaseHTTPMiddleware
|
||||||
|
|
||||||
|
from mdex_errors import MdexError
|
||||||
from metrics_server import REQUEST_COUNTER
|
from metrics_server import REQUEST_COUNTER
|
||||||
from generic_errors import GenericError
|
|
||||||
from simple_logger_handler import setup_logger
|
from simple_logger_handler import setup_logger
|
||||||
|
|
||||||
logger = setup_logger(__name__)
|
logger = setup_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
async def repository_error_handler(request: Request, exc: GenericError):
|
async def repository_error_handler(request: Request, exc: MdexError):
|
||||||
log = logger.warning if exc.client_visible else logger.error
|
log = logger.warning if exc.client_visible else logger.error
|
||||||
log(f"[FastAPI] {request.method} {request.url.path} failed: {exc}")
|
log(f"[FastAPI] {request.method} {request.url.path} failed: {exc}")
|
||||||
return JSONResponse(
|
return JSONResponse(
|
||||||
@@ -25,7 +25,7 @@ async def validation_error_handler(request: Request, exc: RequestValidationError
|
|||||||
|
|
||||||
for err in errors:
|
for err in errors:
|
||||||
original = (err.get("ctx") or {}).get("error")
|
original = (err.get("ctx") or {}).get("error")
|
||||||
if isinstance(original, GenericError):
|
if isinstance(original, MdexError):
|
||||||
return JSONResponse(
|
return JSONResponse(
|
||||||
status_code=original.status_code,
|
status_code=original.status_code,
|
||||||
headers=exc.headers,
|
headers=exc.headers,
|
||||||
@@ -63,6 +63,6 @@ async def prometheus_middleware(request: Request, call_next):
|
|||||||
|
|
||||||
def register_handlers(app: FastAPI) -> None:
|
def register_handlers(app: FastAPI) -> None:
|
||||||
"""Attach the error handlers and the metrics middleware to the app."""
|
"""Attach the error handlers and the metrics middleware to the app."""
|
||||||
app.add_exception_handler(GenericError, repository_error_handler)
|
app.add_exception_handler(MdexError, repository_error_handler)
|
||||||
app.add_exception_handler(RequestValidationError, validation_error_handler)
|
app.add_exception_handler(RequestValidationError, validation_error_handler)
|
||||||
app.add_middleware(BaseHTTPMiddleware, dispatch=prometheus_middleware)
|
app.add_middleware(BaseHTTPMiddleware, dispatch=prometheus_middleware)
|
||||||
@@ -0,0 +1,238 @@
|
|||||||
|
from pprint import pprint
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
import requests
|
||||||
|
from requests.adapters import HTTPAdapter
|
||||||
|
from urllib3.util.retry import Retry
|
||||||
|
|
||||||
|
from mdex_errors import NoMangaName
|
||||||
|
from models import MangaEntry, MdexChapter
|
||||||
|
from simple_logger_handler import setup_logger
|
||||||
|
|
||||||
|
logger = setup_logger(__name__)
|
||||||
|
|
||||||
|
_BASE_URL = "https://api.mangadex.org"
|
||||||
|
_TIMEOUT = 15
|
||||||
|
|
||||||
|
retry_strategy = Retry(
|
||||||
|
total=3,
|
||||||
|
connect=3,
|
||||||
|
read=3,
|
||||||
|
status=3,
|
||||||
|
backoff_factor=1,
|
||||||
|
status_forcelist=(429,),
|
||||||
|
allowed_methods=frozenset(["GET"]),
|
||||||
|
respect_retry_after_header=True,
|
||||||
|
raise_on_status=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
session = requests.Session()
|
||||||
|
session.mount("https://", HTTPAdapter(max_retries=retry_strategy))
|
||||||
|
|
||||||
|
|
||||||
|
def _query_mangadex(path: str, parameters=None) -> Optional[dict]:
|
||||||
|
"""Sends a GET request to the MangaDex manga endpoint.
|
||||||
|
|
||||||
|
Retries on 429 are handled by the shared session. All request, HTTP,
|
||||||
|
and parsing failures are logged and swallowed, so callers only need to
|
||||||
|
check for None.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
path: Path appended to ``{_BASE_URL}/manga/``, e.g. a manga id or
|
||||||
|
``"<manga_id>/feed"``.
|
||||||
|
parameters: Optional query parameters passed to ``requests``.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The parsed JSON response as a dict, or None if the request failed,
|
||||||
|
the body was not valid JSON, was not a JSON object, or contained
|
||||||
|
``{"result": "error"}``.
|
||||||
|
"""
|
||||||
|
url = f"{_BASE_URL}/manga/{path}"
|
||||||
|
try:
|
||||||
|
response = session.get(url, params=parameters, timeout=_TIMEOUT)
|
||||||
|
response.raise_for_status()
|
||||||
|
|
||||||
|
try:
|
||||||
|
data = response.json()
|
||||||
|
except requests.exceptions.JSONDecodeError:
|
||||||
|
logger.error(
|
||||||
|
"Invalid JSON response from MangaDex API: status=%s, path=%s, response=%s",
|
||||||
|
response.status_code,path,response.text[:500],
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
if not isinstance(data,dict):
|
||||||
|
logger.error("Unexpected response shape from MangaDex API: path=%s, type=%s", path, type(data).__name__)
|
||||||
|
return None
|
||||||
|
|
||||||
|
if data.get("result") == "error":
|
||||||
|
logger.error("MangaDex API returned an error result:path=%s, type=%s", path, data.get("errors"))
|
||||||
|
return None
|
||||||
|
|
||||||
|
return data
|
||||||
|
|
||||||
|
except requests.exceptions.HTTPError as e:
|
||||||
|
response = e.response
|
||||||
|
|
||||||
|
if response is None:
|
||||||
|
logger.exception("MangaDex API HTTP error with no response: path=%s", path)
|
||||||
|
return None
|
||||||
|
|
||||||
|
try:
|
||||||
|
error_data = response.json()
|
||||||
|
except requests.exceptions.JSONDecodeError:
|
||||||
|
error_data = response.text[:500]
|
||||||
|
|
||||||
|
if response.status_code == 404:
|
||||||
|
logger.warning("MangaDex resource not found: path=%s", path)
|
||||||
|
else:
|
||||||
|
logger.error("MangaDex API HTTP error: status=%s, path=%s, error=%s",response.status_code,path,error_data )
|
||||||
|
|
||||||
|
except requests.exceptions.Timeout:
|
||||||
|
logger.error("MangaDex API request timed out: path=%s", path)
|
||||||
|
|
||||||
|
except requests.exceptions.ConnectionError:
|
||||||
|
logger.error("Failed to connect to MangaDex API: path=%s", path)
|
||||||
|
|
||||||
|
except requests.exceptions.RequestException:
|
||||||
|
logger.exception("MangaDex API request failed: path=%s", path)
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _parse_anilist_id(raw) -> int:
|
||||||
|
"""Converts the AniList id stored in MangaDex's ``links["al"]`` to an int.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
raw: The raw value of ``links["al"]``. MangaDex stores it as a string,
|
||||||
|
and it may be missing or empty.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The AniList id as an int, or None if the value is missing or cannot be
|
||||||
|
parsed (a warning is logged in the latter case).
|
||||||
|
"""
|
||||||
|
if not raw:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return int(raw)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
logger.warning("Could not parse AniList id from MangaDex links: %r", raw)
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _create_manga_entry(response: dict) -> MangaEntry:
|
||||||
|
"""Builds a MangaEntry from a MangaDex manga response.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
response: The parsed JSON body of a ``GET /manga/{id}`` request.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
A MangaEntry with Japanese, romanized Japanese, and English names
|
||||||
|
(any of which may be None), the MangaDex id, the type, and the
|
||||||
|
AniList id (None if unknown).
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
NoMangaName: If none of the three name fields could be found.
|
||||||
|
KeyError: If required fields such as ``data``, ``attributes``, or
|
||||||
|
``id`` are missing from the response.
|
||||||
|
TypeError: If a field has an unexpected type.
|
||||||
|
"""
|
||||||
|
data = response["data"]
|
||||||
|
attributes = data["attributes"]
|
||||||
|
title = attributes.get("title", {})
|
||||||
|
links = attributes.get("links") or {}
|
||||||
|
|
||||||
|
manga = MangaEntry(
|
||||||
|
name_ja=title.get("ja"),
|
||||||
|
name_ja_ro=title.get("ja-ro"),
|
||||||
|
name_en=next(
|
||||||
|
(
|
||||||
|
entry["en"]
|
||||||
|
for entry in attributes.get("altTitles", [])
|
||||||
|
if "en" in entry
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
),
|
||||||
|
id=data["id"],
|
||||||
|
type=data.get("type"),
|
||||||
|
anilist_link=_parse_anilist_id(links.get("al")),
|
||||||
|
)
|
||||||
|
if all(name is None for name in (manga.name_ja, manga.name_ja_ro, manga.name_en)):
|
||||||
|
raise NoMangaName(manga.id)
|
||||||
|
return manga
|
||||||
|
|
||||||
|
def _query_for_recent_chapters(manga_id: str, limit: int=100) -> list[MdexChapter]:
|
||||||
|
"""Fetches the most recent English chapters of a manga.
|
||||||
|
|
||||||
|
Malformed chapter entries are logged and skipped rather than aborting the
|
||||||
|
whole list.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
manga_id: The MangaDex UUID of the manga.
|
||||||
|
limit: Maximum number of chapters to request, newest first. Clamped
|
||||||
|
to the range 1-500, which is the MangaDex feed maximum.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
A list of MdexChapter objects ordered by chapter number descending.
|
||||||
|
The list is empty if the request failed, the response was malformed,
|
||||||
|
or the manga has no English chapters.
|
||||||
|
"""
|
||||||
|
limit = max(1, min(limit, 500))
|
||||||
|
params={"translatedLanguage[]": ["en"],"order[chapter]":"desc","limit":limit}
|
||||||
|
|
||||||
|
response = _query_mangadex(f"{manga_id}/feed",params)
|
||||||
|
if response is None:
|
||||||
|
logger.warning("Could not fetch chapter data: manga_id=%s", manga_id)
|
||||||
|
return []
|
||||||
|
|
||||||
|
entries = response.get("data", [])
|
||||||
|
results = []
|
||||||
|
for entry in entries:
|
||||||
|
try:
|
||||||
|
chapter_id = entry.get("id")
|
||||||
|
if not chapter_id:
|
||||||
|
logger.warning("Skipping chapter without id manga_id=%s", manga_id)
|
||||||
|
continue
|
||||||
|
|
||||||
|
attributes = entry.get("attributes", {})
|
||||||
|
results.append(MdexChapter(
|
||||||
|
id=chapter_id,
|
||||||
|
chapter=attributes.get("chapter"),
|
||||||
|
title=attributes.get("title"),
|
||||||
|
published=attributes.get("publishAt"),
|
||||||
|
))
|
||||||
|
except (AttributeError, TypeError, ValueError):
|
||||||
|
logger.exception("Skipping malformed chapter entry: manga_id=%s", manga_id)
|
||||||
|
if not results:
|
||||||
|
logger.info("No English chapters found for manga_id=%s", manga_id)
|
||||||
|
return results
|
||||||
|
|
||||||
|
def get_manga_and_latest_chapter_data_by_id(manga_id: str):
|
||||||
|
"""Fetches a manga's metadata together with its latest English chapter.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
manga_id: The MangaDex UUID of the manga.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
A tuple of ``(MangaEntry, MdexChapter | None)``. The chapter is None
|
||||||
|
if the manga has no English chapters yet. Returns None if the manga
|
||||||
|
could not be fetched or its data was malformed.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
NoMangaName: If the manga has no usable title.
|
||||||
|
"""
|
||||||
|
data = _query_mangadex(manga_id, parameters=None)
|
||||||
|
if data is None:
|
||||||
|
return None
|
||||||
|
manga = _create_manga_entry(data)
|
||||||
|
most_recent_chapter = _query_for_recent_chapters(manga_id,1)[0]
|
||||||
|
return manga,most_recent_chapter
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
test_id = "f30a2fd3-cf2c-4f77-9910-51a908459a82"
|
||||||
|
title = "Tokyo Ghoul"
|
||||||
|
params={"title": title}
|
||||||
|
data = _query_mangadex("", params)
|
||||||
|
# Return short description, chapter counts, finished status, tags/genres/demographic, Clickable AL link?, author/artist
|
||||||
|
pprint(data)
|
||||||
|
#pprint(_query_for_recent_chapters(manga_id,10))
|
||||||
|
#data = get_manga_and_latest_chapter_data_by_id(test_id)
|
||||||
|
#pprint(data)
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
from typing import ClassVar
|
from typing import ClassVar
|
||||||
|
|
||||||
|
|
||||||
class GenericError(Exception):
|
class MdexError(Exception):
|
||||||
"""Base class for every GENERIC error the API can explain to a client."""
|
"""Base class for every GENERIC error the API can explain to a client."""
|
||||||
status_code: ClassVar[int] = 502
|
status_code: ClassVar[int] = 502
|
||||||
code: ClassVar[str] = "sm_upstream_error"
|
code: ClassVar[str] = "sm_upstream_error"
|
||||||
@@ -15,7 +15,14 @@ class GenericError(Exception):
|
|||||||
"""Extra HTTP headers the proxy should send with this error."""
|
"""Extra HTTP headers the proxy should send with this error."""
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
|
class NoMangaName(MdexError):
|
||||||
|
"No manga name found for this entry"
|
||||||
|
status_code = 422
|
||||||
|
code = "svm_no_manga_name"
|
||||||
|
client_visible = True
|
||||||
|
|
||||||
|
def __init__(self, manga_id:str | None):
|
||||||
|
self.manga_id = manga_id
|
||||||
|
super().__init__(f"Could not find the manga: {manga_id}")
|
||||||
|
|
||||||
|
|
||||||
@@ -3,6 +3,20 @@ from datetime import datetime
|
|||||||
from pydantic import BaseModel, Field, computed_field
|
from pydantic import BaseModel, Field, computed_field
|
||||||
|
|
||||||
|
|
||||||
|
class MdexChapter(BaseModel):
|
||||||
|
id: str
|
||||||
|
chapter: float
|
||||||
|
title : str | None = None
|
||||||
|
published: datetime | None = None
|
||||||
|
|
||||||
|
class MangaEntry(BaseModel):
|
||||||
|
id: str
|
||||||
|
name_ja: str | None = None
|
||||||
|
name_ja_ro: str | None = None
|
||||||
|
name_en: str | None = None
|
||||||
|
type: str | None = None
|
||||||
|
anilist_link: int | None = None
|
||||||
|
|
||||||
class GenericEntry(BaseModel):
|
class GenericEntry(BaseModel):
|
||||||
name: str
|
name: str
|
||||||
custom_name: str | None = None
|
custom_name: str | None = None
|
||||||
|
|||||||
+1
-1
@@ -13,7 +13,7 @@ logger = setup_logger(__name__)
|
|||||||
|
|
||||||
user_router = APIRouter(dependencies=[Depends(auth.require(Caller.GATEWAY))])
|
user_router = APIRouter(dependencies=[Depends(auth.require(Caller.GATEWAY))])
|
||||||
|
|
||||||
@user_router.post("/entry/add")
|
@user_router.post("/manga/add/mangadex")
|
||||||
def add_entry(
|
def add_entry(
|
||||||
user_id: UserId,
|
user_id: UserId,
|
||||||
db=Depends(get_db),
|
db=Depends(get_db),
|
||||||
|
|||||||
Reference in new issue
Block a user