Overhauled API calls, it now raises meaningful error messagess that can be understood by an upstream client

src/dockerhub_api.py

@@ -1,6 +1,16 @@
+from typing import NoReturn
+from urllib.parse import quote
 
 import requests
 
+from repository_errors import (
+	ApiAuthenticationFailure,
+	OrganisationNotFoundError,
+	RateLimitedError,
+	RepositoryApiError,
+	RepositoryNotFoundError,
+	RepositoryTagNotFoundError,
+)
 from secret_manager import get_secret
 from simple_logger_handler import setup_logger
 
@@ -9,63 +19,149 @@ dockerhub_username = get_secret("DOCKERHUB_USERNAME","/etc/secrets/dockerhub_use
 
 logger = setup_logger(__name__)
 
-def login_and_get_token() -> str:
+_SERVICE = "Docker Hub"
+_API = "https://hub.docker.com/v2"
+_TIMEOUT = (3.05, 10)
+_VERSION_FIELD = "id"
+
+def _request(method: str, url: str, **kwargs) -> requests.Response:
+	"""Send a request and turn transport problems, rate limits and 5xx into typed errors."""
+	try:
+		response = requests.request(method, url, timeout=_TIMEOUT, **kwargs)
+	except requests.RequestException as e:
+		raise RepositoryApiError(_SERVICE, f"request failed: {e}") from e
+
+	if response.status_code == 429:
+		retry_after = response.headers.get("Retry-After", "")
+		raise RateLimitedError(_SERVICE, int(retry_after) if retry_after.isdigit() else None)
+	if response.status_code >= 500:
+		raise RepositoryApiError(_SERVICE, f"server error {response.status_code}")
+	return response
+
+
+def _get_access_token() -> str:
 	"""
-	[DockerHub] Login to Docker Hub and retrieve a JWT token.
+	[DockerHub] Exchange the username and personal access token for a short-lived bearer token.
 
 	Returns:
 		A string JWT token for authenticated requests.
-	Returns None and logs error if login fails.
+	Raises:
+		ApiAuthenticationFailure: The credentials were rejected.
+		RepositoryApiError: Network problem, server error or an unexpected response.
+		RateLimitedError: Docker Hub is rate limiting us.
+
 	"""
-	login_url = "https://hub.docker.com/v2/users/login/"
+	response = _request("POST",f"{_API}/auth/token",json={"identifier": dockerhub_username, "secret": dockerhub_token})
-	response = requests.post(
+
-		login_url,
+	if response.status_code in (401, 403):
-		json={"username": dockerhub_username, "password": dockerhub_token}
+		raise ApiAuthenticationFailure(_SERVICE, f"token request rejected ({response.status_code})")
+	if response.status_code != 200:
+		raise RepositoryApiError(_SERVICE, f"token request returned unexpected status {response.status_code}")
+
+	try:
+		token = response.json().get("access_token")
+	except (ValueError, AttributeError) as e:
+		raise RepositoryApiError(_SERVICE, "token response was not the expected JSON") from e
+	if not token:
+		raise RepositoryApiError(_SERVICE, "token response did not contain an access_token")
+
+	logger.debug("[DockerHub] Access token obtained")
+	return token
+
+
+def _raise_not_found(repo_path: str, headers: dict[str, str], org: str, repo: str, tag: str) -> NoReturn:
+	"""A 404 on the tag could mean a missing tag or a missing repository. Ask which."""
+	response = _request("GET", repo_path, headers=headers)
+	if response.status_code == 200:
+		raise RepositoryTagNotFoundError(org, repo, tag)
+	if response.status_code == 404:
+		raise RepositoryNotFoundError(org, repo)
+	raise RepositoryApiError(
+		_SERVICE,
+		f"could not tell why {org}/{repo}:{tag} was not found (status {response.status_code})",
 	)
 
-	if response.status_code == 200:
+def find_package_version_with_tag(org: str, repo: str, tag: str) -> str:
-		token = response.json().get("token")
-		if token:
-			logger.debug("[DockerHub] Login successful, token obtained")
-			return token
-		else:
-			logger.error("[DockerHub] Login response did not contain token")
-			return ""
-	else:
-		logger.error(f"[DockerHub] Login failed: {response.status_code} - {response.text}")
-		return ""
-
-
-def find_package_version_with_tag(repo: str, tag: str) -> str:
 	"""
 	[DockerHub] Fetch the tag ID for a repository tag.
 
 	Authenticates with Docker Hub, queries the repository for the tag,
-	returns the tag ID if found, else None.
+
+		Raises:
+		ApiAuthenticationFailure: The token request failed, or access to the repository was denied.
+		RepositoryNotFoundError: The organisation/repository does not exist.
+		RepositoryTagNotFoundError: The repository exists but the tag does not.
+		RateLimitedError: Docker Hub is rate limiting us.
+		RepositoryApiError: Network problem, server error or an unexpected response.
+
 	"""
-	token = login_and_get_token()
+	repo_path = f"{_API}/namespaces/{org}/repositories/{repo}"
-	if not token:
+	url = f"{repo_path}/tags/{tag}"
-		logger.error(f"[DockerHub] Cannot fetch tag '{tag}' for repo '{repo}' without token")
-		return ""
 
-	headers = {"Authorization": f"JWT {token}"}
+	headers = {"Authorization": f"Bearer {_get_access_token()}"}
-	tags_url = f"https://hub.docker.com/v2/repositories/{repo}/tags/{tag}?page_size=1"
+	response = _request("GET", url, headers=headers)
 
-	response = requests.get(tags_url, headers=headers)
+	if response.status_code in (401, 403):
+		raise ApiAuthenticationFailure(_SERVICE, f"access to {org}/{repo} denied ({response.status_code})")
+	if response.status_code == 404:
+		_raise_not_found(repo_path, headers, org, repo, tag)
 	if response.status_code != 200:
-		logger.error(f"[DockerHub] Failed to fetch tag '{tag}' for repo '{repo}': {response.status_code} - {response.text}")
+		raise RepositoryApiError(_SERVICE, f"unexpected status {response.status_code} for {org}/{repo}:{tag}")
-		return ""
 
-	data = response.json()
+	try:
-	tag_id = data.get("id")
+		data = response.json()
-	if tag_id:
+	except ValueError as e:
-		logger.info(f"[DockerHub] Found tag '{tag}' for repo '{repo}', ID: {tag_id}")
+		raise RepositoryApiError(_SERVICE, f"response for {org}/{repo}:{tag} was not valid JSON") from e
-		return str(tag_id)
-	else:
-		logger.debug(f"[DockerHub] Tag '{tag}' not found for repo '{repo}'")
-		return ""
 
+	version = data.get(_VERSION_FIELD) if isinstance(data, dict) else None
+	if version is None:
+		raise RepositoryApiError(_SERVICE, f"response for {org}/{repo}:{tag} had no '{_VERSION_FIELD}' field")
+
+	logger.debug(f"[DockerHub] Found tag '{tag}' for repo '{org}/{repo}', {_VERSION_FIELD}: {version}")
+	return str(version)
+
+def _organisation_exists(org: str) -> bool:
+	headers = {"Authorization": f"Bearer {_get_access_token()}"}
+	response = _request("GET",f"{_API}/namespaces/{quote(org, safe='')}/repositories",params={"page_size": 1},headers=headers)
+
+	if response.status_code == 404:
+		return False
+	if response.status_code in (401,403):
+		raise ApiAuthenticationFailure(_SERVICE, f"listing the repositories of {org} was denied ({response.status_code})")
+	if response.status_code != 200:
+		raise RepositoryApiError(_SERVICE, f"unexpected status {response.status_code} when looking up {org}")
+
+	try:
+		count = response.json().get("count")
+	except (ValueError,AttributeError) as e:
+		raise RepositoryApiError(_SERVICE, f"response for {org} was not the expected JSON") from e
+	return count > 0
+
+def validate_repository_combination_and_return_latest_tag(org: str, repo: str, tag: str) -> str:
+	"""
+	[DockerHub] Check that an organisation, repository and tag exist together.
+
+	Raises:
+		OrganisationNotFoundError: The organisation has no repositories visible to us.
+		RepositoryNotFoundError: The organisation exists but the repository does not.
+		RepositoryTagNotFoundError: The repository exists but the tag does not.
+		ApiAuthenticationFailure: The token request failed, or access was denied.
+		RateLimitedError: Docker Hub is rate limiting us.
+		RepositoryApiError: Network problem, server error or an unexpected response.
+	"""
+	try:
+		return find_package_version_with_tag(org, repo, tag)
+	except RepositoryNotFoundError as e:
+		if not _organisation_exists(org):
+			raise OrganisationNotFoundError(org) from e
+		raise
 
 if __name__ == "__main__":
-	#print(find_package_version_with_tag("pihole/pihole", "latest"))
+	#print(find_package_version_with_tag("pihole","pihole", "latest"))
-	print(find_package_version_with_tag("library/mysql","8.4"))
+	#print(find_package_version_with_tag("library","mysql","8.4"))
+	from repository_errors import RepositoryError
+
+	try:
+		print(find_package_version_with_tag("pihole","pihole", "Alatest"))
+	except RepositoryError as e:
+		print(f"{type(e).__name__}: {e}")

src/github_api.py

@@ -1,50 +1,126 @@
+from urllib.parse import quote
 
 import requests
 
+from repository_errors import (
+	ApiAuthenticationFailure,
+	OrganisationNotFoundError,
+	RateLimitedError,
+	RepositoryApiError,
+	RepositoryNotFoundError,
+	RepositoryTagNotFoundError,
+)
 from secret_manager import get_secret
 from simple_logger_handler import setup_logger
 
 github_token = get_secret("GITHUB_TOKEN","/etc/secrets/github_token")
 logger = setup_logger(__name__)
 
+_SERVICE = "GitHub"
+_API = "https://api.github.com"
+_TIMEOUT = (3.05, 10)
+_PER_PAGE = 100
+_HEADERS = {
+	"Authorization": f"Bearer {github_token}",
+	"Accept": "application/vnd.github+json",
+}
+
+def _request(url: str, **kwargs) -> requests.Response:
+	"""GET a URL and turn transport problems, rate limits and 5xx into typed errors."""
+	try:
+		response = requests.get(url, headers=_HEADERS, timeout=_TIMEOUT, **kwargs)
+	except requests.RequestException as e:
+		raise RepositoryApiError(_SERVICE, f"request failed: {e}") from e
+
+	# GitHub signals rate limits as 429, or as 403 with an exhausted quota or a Retry-After header
+	rate_limited = response.status_code == 429 or (
+		response.status_code == 403
+		and (response.headers.get("X-RateLimit-Remaining") == "0" or "Retry-After" in response.headers)
+	)
+	if rate_limited:
+		retry_after = response.headers.get("Retry-After", "")
+		raise RateLimitedError(_SERVICE, int(retry_after) if retry_after.isdigit() else None)
+	if response.status_code >= 500:
+		raise RepositoryApiError(_SERVICE, f"server error {response.status_code}")
+	return response
+
 def find_package_version_with_tag(org: str, package: str, target_tag: str) -> str:
 	"""
-	[GitHub] Search for a container package version with the specified tag.
+	[GitHub] Return the ID of the container package version that carries the tag.
 
-	Iterates through GitHub Package Registry pages for a given org and package,
+	Pages through the versions of the package, newest first, until the tag is found.
-	returns the version ID matching the target_tag, or None if not found.
+
+	Raises:
+		ApiAuthenticationFailure: The token was rejected or lacks access to the package.
+		RepositoryNotFoundError: The package does not exist, or the token cannot see it.
+		RepositoryTagNotFoundError: Every page was searched and none carries the tag.
+		RateLimitedError: GitHub is rate limiting us.
+		RepositoryApiError: Network problem, server error or an unexpected response.
 	"""
-	headers = {
+	url = f"{_API}/orgs/{quote(org, safe='')}/packages/container/{quote(package, safe='')}/versions"
-		"Authorization": f"Bearer {github_token}",
-		"Accept": "application/vnd.github+json"
-	}
 
 	page = 1
-	per_page = 100
-
 	while True:
-		url = f"https://api.github.com/orgs/{org}/packages/container/{package}/versions"
+		response = _request(url, params={"per_page": _PER_PAGE, "page": page})
-		params = {"per_page": per_page, "page": page}
 
-		response = requests.get(url, headers=headers, params=params)
+		if response.status_code in (401, 403):
+			raise ApiAuthenticationFailure(_SERVICE, f"access to {org}/{package} denied ({response.status_code})")
+		if response.status_code == 404:
+			raise RepositoryNotFoundError(org, package)
 		if response.status_code != 200:
-			logger.error(f"[GitHub] Error {response.status_code} fetching {package}: {response.text}")
+			raise RepositoryApiError(_SERVICE, f"unexpected status {response.status_code} for {org}/{package} (page {page})")
-			return ""
 
-		versions = response.json()
+		try:
-		if not versions:
+			versions = response.json()
-			logger.debug(f"[GitHub] Reached end of pages — tag '{target_tag}' not found for {package}.")
+		except ValueError as e:
-			return ""
+			raise RepositoryApiError(_SERVICE, f"response for {org}/{package} was not valid JSON") from e
+		if not isinstance(versions, list):
+			raise RepositoryApiError(_SERVICE, f"response for {org}/{package} was not a list of versions")
 
 		for version in versions:
 			tags = version.get("metadata", {}).get("container", {}).get("tags", [])
 			if target_tag in tags:
-				logger.info(f"[GitHub] Found tag '{target_tag}' for {package} on page {page}, version ID: {version['id']}")
+				version_id = version.get("id")
+				if version_id is None:
+					raise RepositoryApiError(_SERVICE, f"version of {org}/{package}:{target_tag} had no id")
+				logger.info(f"[GitHub] Found tag '{target_tag}' for {package} on page {page}, version ID: {version_id}")
 				return str(version["id"])
 
+		if len(versions) < _PER_PAGE:
+			raise RepositoryTagNotFoundError(org, package, target_tag)
 		page += 1
 		logger.debug(f"[GitHub] Tag '{target_tag}' not found on page {page-1}, moving to page {page}")
 
+def _organisation_exists(org: str) -> bool:
+	"""Ask GitHub whether the organisation exists."""
+	response = _request(f"{_API}/orgs/{quote(org, safe='')}")
+	if response.status_code == 200:
+		return True
+	if response.status_code == 404:
+		return False
+	if response.status_code in (401, 403):
+		raise ApiAuthenticationFailure(_SERVICE, f"looking up organisation {org} was denied ({response.status_code})")
+	raise RepositoryApiError(_SERVICE, f"unexpected status {response.status_code} when looking up {org}")
+
+
+def validate_repository_combination_and_return_latest_tag(org: str, repo: str, tag: str) -> str:
+	"""
+	[GitHub] Check that an organisation, package and tag exist together, and return the tag's version ID.
+
+	Raises:
+		OrganisationNotFoundError: The organisation does not exist.
+		RepositoryNotFoundError: The organisation exists but the package does not (or is not visible to the token).
+		RepositoryTagNotFoundError: The package exists but no version carries the tag.
+		ApiAuthenticationFailure: The token was rejected or lacks access.
+		RateLimitedError: GitHub is rate limiting us.
+		RepositoryApiError: Network problem, server error or an unexpected response.
+	"""
+	try:
+		return find_package_version_with_tag(org, repo, tag)
+	except RepositoryNotFoundError as e:
+		if not _organisation_exists(org):
+			raise OrganisationNotFoundError(org) from e
+		raise
 
 if __name__ == "__main__":
-	find_package_version_with_tag("Suwayomi", "tachidesk", "stable")
+	validate_repository_combination_and_return_latest_tag("Suwayomis", "tachidesk", "stable")

src/repository_errors.py

@@ -0,0 +1,92 @@
+class RepositoryError(Exception):
+	status_code = 502
+	code = "drq_upstream_error"
+	client_visible = False
+
+	@property
+	def client_message(self) -> str:
+		return str(self) if self.client_visible else "The repository could not be checked right now, please try again later"
+
+class InvalidOrganisationError(RepositoryError, ValueError):
+	status_code = 422
+	code = "drq_invalid_organisation"
+	client_visible = True
+	def __init__(self, org:str):
+		super().__init__(f"Invalid organisation name: {org!r}")
+
+class InvalidRepositoryError(RepositoryError, ValueError):
+	status_code = 422
+	code = "drq_invalid_repository"
+	client_visible = True
+	def __init__(self, repo:str):
+		super().__init__(f"Invalid repository name: {repo!r}")
+
+class InvalidTagError(RepositoryError, ValueError):
+	status_code = 422
+	code = "drq_invalid_tag"
+	client_visible = True
+	def __init__(self, tag:str):
+		super().__init__(f"Invalid tag: {tag!r}")
+
+class ApiAuthenticationFailure(RepositoryError):
+	"""The registry rejected our credentials or token.
+
+	Not specific to one repository: every lookup against the same registry
+	will normally fail the same way.
+	"""
+
+	def __init__(self, service: str, detail: str = ""):
+		message = f"{service} authentication failed"
+		super().__init__(f"{message}: {detail}" if detail else message)
+
+
+class RepositoryNotFoundError(RepositoryError):
+	"""The organisation or repository does not exist (or is not visible to us)."""
+	status_code = 422
+	code = "drq_repository_not_found"
+	client_visible = True
+	def __init__(self, org: str, repo: str):
+		self.org = org
+		self.repo = repo
+		super().__init__(f"Repository not found: {org}/{repo}")
+
+
+class RepositoryTagNotFoundError(RepositoryError):
+	"""The repository exists but the tag does not. It may have been deleted."""
+	status_code = 422
+	code = "drq_tag_not_found"
+	client_visible = True
+	def __init__(self, org: str, repo: str, tag: str):
+		self.org = org
+		self.repo = repo
+		self.tag = tag
+		super().__init__(f"Tag not found: {org}/{repo}:{tag}")
+
+
+class RepositoryApiError(RepositoryError):
+	"""Temporary or unexpected API problem (network, timeout, 5xx, bad response).
+
+	Nothing is wrong with the repository itself, so retry on the next run.
+	"""
+
+	def __init__(self, service: str, detail: str):
+		super().__init__(f"{service} API error: {detail}")
+
+
+class RateLimitedError(RepositoryApiError):
+	"""The registry asked us to slow down. Stop the run instead of continuing."""
+	status_code = 503
+	code = "drq_rate_limited"
+	def __init__(self, service: str, retry_after: int | None = None):
+		self.retry_after = retry_after
+		hint = f", retry after {retry_after}s" if retry_after is not None else ""
+		super().__init__(service, f"rate limited{hint}")
+
+class OrganisationNotFoundError(RepositoryError):
+	"""The organisation (namespace) has no repositories visible to us, so it most likely does not exist."""
+	status_code = 422
+	code = "drq_organisation_not_found"
+	client_visible = True
+	def __init__(self, org: str):
+		self.org = org
+		super().__init__(f"Organisation not found: {org}")