diff --git a/src/dockerhub_api.py b/src/dockerhub_api.py index 21bbb40..501b507 100644 --- a/src/dockerhub_api.py +++ b/src/dockerhub_api.py @@ -1,6 +1,16 @@ +from typing import NoReturn +from urllib.parse import quote import requests +from repository_errors import ( + ApiAuthenticationFailure, + OrganisationNotFoundError, + RateLimitedError, + RepositoryApiError, + RepositoryNotFoundError, + RepositoryTagNotFoundError, +) from secret_manager import get_secret from simple_logger_handler import setup_logger @@ -9,63 +19,149 @@ dockerhub_username = get_secret("DOCKERHUB_USERNAME","/etc/secrets/dockerhub_use logger = setup_logger(__name__) -def login_and_get_token() -> str: +_SERVICE = "Docker Hub" +_API = "https://hub.docker.com/v2" +_TIMEOUT = (3.05, 10) +_VERSION_FIELD = "id" + +def _request(method: str, url: str, **kwargs) -> requests.Response: + """Send a request and turn transport problems, rate limits and 5xx into typed errors.""" + try: + response = requests.request(method, url, timeout=_TIMEOUT, **kwargs) + except requests.RequestException as e: + raise RepositoryApiError(_SERVICE, f"request failed: {e}") from e + + if response.status_code == 429: + retry_after = response.headers.get("Retry-After", "") + raise RateLimitedError(_SERVICE, int(retry_after) if retry_after.isdigit() else None) + if response.status_code >= 500: + raise RepositoryApiError(_SERVICE, f"server error {response.status_code}") + return response + + +def _get_access_token() -> str: """ - [DockerHub] Login to Docker Hub and retrieve a JWT token. + [DockerHub] Exchange the username and personal access token for a short-lived bearer token. Returns: A string JWT token for authenticated requests. - Returns None and logs error if login fails. + Raises: + ApiAuthenticationFailure: The credentials were rejected. + RepositoryApiError: Network problem, server error or an unexpected response. + RateLimitedError: Docker Hub is rate limiting us. + """ - login_url = "https://hub.docker.com/v2/users/login/" - response = requests.post( - login_url, - json={"username": dockerhub_username, "password": dockerhub_token} + response = _request("POST",f"{_API}/auth/token",json={"identifier": dockerhub_username, "secret": dockerhub_token}) + + if response.status_code in (401, 403): + raise ApiAuthenticationFailure(_SERVICE, f"token request rejected ({response.status_code})") + if response.status_code != 200: + raise RepositoryApiError(_SERVICE, f"token request returned unexpected status {response.status_code}") + + try: + token = response.json().get("access_token") + except (ValueError, AttributeError) as e: + raise RepositoryApiError(_SERVICE, "token response was not the expected JSON") from e + if not token: + raise RepositoryApiError(_SERVICE, "token response did not contain an access_token") + + logger.debug("[DockerHub] Access token obtained") + return token + + +def _raise_not_found(repo_path: str, headers: dict[str, str], org: str, repo: str, tag: str) -> NoReturn: + """A 404 on the tag could mean a missing tag or a missing repository. Ask which.""" + response = _request("GET", repo_path, headers=headers) + if response.status_code == 200: + raise RepositoryTagNotFoundError(org, repo, tag) + if response.status_code == 404: + raise RepositoryNotFoundError(org, repo) + raise RepositoryApiError( + _SERVICE, + f"could not tell why {org}/{repo}:{tag} was not found (status {response.status_code})", ) - if response.status_code == 200: - token = response.json().get("token") - if token: - logger.debug("[DockerHub] Login successful, token obtained") - return token - else: - logger.error("[DockerHub] Login response did not contain token") - return "" - else: - logger.error(f"[DockerHub] Login failed: {response.status_code} - {response.text}") - return "" - - -def find_package_version_with_tag(repo: str, tag: str) -> str: +def find_package_version_with_tag(org: str, repo: str, tag: str) -> str: """ [DockerHub] Fetch the tag ID for a repository tag. Authenticates with Docker Hub, queries the repository for the tag, - returns the tag ID if found, else None. + + Raises: + ApiAuthenticationFailure: The token request failed, or access to the repository was denied. + RepositoryNotFoundError: The organisation/repository does not exist. + RepositoryTagNotFoundError: The repository exists but the tag does not. + RateLimitedError: Docker Hub is rate limiting us. + RepositoryApiError: Network problem, server error or an unexpected response. + """ - token = login_and_get_token() - if not token: - logger.error(f"[DockerHub] Cannot fetch tag '{tag}' for repo '{repo}' without token") - return "" + repo_path = f"{_API}/namespaces/{org}/repositories/{repo}" + url = f"{repo_path}/tags/{tag}" - headers = {"Authorization": f"JWT {token}"} - tags_url = f"https://hub.docker.com/v2/repositories/{repo}/tags/{tag}?page_size=1" + headers = {"Authorization": f"Bearer {_get_access_token()}"} + response = _request("GET", url, headers=headers) - response = requests.get(tags_url, headers=headers) + if response.status_code in (401, 403): + raise ApiAuthenticationFailure(_SERVICE, f"access to {org}/{repo} denied ({response.status_code})") + if response.status_code == 404: + _raise_not_found(repo_path, headers, org, repo, tag) if response.status_code != 200: - logger.error(f"[DockerHub] Failed to fetch tag '{tag}' for repo '{repo}': {response.status_code} - {response.text}") - return "" + raise RepositoryApiError(_SERVICE, f"unexpected status {response.status_code} for {org}/{repo}:{tag}") - data = response.json() - tag_id = data.get("id") - if tag_id: - logger.info(f"[DockerHub] Found tag '{tag}' for repo '{repo}', ID: {tag_id}") - return str(tag_id) - else: - logger.debug(f"[DockerHub] Tag '{tag}' not found for repo '{repo}'") - return "" + try: + data = response.json() + except ValueError as e: + raise RepositoryApiError(_SERVICE, f"response for {org}/{repo}:{tag} was not valid JSON") from e + version = data.get(_VERSION_FIELD) if isinstance(data, dict) else None + if version is None: + raise RepositoryApiError(_SERVICE, f"response for {org}/{repo}:{tag} had no '{_VERSION_FIELD}' field") + + logger.debug(f"[DockerHub] Found tag '{tag}' for repo '{org}/{repo}', {_VERSION_FIELD}: {version}") + return str(version) + +def _organisation_exists(org: str) -> bool: + headers = {"Authorization": f"Bearer {_get_access_token()}"} + response = _request("GET",f"{_API}/namespaces/{quote(org, safe='')}/repositories",params={"page_size": 1},headers=headers) + + if response.status_code == 404: + return False + if response.status_code in (401,403): + raise ApiAuthenticationFailure(_SERVICE, f"listing the repositories of {org} was denied ({response.status_code})") + if response.status_code != 200: + raise RepositoryApiError(_SERVICE, f"unexpected status {response.status_code} when looking up {org}") + + try: + count = response.json().get("count") + except (ValueError,AttributeError) as e: + raise RepositoryApiError(_SERVICE, f"response for {org} was not the expected JSON") from e + return count > 0 + +def validate_repository_combination_and_return_latest_tag(org: str, repo: str, tag: str) -> str: + """ + [DockerHub] Check that an organisation, repository and tag exist together. + + Raises: + OrganisationNotFoundError: The organisation has no repositories visible to us. + RepositoryNotFoundError: The organisation exists but the repository does not. + RepositoryTagNotFoundError: The repository exists but the tag does not. + ApiAuthenticationFailure: The token request failed, or access was denied. + RateLimitedError: Docker Hub is rate limiting us. + RepositoryApiError: Network problem, server error or an unexpected response. + """ + try: + return find_package_version_with_tag(org, repo, tag) + except RepositoryNotFoundError as e: + if not _organisation_exists(org): + raise OrganisationNotFoundError(org) from e + raise if __name__ == "__main__": - #print(find_package_version_with_tag("pihole/pihole", "latest")) - print(find_package_version_with_tag("library/mysql","8.4")) \ No newline at end of file + #print(find_package_version_with_tag("pihole","pihole", "latest")) + #print(find_package_version_with_tag("library","mysql","8.4")) + from repository_errors import RepositoryError + + try: + print(find_package_version_with_tag("pihole","pihole", "Alatest")) + except RepositoryError as e: + print(f"{type(e).__name__}: {e}") diff --git a/src/github_api.py b/src/github_api.py index c2806d2..e2cc466 100644 --- a/src/github_api.py +++ b/src/github_api.py @@ -1,50 +1,126 @@ +from urllib.parse import quote import requests +from repository_errors import ( + ApiAuthenticationFailure, + OrganisationNotFoundError, + RateLimitedError, + RepositoryApiError, + RepositoryNotFoundError, + RepositoryTagNotFoundError, +) from secret_manager import get_secret from simple_logger_handler import setup_logger github_token = get_secret("GITHUB_TOKEN","/etc/secrets/github_token") logger = setup_logger(__name__) +_SERVICE = "GitHub" +_API = "https://api.github.com" +_TIMEOUT = (3.05, 10) +_PER_PAGE = 100 +_HEADERS = { + "Authorization": f"Bearer {github_token}", + "Accept": "application/vnd.github+json", +} + +def _request(url: str, **kwargs) -> requests.Response: + """GET a URL and turn transport problems, rate limits and 5xx into typed errors.""" + try: + response = requests.get(url, headers=_HEADERS, timeout=_TIMEOUT, **kwargs) + except requests.RequestException as e: + raise RepositoryApiError(_SERVICE, f"request failed: {e}") from e + + # GitHub signals rate limits as 429, or as 403 with an exhausted quota or a Retry-After header + rate_limited = response.status_code == 429 or ( + response.status_code == 403 + and (response.headers.get("X-RateLimit-Remaining") == "0" or "Retry-After" in response.headers) + ) + if rate_limited: + retry_after = response.headers.get("Retry-After", "") + raise RateLimitedError(_SERVICE, int(retry_after) if retry_after.isdigit() else None) + if response.status_code >= 500: + raise RepositoryApiError(_SERVICE, f"server error {response.status_code}") + return response + def find_package_version_with_tag(org: str, package: str, target_tag: str) -> str: """ - [GitHub] Search for a container package version with the specified tag. + [GitHub] Return the ID of the container package version that carries the tag. - Iterates through GitHub Package Registry pages for a given org and package, - returns the version ID matching the target_tag, or None if not found. + Pages through the versions of the package, newest first, until the tag is found. + + Raises: + ApiAuthenticationFailure: The token was rejected or lacks access to the package. + RepositoryNotFoundError: The package does not exist, or the token cannot see it. + RepositoryTagNotFoundError: Every page was searched and none carries the tag. + RateLimitedError: GitHub is rate limiting us. + RepositoryApiError: Network problem, server error or an unexpected response. """ - headers = { - "Authorization": f"Bearer {github_token}", - "Accept": "application/vnd.github+json" - } + url = f"{_API}/orgs/{quote(org, safe='')}/packages/container/{quote(package, safe='')}/versions" page = 1 - per_page = 100 - while True: - url = f"https://api.github.com/orgs/{org}/packages/container/{package}/versions" - params = {"per_page": per_page, "page": page} + response = _request(url, params={"per_page": _PER_PAGE, "page": page}) - response = requests.get(url, headers=headers, params=params) + if response.status_code in (401, 403): + raise ApiAuthenticationFailure(_SERVICE, f"access to {org}/{package} denied ({response.status_code})") + if response.status_code == 404: + raise RepositoryNotFoundError(org, package) if response.status_code != 200: - logger.error(f"[GitHub] Error {response.status_code} fetching {package}: {response.text}") - return "" + raise RepositoryApiError(_SERVICE, f"unexpected status {response.status_code} for {org}/{package} (page {page})") - versions = response.json() - if not versions: - logger.debug(f"[GitHub] Reached end of pages — tag '{target_tag}' not found for {package}.") - return "" + try: + versions = response.json() + except ValueError as e: + raise RepositoryApiError(_SERVICE, f"response for {org}/{package} was not valid JSON") from e + if not isinstance(versions, list): + raise RepositoryApiError(_SERVICE, f"response for {org}/{package} was not a list of versions") for version in versions: tags = version.get("metadata", {}).get("container", {}).get("tags", []) if target_tag in tags: - logger.info(f"[GitHub] Found tag '{target_tag}' for {package} on page {page}, version ID: {version['id']}") + version_id = version.get("id") + if version_id is None: + raise RepositoryApiError(_SERVICE, f"version of {org}/{package}:{target_tag} had no id") + logger.info(f"[GitHub] Found tag '{target_tag}' for {package} on page {page}, version ID: {version_id}") return str(version["id"]) + if len(versions) < _PER_PAGE: + raise RepositoryTagNotFoundError(org, package, target_tag) page += 1 logger.debug(f"[GitHub] Tag '{target_tag}' not found on page {page-1}, moving to page {page}") +def _organisation_exists(org: str) -> bool: + """Ask GitHub whether the organisation exists.""" + response = _request(f"{_API}/orgs/{quote(org, safe='')}") + if response.status_code == 200: + return True + if response.status_code == 404: + return False + if response.status_code in (401, 403): + raise ApiAuthenticationFailure(_SERVICE, f"looking up organisation {org} was denied ({response.status_code})") + raise RepositoryApiError(_SERVICE, f"unexpected status {response.status_code} when looking up {org}") + + +def validate_repository_combination_and_return_latest_tag(org: str, repo: str, tag: str) -> str: + """ + [GitHub] Check that an organisation, package and tag exist together, and return the tag's version ID. + + Raises: + OrganisationNotFoundError: The organisation does not exist. + RepositoryNotFoundError: The organisation exists but the package does not (or is not visible to the token). + RepositoryTagNotFoundError: The package exists but no version carries the tag. + ApiAuthenticationFailure: The token was rejected or lacks access. + RateLimitedError: GitHub is rate limiting us. + RepositoryApiError: Network problem, server error or an unexpected response. + """ + try: + return find_package_version_with_tag(org, repo, tag) + except RepositoryNotFoundError as e: + if not _organisation_exists(org): + raise OrganisationNotFoundError(org) from e + raise if __name__ == "__main__": - find_package_version_with_tag("Suwayomi", "tachidesk", "stable") \ No newline at end of file + validate_repository_combination_and_return_latest_tag("Suwayomis", "tachidesk", "stable") \ No newline at end of file diff --git a/src/repository_errors.py b/src/repository_errors.py new file mode 100644 index 0000000..a986dd7 --- /dev/null +++ b/src/repository_errors.py @@ -0,0 +1,92 @@ +class RepositoryError(Exception): + status_code = 502 + code = "drq_upstream_error" + client_visible = False + + @property + def client_message(self) -> str: + return str(self) if self.client_visible else "The repository could not be checked right now, please try again later" + +class InvalidOrganisationError(RepositoryError, ValueError): + status_code = 422 + code = "drq_invalid_organisation" + client_visible = True + def __init__(self, org:str): + super().__init__(f"Invalid organisation name: {org!r}") + +class InvalidRepositoryError(RepositoryError, ValueError): + status_code = 422 + code = "drq_invalid_repository" + client_visible = True + def __init__(self, repo:str): + super().__init__(f"Invalid repository name: {repo!r}") + +class InvalidTagError(RepositoryError, ValueError): + status_code = 422 + code = "drq_invalid_tag" + client_visible = True + def __init__(self, tag:str): + super().__init__(f"Invalid tag: {tag!r}") + +class ApiAuthenticationFailure(RepositoryError): + """The registry rejected our credentials or token. + + Not specific to one repository: every lookup against the same registry + will normally fail the same way. + """ + + def __init__(self, service: str, detail: str = ""): + message = f"{service} authentication failed" + super().__init__(f"{message}: {detail}" if detail else message) + + +class RepositoryNotFoundError(RepositoryError): + """The organisation or repository does not exist (or is not visible to us).""" + status_code = 422 + code = "drq_repository_not_found" + client_visible = True + def __init__(self, org: str, repo: str): + self.org = org + self.repo = repo + super().__init__(f"Repository not found: {org}/{repo}") + + +class RepositoryTagNotFoundError(RepositoryError): + """The repository exists but the tag does not. It may have been deleted.""" + status_code = 422 + code = "drq_tag_not_found" + client_visible = True + def __init__(self, org: str, repo: str, tag: str): + self.org = org + self.repo = repo + self.tag = tag + super().__init__(f"Tag not found: {org}/{repo}:{tag}") + + +class RepositoryApiError(RepositoryError): + """Temporary or unexpected API problem (network, timeout, 5xx, bad response). + + Nothing is wrong with the repository itself, so retry on the next run. + """ + + def __init__(self, service: str, detail: str): + super().__init__(f"{service} API error: {detail}") + + +class RateLimitedError(RepositoryApiError): + """The registry asked us to slow down. Stop the run instead of continuing.""" + status_code = 503 + code = "drq_rate_limited" + def __init__(self, service: str, retry_after: int | None = None): + self.retry_after = retry_after + hint = f", retry after {retry_after}s" if retry_after is not None else "" + super().__init__(service, f"rate limited{hint}") + +class OrganisationNotFoundError(RepositoryError): + """The organisation (namespace) has no repositories visible to us, so it most likely does not exist.""" + status_code = 422 + code = "drq_organisation_not_found" + client_visible = True + def __init__(self, org: str): + self.org = org + super().__init__(f"Organisation not found: {org}") \ No newline at end of file