RMQ: Added Prometheus metrics and cleaned up code for deployment

This commit is contained in:
florian committed 2025-10-14 21:54:23 +02:00
1 parent f8b66d742f
commit 37476c814e
5 files changed
+77 -12

No files matched your search

+17 -3
View File
@@ -8,6 +8,7 @@ import json
from db import db_manager
from send_notification import send_notification
import signal
import metrics
logger = setup_logger(__name__)
@@ -31,6 +32,7 @@ class RabbitMQConsumer:
self.queue: aio_pika.Queue | None = None
self._closing = False
self.db_manager = db_manager
self.queue_name = "notifications"
self.retry_queue_name = "notifications_retry"
self.dlq_queue_name = "notifications_dlq"
self.max_retries = 5
@@ -42,7 +44,7 @@ class RabbitMQConsumer:
self.exchange = await self.channel.declare_exchange(
self.exchange_name, aio_pika.ExchangeType.TOPIC, durable=True
)
self.queue = await self.channel.declare_queue("notifications", durable=True)
self.queue = await self.channel.declare_queue(self.queue_name, durable=True)
await self.queue.bind(self.exchange, routing_key="notify.user.*")
retry_queue_args = {
"x-message-ttl": self.retry_ttl,
@@ -54,6 +56,11 @@ class RabbitMQConsumer:
)
await self.channel.declare_queue(self.dlq_queue_name, durable=True)
metrics.start_metrics_server(port=9000)
asyncio.create_task(metrics.update_queue_gauge(self.channel, self.queue_name))
asyncio.create_task(metrics.update_queue_gauge(self.channel, self.retry_queue_name))
asyncio.create_task(metrics.update_queue_gauge(self.channel, self.dlq_queue_name))
logger.info("[RMQ] Connected, exchange, retry, and DLQ queues ready.")
async def send_message_to_dlq(self, uuid: str, message: aio_pika.IncomingMessage):
@@ -68,6 +75,7 @@ class RabbitMQConsumer:
routing_key=self.dlq_queue_name
)
logger.warning(f"Message sent to DLQ: {data['uuid']}")
metrics.MSG_FAILED.inc()
async def send_message_to_retry_queue(self, uuid: str, message: aio_pika.IncomingMessage):
data = json.loads(message.body.decode())
@@ -75,6 +83,12 @@ class RabbitMQConsumer:
retry_count = data.get("retry_count", 0) + 1
data["retry_count"] = retry_count
metrics.MSG_RETRY.labels(
queue_name=self.retry_queue_name,
uuid=uuid,
retry_count=retry_count
).inc()
if retry_count > self.max_retries:
await self.send_message_to_dlq(uuid, message)
return
@@ -111,8 +125,7 @@ class RabbitMQConsumer:
token_map[uuid] = decrypted_token
response = await send_notification(message=data,push_tokens=token_map)
await self.validate_delivery(response,message)
logger.debug(response)
except json.JSONDecodeError as e:
logger.error(f"[RMQ] Bad message, discarding: {e}")
await message.nack(requeue=False)
@@ -131,6 +144,7 @@ class RabbitMQConsumer:
if status == "ok" and api_status == "ok":
logger.info(f"Notification delivered successfully to {uuid}")
metrics.MSG_PUBLISHED.inc()
if status == "ok" and api_status == "error":
api_error = data_list[0].get("details", {}).get("error")