RMQ: Added Prometheus metrics and cleaned up code for deployment
This commit is contained in:
1 parent
f8b66d742f
commit
37476c814e
5 files changed
+77
-12
No files matched your search
+17
-3
@@ -8,6 +8,7 @@ import json
|
||||
from db import db_manager
|
||||
from send_notification import send_notification
|
||||
import signal
|
||||
import metrics
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
@@ -31,6 +32,7 @@ class RabbitMQConsumer:
|
||||
self.queue: aio_pika.Queue | None = None
|
||||
self._closing = False
|
||||
self.db_manager = db_manager
|
||||
self.queue_name = "notifications"
|
||||
self.retry_queue_name = "notifications_retry"
|
||||
self.dlq_queue_name = "notifications_dlq"
|
||||
self.max_retries = 5
|
||||
@@ -42,7 +44,7 @@ class RabbitMQConsumer:
|
||||
self.exchange = await self.channel.declare_exchange(
|
||||
self.exchange_name, aio_pika.ExchangeType.TOPIC, durable=True
|
||||
)
|
||||
self.queue = await self.channel.declare_queue("notifications", durable=True)
|
||||
self.queue = await self.channel.declare_queue(self.queue_name, durable=True)
|
||||
await self.queue.bind(self.exchange, routing_key="notify.user.*")
|
||||
retry_queue_args = {
|
||||
"x-message-ttl": self.retry_ttl,
|
||||
@@ -54,6 +56,11 @@ class RabbitMQConsumer:
|
||||
)
|
||||
await self.channel.declare_queue(self.dlq_queue_name, durable=True)
|
||||
|
||||
metrics.start_metrics_server(port=9000)
|
||||
asyncio.create_task(metrics.update_queue_gauge(self.channel, self.queue_name))
|
||||
asyncio.create_task(metrics.update_queue_gauge(self.channel, self.retry_queue_name))
|
||||
asyncio.create_task(metrics.update_queue_gauge(self.channel, self.dlq_queue_name))
|
||||
|
||||
logger.info("[RMQ] Connected, exchange, retry, and DLQ queues ready.")
|
||||
|
||||
async def send_message_to_dlq(self, uuid: str, message: aio_pika.IncomingMessage):
|
||||
@@ -68,6 +75,7 @@ class RabbitMQConsumer:
|
||||
routing_key=self.dlq_queue_name
|
||||
)
|
||||
logger.warning(f"Message sent to DLQ: {data['uuid']}")
|
||||
metrics.MSG_FAILED.inc()
|
||||
|
||||
async def send_message_to_retry_queue(self, uuid: str, message: aio_pika.IncomingMessage):
|
||||
data = json.loads(message.body.decode())
|
||||
@@ -75,6 +83,12 @@ class RabbitMQConsumer:
|
||||
retry_count = data.get("retry_count", 0) + 1
|
||||
data["retry_count"] = retry_count
|
||||
|
||||
metrics.MSG_RETRY.labels(
|
||||
queue_name=self.retry_queue_name,
|
||||
uuid=uuid,
|
||||
retry_count=retry_count
|
||||
).inc()
|
||||
|
||||
if retry_count > self.max_retries:
|
||||
await self.send_message_to_dlq(uuid, message)
|
||||
return
|
||||
@@ -111,8 +125,7 @@ class RabbitMQConsumer:
|
||||
token_map[uuid] = decrypted_token
|
||||
response = await send_notification(message=data,push_tokens=token_map)
|
||||
await self.validate_delivery(response,message)
|
||||
logger.debug(response)
|
||||
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"[RMQ] Bad message, discarding: {e}")
|
||||
await message.nack(requeue=False)
|
||||
@@ -131,6 +144,7 @@ class RabbitMQConsumer:
|
||||
|
||||
if status == "ok" and api_status == "ok":
|
||||
logger.info(f"Notification delivered successfully to {uuid}")
|
||||
metrics.MSG_PUBLISHED.inc()
|
||||
|
||||
if status == "ok" and api_status == "error":
|
||||
api_error = data_list[0].get("details", {}).get("error")
|
||||
|
||||
Reference in new issue
Block a user