notifier/backend-push-notifications
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Unified logging behaviour

Browse Source 
- Logger doesn't start with log level DEBUG by default, instead reads a environment variable
- Added extensive debug logging
- Wrote a readme
- Changed database healthcheck loop to only catch pymsql errors
This commit is contained in:
florian committed 2025-10-17 10:58:37 +02:00
1 parent 37476c814e
commit b272e069d4
6 files changed
+120 -35
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/rabbitmq_handler.py
+35 -17
View File
@@ -23,8 +23,9 @@ RABBITMQ_URL = f"amqp://{rmq_username}:{rmq_password}@{rmq_host}/{rmq_vhost}"
class RabbitMQConsumer:
def __init__(self, url=RABBITMQ_URL, db_manager=db_manager, exchange_name=rmq_exchange):
def __init__(self, url=RABBITMQ_URL, db_manager=db_manager, exchange_name=rmq_exchange,rmq_host=rmq_host):
self.url = url
self.rmq_host = rmq_host
self.exchange_name = exchange_name
self.connection: aio_pika.RobustConnection | None = None
self.channel: aio_pika.RobustChannel | None = None
@@ -37,8 +38,11 @@ class RabbitMQConsumer:
self.dlq_queue_name = "notifications_dlq"
self.max_retries = 5
self.retry_ttl = 5 * 60 * 10000
logger.debug(f"[RabbitMQ] Initialized consumer for exchange {self.exchange_name}")
async def connect(self):
logger.info(f"[RabbitMQ] Connecting to {self.rmq_host}...")
self.connection = await aio_pika.connect_robust(self.url)
self.channel = await self.connection.channel()
self.exchange = await self.channel.declare_exchange(
@@ -46,6 +50,8 @@ class RabbitMQConsumer:
)
self.queue = await self.channel.declare_queue(self.queue_name, durable=True)
await self.queue.bind(self.exchange, routing_key="notify.user.*")
logger.debug(f"[RabbitMQ] Bound queue '{self.queue_name}' to exchange '{self.exchange_name}'")
retry_queue_args = {
"x-message-ttl": self.retry_ttl,
"x-dead-letter-exchange": self.exchange_name,
@@ -61,7 +67,7 @@ class RabbitMQConsumer:
asyncio.create_task(metrics.update_queue_gauge(self.channel, self.retry_queue_name))
asyncio.create_task(metrics.update_queue_gauge(self.channel, self.dlq_queue_name))
logger.info("[RMQ] Connected, exchange, retry, and DLQ queues ready.")
logger.info("[RabbitMQ] Connected, exchange, retry, and DLQ queues ready.")
async def send_message_to_dlq(self, uuid: str, message: aio_pika.IncomingMessage):
data = json.loads(message.body.decode())
@@ -74,7 +80,7 @@ class RabbitMQConsumer:
),
routing_key=self.dlq_queue_name
)
logger.warning(f"Message sent to DLQ: {data['uuid']}")
logger.warning(f"[RabbitMQ] Message sent to DLQ: {data['uuid']}")
metrics.MSG_FAILED.inc()
async def send_message_to_retry_queue(self, uuid: str, message: aio_pika.IncomingMessage):
@@ -88,8 +94,10 @@ class RabbitMQConsumer:
uuid=uuid,
retry_count=retry_count
).inc()
logger.debug(f"[RabbitMQ] Metrics updated for retry: uuid={uuid}, retry_count={retry_count}")
if retry_count > self.max_retries:
logger.debug(f"[RabbitMQ] Retry count exceeded max_retries={self.max_retries}, sending to DLQ")
await self.send_message_to_dlq(uuid, message)
return
await self.channel.default_exchange.publish(
@@ -100,23 +108,25 @@ class RabbitMQConsumer:
),
routing_key=self.retry_queue_name
)
logger.info(f"Message requeued for retry #{retry_count}")
logger.info(f"[RabbitMQ] Message requeued for retry #{retry_count}")
async def handle_message(self, message: aio_pika.IncomingMessage):
if self._closing:
logger.debug("[RabbitMQ] Skipping message because consumer is closing")
return
async with message.process():
try:
data = json.loads(message.body.decode())
logger.info(f"[RMQ] Received: {data}")
logger.debug(f"[RabbitMQ] Received message: {data}")
uuid = data.get("uuid")
if uuid:
encrypted_tokens = await database_lookup_by_uuid(uuid, db_manager)
else:
encrypted_tokens = await database_lookup_by_user_id(message.routing_key, db_manager)
if not encrypted_tokens:
logger.warning(f"No push tokens found for user {message.routing_key}")
logger.warning(f"[RabbitMQ] No push tokens found for routing key {message.routing_key}")
return
token_map = {row["uuid"]: row["token"].decode() for row in encrypted_tokens}
@@ -127,13 +137,13 @@ class RabbitMQConsumer:
await self.validate_delivery(response,message)
except json.JSONDecodeError as e:
logger.error(f"[RMQ] Bad message, discarding: {e}")
logger.error(f"[RabbitMQ] Bad message, discarding: {e}", exc_info=True)
await message.nack(requeue=False)
except AMQPException as e:
logger.error(f"[RMQ] RabbitMQ error: {e}")
logger.error(f"[RabbitMQ] RabbitMQ error: {e}", exc_info=True)
await message.nack(requeue=True)
except Exception as e:
logger.critical(f"[RMQ] Fatal error: {e}")
logger.critical(f"[RabbitMQ] Fatal error: {e}", exc_info=True)
raise
async def validate_delivery(self,response,message: aio_pika.IncomingMessage):
@@ -143,45 +153,53 @@ class RabbitMQConsumer:
api_status = data_list[0].get("status") if data_list else None
if status == "ok" and api_status == "ok":
logger.info(f"Notification delivered successfully to {uuid}")
logger.info(f"[RabbitMQ] Notification delivered successfully to {uuid}")
metrics.MSG_PUBLISHED.inc()
logger.debug(f"[RabbitMQ] Metrics updated for published message: uuid={uuid}")
if status == "ok" and api_status == "error":
api_error = data_list[0].get("details", {}).get("error")
if api_error == "DeviceNotRegistered":
expired = await remove_inactive_push_token(uuid, self.db_manager)
if expired:
logger.info(f"Device no longer registered for uuid {uuid}, setting as expired")
logger.info(f"[RabbitMQ] Device no longer registered for uuid {uuid}, marked expired")
else:
logger.error(f"Failed expiring token for uuid: {uuid}")
logger.error(f"[RabbitMQ] Failed expiring token for uuid: {uuid}")
else:
await self.send_message_to_dlq(uuid, message)
logger.debug(f"[RabbitMQ] Message sent to DLQ due to API error: uuid={uuid}")
if status == "error":
await self.send_message_to_dlq(uuid, message)
logger.debug(f"[RabbitMQ] Message sent to DLQ due to status=error: uuid={uuid}")
if status == "failure":
await self.send_message_to_retry_queue(uuid, message)
logger.debug(f"[RabbitMQ] Message requeued for retry due to status=failure: uuid={uuid}")
async def consume(self):
if self._closing:
logger.debug("[RabbitMQ] Consumer is closing, not starting consume")
return
if not self.queue:
raise RuntimeError("Queue not initialized")
raise RuntimeError("[RabbitMQ] Queue not initialized")
await self.queue.consume(self.handle_message, no_ack=False)
logger.info("[RMQ] Consuming messages...")
logger.info("[RabbitMQ] Consuming messages...")
async def close(self):
self._closing = True
if self.connection:
await self.connection.close()
logger.info("[aio-pika] Connection closed.")
logger.info("[RabbitMQ] Connection closed")
async def main():
logger.info("Starting application")
logger.info("[APP] Starting application")
await db_manager.connect()
consumer = RabbitMQConsumer(db_manager=db_manager)
await consumer.connect()
@@ -192,7 +210,7 @@ async def main():
asyncio.get_running_loop().add_signal_handler(sig, stop_event.set)
await stop_event.wait()
logger.info("Stopping application")
logger.info("[APP] Stopping application")
await consumer.close()
await db_manager.close()