Initial draft

src/rabbitmq_handler.py

@@ -0,0 +1,134 @@
+import asyncio
+import aio_pika
+from aio_pika.exceptions import AMQPException
+from secret_manager import return_credentials
+import os
+from simple_logger_handler import setup_logger
+import json
+import signal
+from urllib.parse import quote
+import weakref
+
+logger = setup_logger(__name__)
+
+class RabbitMQConsumer:
+	"""Pull-based RabbitMQ consumer. Messages are fetched on demand, one queue per user."""
+	def __init__(self):
+		self.rmq_host = os.getenv("BACKEND_PUSH_NOTIFICATIONS_RMQ_HOST", "localhost")
+		self.rmq_vhost = os.getenv("BACKEND_PUSH_NOTIFICATIONS_RMQ_VHOST", "app_notifications")
+		self.rmq_exchange = os.getenv("BACKEND_PUSH_NOTIFICATIONS_RMQ_EXCHANGE", "app_notifications")
+		self.pull_queue_prefix = os.getenv("BACKEND_PULL_NOTIFICATIONS_RMQ_QUEUE_PREFIX", "notifications-pull")
+		self.pull_routing_prefix = os.getenv("BACKEND_PULL_NOTIFICATIONS_RMQ_ROUTING_PREFIX", "notify.pull")
+
+		self._user_locks: weakref.WeakValueDictionary[int, asyncio.Lock] = weakref.WeakValueDictionary()
+		self.connection: aio_pika.RobustConnection | None = None
+		self.channel: aio_pika.RobustChannel | None = None
+		self.exchange: aio_pika.Exchange | None = None
+		self._closing = False
+		logger.debug(f"[RabbitMQ] Initialized consumer for exchange {self.rmq_exchange}")
+
+	def _lock_for(self, user_id: int) -> asyncio.Lock:
+		"""
+		Return the asyncio lock for this user, creating it if needed. Locks are held
+		weakly, so they are cleaned up automatically once nobody is using them.
+		"""
+		lock = self._user_locks.get(user_id)
+		if lock is None:
+			lock = asyncio.Lock()
+			self._user_locks[user_id] = lock
+		return lock
+
+	async def connect(self):
+		username = return_credentials("/etc/secrets/rmq_username")
+		password = return_credentials("/etc/secrets/rmq_password")
+		url = (
+			f"amqp://{quote(username, safe='')}:{quote(password, safe='')}"
+			f"@{self.rmq_host}/{quote(self.rmq_vhost, safe='')}"
+		)
+
+		logger.info(f"[RabbitMQ] Connecting to {self.rmq_host}...")
+		self.connection = await aio_pika.connect_robust(url)
+		self.channel = await self.connection.channel()
+		self.exchange = await self.channel.declare_exchange(
+			self.rmq_exchange, aio_pika.ExchangeType.TOPIC, durable=True
+		)
+		logger.info("[RabbitMQ] Connected to RabbitMQ")
+
+	async def ensure_user_queue(self, user_id: int) -> aio_pika.Queue:
+		"""
+		Declare the user's durable queue (with expiry and message TTL) and bind it to
+		the exchange with the user's routing key. Safe to call repeatedly, recreates it if it expired, and resets its
+		idle-expiry timer.
+		"""
+		queue = await self.channel.declare_queue(
+			f"{self.pull_queue_prefix}.{user_id}",
+			durable=True,
+			arguments={
+				"x-expires": 30 * 24 * 3600 * 1000,
+				"x-message-ttl": 7 * 24 * 3600 * 1000,
+			},
+		)
+		await queue.bind(self.exchange, routing_key=f"{self.pull_routing_prefix}.{user_id}")
+		return queue
+		
+	async def fetch_messages(self, user_id: int, max_messages: int = 1000) -> list[dict]:
+		"""Drain up to max_messages queued messages for this user and return their
+		decoded JSON bodies.
+
+		- Only one fetch per user runs at a time; different users run concurrently.
+		- Messages are acked after the whole batch has been read, so an error mid-read
+		  puts the batch back on the queue and re-raises.
+		- Undecodable messages are rejected without requeue (discarded).
+		- Returns an empty list if nothing is queued.
+		- Delivery is at-most-once towards the client: messages are acked before the
+		  HTTP response is sent.
+		"""
+
+		if self._closing:
+			raise RuntimeError("[RabbitMQ] Consumer is closing")
+		if not self.channel:
+			raise RuntimeError("[RabbitMQ] Queue not initialized, call connect() first")
+
+		user_id = int(user_id)
+
+		async with self._lock_for(user_id):  #one drain at a time
+			queue = await self.ensure_user_queue(user_id)
+			results: list[dict] = []
+			to_ack: list[aio_pika.IncomingMessage] = []
+			try:
+				while len(to_ack) < max_messages:
+					message = await queue.get(no_ack=False, fail=False, timeout=5)
+					if message is None:
+						break
+
+					try:
+						data = json.loads(message.body.decode())
+					except (json.JSONDecodeError, UnicodeDecodeError) as e:
+						logger.error(f"[RabbitMQ] Bad message, discarding: {e}", exc_info=True)
+						await message.reject(requeue=False)
+						continue
+
+					results.append(data)
+					to_ack.append(message)
+
+				for message in to_ack:
+					await message.ack()
+
+				logger.debug(f"[RabbitMQ] Fetched {len(results)} message(s)")
+				return results
+
+			except (AMQPException, OSError) as e:
+				logger.error(f"[RabbitMQ] Error while fetching, requeueing batch: {e}", exc_info=True)
+				for message in to_ack:
+					try:
+						await message.nack(requeue=True)
+					except Exception:
+						pass  # channel is gone; the broker requeues unacked messages itself
+				raise
+
+	async def close(self):
+		self._closing = True
+		if self.connection:
+			await self.connection.close()
+			logger.info("[RabbitMQ] Connection closed")
+