Migration to UV, introducing db_write, mqtt_ingestion, db_migrations, half working prototype.

services/mqtt_ingestion/.dockerignore

@@ -0,0 +1,6 @@
+.env
+.venv/
+__pycache__/
+*.pyc
+*.pyo
+*.pyd

services/mqtt_ingestion/.env.example

@@ -0,0 +1,13 @@
+# Redis Configuration
+REDIS_HOST=localhost
+REDIS_PORT=6379
+REDIS_DB=0
+REDIS_PASSWORD=
+
+# MQTT Configuration
+MQTT_BROKER=localhost
+MQTT_PORT=1883
+MQTT_USERNAME=
+MQTT_PASSWORD=
+MQTT_TOPIC_PATTERN=devices/#
+MQTT_KEEPALIVE=60

services/mqtt_ingestion/.python-version

@@ -0,0 +1 @@
+3.13

services/mqtt_ingestion/Dockerfile

@@ -0,0 +1,32 @@
+# Stage 1: Builder - Install dependencies with uv
+FROM ghcr.io/astral-sh/uv:python3.13-alpine AS builder
+
+WORKDIR /app
+
+ENV UV_COMPILE_BYTECODE=1
+
+COPY pyproject.toml uv.lock ./
+
+RUN uv sync --frozen --no-dev --no-install-project
+
+COPY . .
+
+RUN uv sync --frozen --no-dev
+
+
+FROM python:3.13-alpine
+
+WORKDIR /app
+
+COPY --from=builder /app/.venv /app/.venv
+
+COPY --from=builder /app/*.py /app/
+
+RUN adduser -D -u 1000 appuser && \
+    chown -R appuser:appuser /app
+
+USER appuser
+
+ENV PATH="/app/.venv/bin:$PATH"
+
+CMD ["python", "main.py"]

services/mqtt_ingestion/README.md

@@ -0,0 +1,3 @@
+# MQTT Ingestion service
+
+This service ingests all the data from the MQTT topics and sends the data to Redis

services/mqtt_ingestion/config.py

@@ -18,7 +18,8 @@ class MQTTConfig:
     port: int = 1883
     username: Optional[str] = None
     password: Optional[str] = None
-    topic: str = "#"
+    topic_pattern: str = "devices/#"
+    keepalive: int = 60
 
 @dataclass
 class Payload:
@@ -40,8 +41,9 @@ class Config:
             broker=os.getenv('MQTT_BROKER', 'localhost'),
             port=int(os.getenv('MQTT_PORT', 1883)),
             username=os.getenv('MQTT_USERNAME', None),
-            password=None,
-            topic="#"
+            password=os.getenv('MQTT_PASSWORD', None),
+            topic_pattern=os.getenv('MQTT_TOPIC_PATTERN', 'devices/#'),
+            keepalive=int(os.getenv('MQTT_KEEPALIVE', 60))
         )
 
 config = Config()

services/mqtt_ingestion/main.py

@@ -1,11 +1,9 @@
 import logging
 import signal
 import sys
-from config import config
 from mqtt_client import MQTTClient
 from redis_writer import RedisWriter
 
-# Setup logging
 logging.basicConfig(
     level=getattr(logging,'INFO'),
     format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
@@ -18,7 +16,6 @@ class MQTTIngestionService:
         self.redis_writer = None
         self.mqtt_client = None
         
-        # Setup signal handlers for graceful shutdown
         signal.signal(signal.SIGTERM, self._signal_handler)
         signal.signal(signal.SIGINT, self._signal_handler)
     
@@ -43,13 +40,10 @@ class MQTTIngestionService:
         logger.info("Starting MQTT Ingestion Service...")
         
         try:
-            # Initialize Redis writer
             self.redis_writer = RedisWriter()
             
-            # Initialize MQTT client with our message handler
             self.mqtt_client = MQTTClient(self._handle_sensor_data)
             
-            # Connect to MQTT
             if not self.mqtt_client.connect():
                 logger.error("Failed to connect to MQTT, exiting")
                 return False
@@ -57,7 +51,6 @@ class MQTTIngestionService:
             self.running = True
             logger.info("Service started successfully")
             
-            # Start MQTT loop (this blocks)
             self.mqtt_client.start_loop()
             
         except Exception as e:
@@ -74,11 +67,9 @@ class MQTTIngestionService:
         logger.info("Stopping service...")
         self.running = False
         
-        # Stop MQTT client
         if self.mqtt_client:
             self.mqtt_client.stop()
         
-        # Close Redis connection
         if self.redis_writer:
             self.redis_writer.close()
         
@@ -89,7 +80,6 @@ class MQTTIngestionService:
         if not self.running:
             return False
             
-        # Check Redis connection
         if not self.redis_writer or not self.redis_writer.health_check():
             return False
             
@@ -97,13 +87,9 @@ class MQTTIngestionService:
 
 def main():
     """Entry point"""
-    service = MQTTIngestionService(
-        redis_config=config.redis,
-        mqtt_config=config.mqtt
-    )
+    service = MQTTIngestionService()
     
     try:
-        # Start the service (blocks until shutdown)
         success = service.start()
         if not success:
             sys.exit(1)

services/mqtt_ingestion/mqtt_client.py

@@ -20,7 +20,6 @@ class MQTTClient:
         self.client.on_message = self._on_message
         self.client.on_disconnect = self._on_disconnect
         
-        # Set credentials if provided
         if config.mqtt.username:
             self.client.username_pw_set(
                 config.mqtt.username, 
@@ -30,7 +29,6 @@ class MQTTClient:
     def _on_connect(self, client, userdata, flags, rc):
         if rc == 0:
             logger.info(f"Connected to MQTT broker {config.mqtt.broker}")
-            # Subscribe to all device topics: devices/+/+
             client.subscribe(config.mqtt.topic_pattern)
             logger.info(f"Subscribed to {config.mqtt.topic_pattern}")
         else:
@@ -38,7 +36,6 @@ class MQTTClient:
     
     def _on_message(self, client, userdata, msg):
         try:
-            # Parse topic: devices/{device_id}/{sensor_type}
             topic_parts = msg.topic.split('/')
             if len(topic_parts) != 3 or topic_parts[0] != 'devices':
                 logger.warning(f"Invalid topic format: {msg.topic}")
@@ -47,14 +44,12 @@ class MQTTClient:
             device_id = topic_parts[1]
             sensor_type = topic_parts[2]
             
-            # Parse payload as float
             try:
                 value = float(msg.payload.decode())
             except ValueError:
                 logger.error(f"Invalid payload for {msg.topic}: {msg.payload}")
                 return
             
-            # Call the handler (this will be our Redis writer)
             self.message_handler(device_id, sensor_type, value)
             
         except Exception as e:

services/mqtt_ingestion/pyproject.toml

@@ -0,0 +1,11 @@
+[project]
+name = "mqtt-ingestion"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.13"
+dependencies = [
+    "dotenv>=0.9.9",
+    "paho-mqtt>=2.1.0",
+    "redis>=6.4.0",
+]

services/mqtt_ingestion/redis_writer.py

@@ -1,37 +1,67 @@
 import redis
 import logging
 from datetime import datetime
-from typing import Optional
-from config import Payload
+from config import config
+
+logger = logging.getLogger(__name__)
 
 class RedisWriter:
-    def __init__(self, host: str, port: int, db: int, password: Optional[str] = None):
+    def __init__(self):
+        """Initialize Redis writer with config from environment"""
         self.logger = logging.getLogger(__name__)
-        self.redis_client = redis.StrictRedis(host=host, port=port, db=db, password=password)
+        self.redis_client = redis.StrictRedis(
+            host=config.redis.host,
+            port=config.redis.port,
+            db=config.redis.db,
+            password=config.redis.password
+        )
         try:
             self.redis_client.ping()
-            self.logger.info("Connected to Redis server successfully.")
+            self.logger.info(f"Connected to Redis at {config.redis.host}:{config.redis.port}")
         except redis.ConnectionError as e:
             self.logger.error(f"Failed to connect to Redis server: {e}")
             raise
 
-    def write_message(self, topic: str, payload: Payload):
+    def write_sensor_data(self, device_id: str, sensor_type: str, value: float) -> bool:
         """
-        Write a message to a Redis stream with the topic and payload.
-        - Stream: mqtt_stream: {device_id}:{sensor_type}
+        Write sensor data to Redis streams and latest values hash.
+        - Stream: mqtt_stream:{device_id}:{sensor_type}
+        - Hash: mqtt_latest:{device_id}
         """
-        device_id = payload.device_id
-        sensor_type = payload.sensor_type
         timestamp = datetime.utcnow().isoformat()
-        stream_key= f"mqtt_stream:{device_id}:{sensor_type}"
+        stream_key = f"mqtt_stream:{device_id}:{sensor_type}"
+        hash_key = f"mqtt_latest:{device_id}"
+        
         stream_data = {
-            "value": str(payload),
-            "source": "mqtt",
+            "value": str(value),
             "timestamp": timestamp
         }
+        
         try:
-            message_id = self.redis_client.xadd(stream_key, stream_data,maxlen=1000)
-            self.logger.info(f"Message written to Redis: {stream_data}")
-            return message_id
+            # Write to stream
+            self.redis_client.xadd(stream_key, stream_data, maxlen=1000)
+            
+            # Update latest value hash
+            self.redis_client.hset(hash_key, sensor_type, str(value))
+            self.redis_client.hset(hash_key, f"{sensor_type}_time", timestamp)
+            
+            return True
         except redis.RedisError as e:
-            self.logger.error(f"Failed to write message to Redis: {e}")
+            self.logger.error(f"Failed to write to Redis: {e}")
+            return False
+    
+    def health_check(self) -> bool:
+        """Check if Redis connection is healthy"""
+        try:
+            self.redis_client.ping()
+            return True
+        except redis.RedisError:
+            return False
+    
+    def close(self):
+        """Close Redis connection"""
+        try:
+            self.redis_client.close()
+            self.logger.info("Redis connection closed")
+        except Exception as e:
+            self.logger.error(f"Error closing Redis connection: {e}")

services/mqtt_ingestion/requirements.txt

@@ -1,2 +0,0 @@
-redis
-paho-mqtt

services/mqtt_ingestion/uv.lock

@@ -0,0 +1,58 @@
+version = 1
+revision = 3
+requires-python = ">=3.13"
+
+[[package]]
+name = "dotenv"
+version = "0.9.9"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "python-dotenv" },
+]
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/b2/b7/545d2c10c1fc15e48653c91efde329a790f2eecfbbf2bd16003b5db2bab0/dotenv-0.9.9-py2.py3-none-any.whl", hash = "sha256:29cf74a087b31dafdb5a446b6d7e11cbce8ed2741540e2339c69fbef92c94ce9", size = 1892, upload-time = "2025-02-19T22:15:01.647Z" },
+]
+
+[[package]]
+name = "mqtt-ingestion"
+version = "0.1.0"
+source = { virtual = "." }
+dependencies = [
+    { name = "dotenv" },
+    { name = "paho-mqtt" },
+    { name = "redis" },
+]
+
+[package.metadata]
+requires-dist = [
+    { name = "dotenv", specifier = ">=0.9.9" },
+    { name = "paho-mqtt", specifier = ">=2.1.0" },
+    { name = "redis", specifier = ">=6.4.0" },
+]
+
+[[package]]
+name = "paho-mqtt"
+version = "2.1.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/39/15/0a6214e76d4d32e7f663b109cf71fb22561c2be0f701d67f93950cd40542/paho_mqtt-2.1.0.tar.gz", hash = "sha256:12d6e7511d4137555a3f6ea167ae846af2c7357b10bc6fa4f7c3968fc1723834", size = 148848, upload-time = "2024-04-29T19:52:55.591Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c4/cb/00451c3cf31790287768bb12c6bec834f5d292eaf3022afc88e14b8afc94/paho_mqtt-2.1.0-py3-none-any.whl", hash = "sha256:6db9ba9b34ed5bc6b6e3812718c7e06e2fd7444540df2455d2c51bd58808feee", size = 67219, upload-time = "2024-04-29T19:52:48.345Z" },
+]
+
+[[package]]
+name = "python-dotenv"
+version = "1.1.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/f6/b0/4bc07ccd3572a2f9df7e6782f52b0c6c90dcbb803ac4a167702d7d0dfe1e/python_dotenv-1.1.1.tar.gz", hash = "sha256:a8a6399716257f45be6a007360200409fce5cda2661e3dec71d23dc15f6189ab", size = 41978, upload-time = "2025-06-24T04:21:07.341Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/5f/ed/539768cf28c661b5b068d66d96a2f155c4971a5d55684a514c1a0e0dec2f/python_dotenv-1.1.1-py3-none-any.whl", hash = "sha256:31f23644fe2602f88ff55e1f5c79ba497e01224ee7737937930c448e4d0e24dc", size = 20556, upload-time = "2025-06-24T04:21:06.073Z" },
+]
+
+[[package]]
+name = "redis"
+version = "6.4.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/0d/d6/e8b92798a5bd67d659d51a18170e91c16ac3b59738d91894651ee255ed49/redis-6.4.0.tar.gz", hash = "sha256:b01bc7282b8444e28ec36b261df5375183bb47a07eb9c603f284e89cbc5ef010", size = 4647399, upload-time = "2025-08-07T08:10:11.441Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e8/02/89e2ed7e85db6c93dfa9e8f691c5087df4e3551ab39081a4d7c6d1f90e05/redis-6.4.0-py3-none-any.whl", hash = "sha256:f0544fa9604264e9464cdf4814e7d4830f74b165d52f2a330a760a88dd248b7f", size = 279847, upload-time = "2025-08-07T08:10:09.84Z" },
+]