Working device manager, added folders for better organization.

services/mqtt_ingestion/src/config.py

@@ -0,0 +1,53 @@
+import os
+from dataclasses import dataclass
+import dotenv
+from typing import Optional
+
+dotenv.load_dotenv()
+
+
+@dataclass
+class RedisConfig:
+    host: str
+    port: int = 6379
+    db: int = 0
+    password: Optional[str] = None
+
+
+@dataclass
+class MQTTConfig:
+    broker: str
+    port: int = 1883
+    username: Optional[str] = None
+    password: Optional[str] = None
+    topic_pattern: str = "devices/#"
+    keepalive: int = 60
+
+
+@dataclass
+class Payload:
+    device_id: str
+    sensor_type: str
+    value: float
+    timestamp: Optional[str] = None
+
+
+class Config:
+    def __init__(self):
+        self.redis = RedisConfig(
+            host=os.getenv("REDIS_HOST", "localhost"),
+            port=int(os.getenv("REDIS_PORT", 6379)),
+            db=int(os.getenv("REDIS_DB", 0)),
+            password=os.getenv("REDIS_PASSWORD", None),
+        )
+        self.mqtt = MQTTConfig(
+            broker=os.getenv("MQTT_BROKER", "localhost"),
+            port=int(os.getenv("MQTT_PORT", 1883)),
+            username=os.getenv("MQTT_USERNAME", None),
+            password=os.getenv("MQTT_PASSWORD", None),
+            topic_pattern=os.getenv("MQTT_TOPIC_PATTERN", "devices/#"),
+            keepalive=int(os.getenv("MQTT_KEEPALIVE", 60)),
+        )
+
+
+config = Config()

services/mqtt_ingestion/src/mqtt_client.py

@@ -0,0 +1,79 @@
+import logging
+import paho.mqtt.client as mqtt
+from typing import Callable
+from src.config import config
+
+logger = logging.getLogger(__name__)
+
+
+class MQTTClient:
+    def __init__(self, message_handler: Callable[[str, str, float], None]):
+        """
+        Args:
+            message_handler: Function that takes (device_id, sensor_type, value)
+        """
+        self.message_handler = message_handler
+        self.client = mqtt.Client()
+        self._setup_callbacks()
+
+    def _setup_callbacks(self):
+        self.client.on_connect = self._on_connect
+        self.client.on_message = self._on_message
+        self.client.on_disconnect = self._on_disconnect
+
+        if config.mqtt.username:
+            self.client.username_pw_set(config.mqtt.username, config.mqtt.password)
+
+    def _on_connect(self, client, userdata, flags, rc):
+        if rc == 0:
+            logger.info(f"Connected to MQTT broker {config.mqtt.broker}")
+            client.subscribe(config.mqtt.topic_pattern)
+            logger.info(f"Subscribed to {config.mqtt.topic_pattern}")
+        else:
+            logger.error(f"Failed to connect to MQTT broker, code: {rc}")
+
+    def _on_message(self, client, userdata, msg):
+        try:
+            topic_parts = msg.topic.split("/")
+            if len(topic_parts) != 3 or topic_parts[0] != "devices":
+                logger.warning(f"Invalid topic format: {msg.topic}")
+                return
+
+            device_id = topic_parts[1]
+            sensor_type = topic_parts[2]
+
+            try:
+                value = float(msg.payload.decode())
+            except ValueError:
+                logger.error(f"Invalid payload for {msg.topic}: {msg.payload}")
+                return
+
+            self.message_handler(device_id, sensor_type, value)
+
+        except Exception as e:
+            logger.error(f"Error processing MQTT message: {e}")
+
+    def _on_disconnect(self, client, userdata, rc):
+        if rc != 0:
+            logger.warning("Unexpected MQTT disconnection")
+        else:
+            logger.info("MQTT client disconnected")
+
+    def connect(self):
+        """Connect to MQTT broker"""
+        try:
+            self.client.connect(
+                config.mqtt.broker, config.mqtt.port, config.mqtt.keepalive
+            )
+            return True
+        except Exception as e:
+            logger.error(f"Failed to connect to MQTT: {e}")
+            return False
+
+    def start_loop(self):
+        """Start the MQTT loop (blocking)"""
+        self.client.loop_forever()
+
+    def stop(self):
+        """Stop the MQTT client"""
+        self.client.disconnect()

services/mqtt_ingestion/src/redis_writer.py

@@ -0,0 +1,72 @@
+import redis
+import logging
+from datetime import datetime
+from src.config import config
+
+logger = logging.getLogger(__name__)
+
+
+class RedisWriter:
+    def __init__(self):
+        """Initialize Redis writer with config from environment"""
+        self.logger = logging.getLogger(__name__)
+        self.redis_client = redis.StrictRedis(
+            host=config.redis.host,
+            port=config.redis.port,
+            db=config.redis.db,
+            password=config.redis.password,
+        )
+        try:
+            self.redis_client.ping()
+            self.logger.info(
+                f"Connected to Redis at {config.redis.host}:{config.redis.port}"
+            )
+        except redis.ConnectionError as e:
+            self.logger.error(f"Failed to connect to Redis server: {e}")
+            raise
+
+    def write_sensor_data(self, device_id: str, sensor_type: str, value: float) -> bool:
+        """
+        Write sensor data to single Redis stream for all devices.
+        - Stream: mqtt:ingestion (single stream for scalability)
+        - Hash: mqtt_latest:{device_id} (for quick dashboard access)
+        """
+        timestamp = datetime.utcnow().isoformat()
+
+        stream_key = "mqtt:ingestion"
+        hash_key = f"mqtt_latest:{device_id}"
+
+        stream_data = {
+            "device_id": device_id,
+            "metric": sensor_type,
+            "value": str(value),
+            "timestamp": timestamp,
+        }
+
+        try:
+            # Write to single stream
+            self.redis_client.xadd(stream_key, stream_data, maxlen=10000)
+
+            self.redis_client.hset(hash_key, sensor_type, str(value))
+            self.redis_client.hset(hash_key, f"{sensor_type}_time", timestamp)
+
+            return True
+        except redis.RedisError as e:
+            self.logger.error(f"Failed to write to Redis: {e}")
+            return False
+
+    def health_check(self) -> bool:
+        """Check if Redis connection is healthy"""
+        try:
+            self.redis_client.ping()
+            return True
+        except redis.RedisError:
+            return False
+
+    def close(self):
+        """Close Redis connection"""
+        try:
+            self.redis_client.close()
+            self.logger.info("Redis connection closed")
+        except Exception as e:
+            self.logger.error(f"Error closing Redis connection: {e}")