ferdzo/iotDashboard
1
0
Fork 0
mirror of https://github.com/ferdzo/iotDashboard.git synced 2026-04-05 17:16:26 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Functioning mqtt ingestion and db write, formating changes, device manager initiated

Browse Source
This commit is contained in:
Andrej Mickov
2025-10-30 00:32:25 +01:00
parent 0b96c72f45
commit 12d3720421
45 changed files with 2168 additions and 820 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
services/mqtt_ingestion/config.py
View File

@@ -5,6 +5,7 @@ from typing import Optional
dotenv.load_dotenv()
@dataclass
class RedisConfig:
host: str
@@ -12,6 +13,7 @@ class RedisConfig:
db: int = 0
password: Optional[str] = None
@dataclass
class MQTTConfig:
broker: str
@@ -21,6 +23,7 @@ class MQTTConfig:
topic_pattern: str = "devices/#"
keepalive: int = 60
@dataclass
class Payload:
device_id: str
@@ -32,18 +35,19 @@ class Payload:
class Config:
def __init__(self):
self.redis = RedisConfig(
host=os.getenv('REDIS_HOST', 'localhost'),
port=int(os.getenv('REDIS_PORT', 6379)),
db=int(os.getenv('REDIS_DB', 0)),
password=os.getenv('REDIS_PASSWORD', None)
host=os.getenv("REDIS_HOST", "localhost"),
port=int(os.getenv("REDIS_PORT", 6379)),
db=int(os.getenv("REDIS_DB", 0)),
password=os.getenv("REDIS_PASSWORD", None),
)
self.mqtt = MQTTConfig(
broker=os.getenv('MQTT_BROKER', 'localhost'),
port=int(os.getenv('MQTT_PORT', 1883)),
username=os.getenv('MQTT_USERNAME', None),
password=os.getenv('MQTT_PASSWORD', None),
topic_pattern=os.getenv('MQTT_TOPIC_PATTERN', 'devices/#'),
keepalive=int(os.getenv('MQTT_KEEPALIVE', 60))
broker=os.getenv("MQTT_BROKER", "localhost"),
port=int(os.getenv("MQTT_PORT", 1883)),
username=os.getenv("MQTT_USERNAME", None),
password=os.getenv("MQTT_PASSWORD", None),
topic_pattern=os.getenv("MQTT_TOPIC_PATTERN", "devices/#"),
keepalive=int(os.getenv("MQTT_KEEPALIVE", 60)),
)
config = Config()
config = Config()

47
services/mqtt_ingestion/main.py
View File

@@ -5,25 +5,26 @@ from mqtt_client import MQTTClient
from redis_writer import RedisWriter
logging.basicConfig(
level=getattr(logging,'INFO'),
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
level=getattr(logging, "INFO"),
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
)
logger = logging.getLogger(__name__)
class MQTTIngestionService:
def __init__(self):
self.running = False
self.redis_writer = None
self.mqtt_client = None
signal.signal(signal.SIGTERM, self._signal_handler)
signal.signal(signal.SIGINT, self._signal_handler)
def _signal_handler(self, signum, frame):
"""Handle shutdown signals"""
logger.info(f"Received signal {signum}, shutting down...")
self.stop()
def _handle_sensor_data(self, device_id: str, sensor_type: str, value: float):
"""
This function is called by MQTT client when a message arrives.
@@ -34,61 +35,62 @@ class MQTTIngestionService:
logger.info(f"Processed {device_id}/{sensor_type}: {value}")
else:
logger.error(f"Failed to process {device_id}/{sensor_type}: {value}")
def start(self):
"""Start the service"""
logger.info("Starting MQTT Ingestion Service...")
try:
self.redis_writer = RedisWriter()
self.mqtt_client = MQTTClient(self._handle_sensor_data)
if not self.mqtt_client.connect():
logger.error("Failed to connect to MQTT, exiting")
return False
self.running = True
logger.info("Service started successfully")
self.mqtt_client.start_loop()
except Exception as e:
logger.error(f"Service startup failed: {e}")
return False
return True
def stop(self):
"""Stop the service gracefully"""
if not self.running:
return
logger.info("Stopping service...")
self.running = False
if self.mqtt_client:
self.mqtt_client.stop()
if self.redis_writer:
self.redis_writer.close()
logger.info("Service stopped")
def health_check(self) -> bool:
"""Check if service is healthy"""
if not self.running:
return False
if not self.redis_writer or not self.redis_writer.health_check():
return False
return True
def main():
"""Entry point"""
service = MQTTIngestionService()
try:
success = service.start()
if not success:
@@ -101,5 +103,6 @@ def main():
finally:
service.stop()
if __name__ == "__main__":
main()

40
services/mqtt_ingestion/mqtt_client.py
View File

@@ -5,6 +5,7 @@ from config import config
logger = logging.getLogger(__name__)
class MQTTClient:
def __init__(self, message_handler: Callable[[str, str, float], None]):
"""
@@ -14,18 +15,15 @@ class MQTTClient:
self.message_handler = message_handler
self.client = mqtt.Client()
self._setup_callbacks()
def _setup_callbacks(self):
self.client.on_connect = self._on_connect
self.client.on_message = self._on_message
self.client.on_disconnect = self._on_disconnect
if config.mqtt.username:
self.client.username_pw_set(
config.mqtt.username,
config.mqtt.password
)
self.client.username_pw_set(config.mqtt.username, config.mqtt.password)
def _on_connect(self, client, userdata, flags, rc):
if rc == 0:
logger.info(f"Connected to MQTT broker {config.mqtt.broker}")
@@ -33,51 +31,49 @@ class MQTTClient:
logger.info(f"Subscribed to {config.mqtt.topic_pattern}")
else:
logger.error(f"Failed to connect to MQTT broker, code: {rc}")
def _on_message(self, client, userdata, msg):
try:
topic_parts = msg.topic.split('/')
if len(topic_parts) != 3 or topic_parts[0] != 'devices':
topic_parts = msg.topic.split("/")
if len(topic_parts) != 3 or topic_parts[0] != "devices":
logger.warning(f"Invalid topic format: {msg.topic}")
return
device_id = topic_parts[1]
sensor_type = topic_parts[2]
try:
value = float(msg.payload.decode())
except ValueError:
logger.error(f"Invalid payload for {msg.topic}: {msg.payload}")
return
self.message_handler(device_id, sensor_type, value)
except Exception as e:
logger.error(f"Error processing MQTT message: {e}")
def _on_disconnect(self, client, userdata, rc):
if rc != 0:
logger.warning("Unexpected MQTT disconnection")
else:
logger.info("MQTT client disconnected")
def connect(self):
"""Connect to MQTT broker"""
try:
self.client.connect(
config.mqtt.broker,
config.mqtt.port,
config.mqtt.keepalive
config.mqtt.broker, config.mqtt.port, config.mqtt.keepalive
)
return True
except Exception as e:
logger.error(f"Failed to connect to MQTT: {e}")
return False
def start_loop(self):
"""Start the MQTT loop (blocking)"""
self.client.loop_forever()
def stop(self):
"""Stop the MQTT client"""
self.client.disconnect()
self.client.disconnect()

39
services/mqtt_ingestion/redis_writer.py
View File

@@ -5,6 +5,7 @@ from config import config
logger = logging.getLogger(__name__)
class RedisWriter:
def __init__(self):
"""Initialize Redis writer with config from environment"""
@@ -13,43 +14,47 @@ class RedisWriter:
host=config.redis.host,
port=config.redis.port,
db=config.redis.db,
password=config.redis.password
password=config.redis.password,
)
try:
self.redis_client.ping()
self.logger.info(f"Connected to Redis at {config.redis.host}:{config.redis.port}")
self.logger.info(
f"Connected to Redis at {config.redis.host}:{config.redis.port}"
)
except redis.ConnectionError as e:
self.logger.error(f"Failed to connect to Redis server: {e}")
raise
def write_sensor_data(self, device_id: str, sensor_type: str, value: float) -> bool:
"""
Write sensor data to Redis streams and latest values hash.
- Stream: mqtt_stream:{device_id}:{sensor_type}
- Hash: mqtt_latest:{device_id}
Write sensor data to single Redis stream for all devices.
- Stream: mqtt:ingestion (single stream for scalability)
- Hash: mqtt_latest:{device_id} (for quick dashboard access)
"""
timestamp = datetime.utcnow().isoformat()
stream_key = f"mqtt_stream:{device_id}:{sensor_type}"
stream_key = "mqtt:ingestion"
hash_key = f"mqtt_latest:{device_id}"
stream_data = {
"device_id": device_id,
"metric": sensor_type,
"value": str(value),
"timestamp": timestamp
"timestamp": timestamp,
}
try:
# Write to stream
self.redis_client.xadd(stream_key, stream_data, maxlen=1000)
# Update latest value hash
# Write to single stream
self.redis_client.xadd(stream_key, stream_data, maxlen=10000)
self.redis_client.hset(hash_key, sensor_type, str(value))
self.redis_client.hset(hash_key, f"{sensor_type}_time", timestamp)
return True
except redis.RedisError as e:
self.logger.error(f"Failed to write to Redis: {e}")
return False
def health_check(self) -> bool:
"""Check if Redis connection is healthy"""
try:
@@ -57,11 +62,11 @@ class RedisWriter:
return True
except redis.RedisError:
return False
def close(self):
"""Close Redis connection"""
try:
self.redis_client.close()
self.logger.info("Redis connection closed")
except Exception as e:
self.logger.error(f"Error closing Redis connection: {e}")
self.logger.error(f"Error closing Redis connection: {e}")