From afd195dab981f86db159f700405b0bba0536a232 Mon Sep 17 00:00:00 2001 From: Andrej Mickov Date: Mon, 9 Feb 2026 01:10:52 +0100 Subject: [PATCH] Basic data ingestion and analytics --- .env.example | 6 + API-DOCUMENTATION.md | 871 ++++++++++++++++++++++++++++++++++ README.md | 101 +++- UPSTREAM-API-DOCUMENTATION.md | 556 ++++++++++++++++++++++ background-tracker.ts | 406 ++++++++++++++++ bun.lock | 99 +++- infrastructure/compose.yml | 24 + lib/database.ts | 464 ++++++++++++++++++ package.json | 10 +- public/analytics.html | 500 +++++++++++++++++++ public/index.html | 23 +- server.ts | 162 ++++++- 12 files changed, 3203 insertions(+), 19 deletions(-) create mode 100644 .env.example create mode 100644 API-DOCUMENTATION.md create mode 100644 UPSTREAM-API-DOCUMENTATION.md create mode 100644 background-tracker.ts create mode 100644 infrastructure/compose.yml create mode 100644 lib/database.ts create mode 100644 public/analytics.html diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..05ce217 --- /dev/null +++ b/.env.example @@ -0,0 +1,6 @@ +# Database Configuration (TimescaleDB) +POSTGRES_HOST=localhost +POSTGRES_PORT=5432 +POSTGRES_DB=iot_data +POSTGRES_USER=postgres +POSTGRES_PASSWORD=example diff --git a/API-DOCUMENTATION.md b/API-DOCUMENTATION.md new file mode 100644 index 0000000..3460309 --- /dev/null +++ b/API-DOCUMENTATION.md @@ -0,0 +1,871 @@ +# GTFS API Documentation + +Comprehensive API documentation for the Skopje Bus Tracker GTFS-based API. + +## Table of Contents + +- [Overview](#overview) +- [OpenAPI Specification](#openapi-specification) +- [Base URL](#base-url) +- [GTFS Static Data Endpoints](#gtfs-static-data-endpoints) +- [Real-Time Data Endpoints](#real-time-data-endpoints) +- [Historical Data Endpoints](#historical-data-endpoints) +- [Statistics & Analytics Endpoints](#statistics--analytics-endpoints) +- [Data Models](#data-models) +- [Error Handling](#error-handling) +- [Rate Limiting](#rate-limiting) + +--- + +## Overview + +This API provides access to Skopje public transport data based on the **GTFS (General Transit Feed Specification)** standard. It combines static GTFS data with real-time information and historical analytics stored in TimescaleDB. + +### Key Features +- **GTFS-compliant** static data (stops, routes) +- **Real-time** vehicle positions and arrival predictions +- **Historical** time-series data for analytics +- **Statistical** analysis of delays and patterns + +### Upstream API + +This application consumes data from the **ModeShift GTFS API**. For documentation on the upstream API endpoints: +- See **[UPSTREAM-API-DOCUMENTATION.md](UPSTREAM-API-DOCUMENTATION.md)** + +--- + +## OpenAPI Specification + +A complete **OpenAPI 3.0** specification is available at [openapi.yaml](openapi.yaml). This machine-readable specification includes: + +- All API endpoints with parameters and responses +- Request/response schemas and data models +- Examples for all operations +- Error response formats + +### Using the OpenAPI Spec + +**Interactive Documentation (Local):** + +Once the server is running, visit: +``` +http://localhost:3000/api-docs.html +``` + +This provides an interactive Swagger UI where you can: +- Browse all API endpoints +- View request/response schemas +- Try out API calls directly from the browser +- Download the OpenAPI specification + +**Interactive Documentation with Swagger UI (Alternative):** +```bash +# Using Docker +docker run -p 8080:8080 -e SWAGGER_JSON=/openapi.yaml -v $(pwd)/openapi.yaml:/openapi.yaml swaggerapi/swagger-ui + +# Or use online editor +# Visit: https://editor.swagger.io/ +# Then paste the contents of openapi.yaml +``` + +**Generate Client Libraries:** +```bash +# Install OpenAPI Generator +npm install -g @openapitools/openapi-generator-cli + +# Generate TypeScript client +openapi-generator-cli generate -i openapi.yaml -g typescript-fetch -o ./client/typescript + +# Generate Python client +openapi-generator-cli generate -i openapi.yaml -g python -o ./client/python + +# Generate Go client +openapi-generator-cli generate -i openapi.yaml -g go -o ./client/go +``` + +**VS Code Integration:** +Install the "OpenAPI (Swagger) Editor" extension to view and edit the spec with validation and IntelliSense. + +--- + +## Base URL + +``` +http://localhost:3000 +``` + +For production, replace with your deployed server URL. + +--- + +## GTFS Static Data Endpoints + +### Get Configuration + +Retrieve configuration including default stop and route information. + +**Endpoint:** `GET /api/config` + +**Query Parameters:** +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `stopId` | string | No | Specific stop ID to query (defaults to configured default) | +| `routeId` | string | No | Specific route ID to query (defaults to configured default) | + +**Response:** +```json +{ + "stop": { + "id": "1563", + "code": "363", + "name": "KARPOS II - STIV NAUMOV", + "lat": 41.97964, + "lon": 21.41283 + }, + "route": { + "id": "125", + "shortName": "7", + "longName": "Centar - GTC - Karpos 4 - Karposh 2" + }, + "defaults": { + "stopId": "1563", + "routeId": "125" + } +} +``` + +**Example:** +```bash +curl "http://localhost:3000/api/config?stopId=1571&routeId=125" +``` + +--- + +### Get All Stops + +Retrieve all GTFS stops in the system. + +**Endpoint:** `GET /api/stops` + +**Query Parameters:** None + +**Response:** +```json +[ + { + "id": "1563", + "code": "363", + "name": "KARPOS II - STIV NAUMOV", + "lat": 41.97964, + "lon": 21.41283 + }, + { + "id": "1571", + "code": "371", + "name": "AMERICAN COLLEGE - VASIL GLAVINOV", + "lat": 42.00437, + "lon": 21.39687 + } +] +``` + +**Example:** +```bash +curl "http://localhost:3000/api/stops" +``` + +**Use Cases:** +- Populate stop selection dropdowns +- Build stop search functionality +- Generate maps with all stops + +--- + +### Get All Routes + +Retrieve all GTFS routes in the system. + +**Endpoint:** `GET /api/routes` + +**Query Parameters:** None + +**Response:** +```json +[ + { + "id": "125", + "shortName": "7", + "longName": "Centar - GTC - Karpos 4 - Karposh 2" + }, + { + "id": "128", + "shortName": "2", + "longName": "Kisela Voda - Centar - 11 Oktomvri - Avtokomanda" + } +] +``` + +**Example:** +```bash +curl "http://localhost:3000/api/routes" +``` + +**Use Cases:** +- Display available routes +- Filter by route number +- Route selection interfaces + +--- + +## Real-Time Data Endpoints + +### Get Real-Time Arrivals + +Get upcoming bus arrivals at a specific stop with real-time predictions. + +**Endpoint:** `GET /api/arrivals` + +**Query Parameters:** +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `stopId` | string | No | Stop ID to query (defaults to configured default) | +| `routeId` | string | No | Filter by specific route ID (defaults to configured default) | + +**Response:** +```json +[ + { + "arrivalTime": "2026-02-07T14:35:00.000Z", + "scheduledTime": "2026-02-07T14:33:00.000Z", + "minutesUntil": 8, + "delaySeconds": 120, + "headsign": "Karpos 2", + "isRealtime": true, + "realtimeState": "UPDATED" + }, + { + "arrivalTime": "2026-02-07T14:48:00.000Z", + "scheduledTime": "2026-02-07T14:48:00.000Z", + "minutesUntil": 21, + "delaySeconds": 0, + "headsign": "Centar", + "isRealtime": true, + "realtimeState": "SCHEDULED" + } +] +``` + +**Fields:** +- `arrivalTime`: Predicted arrival time (ISO 8601) +- `scheduledTime`: Originally scheduled time (ISO 8601) +- `minutesUntil`: Minutes until arrival (can be negative if bus passed) +- `delaySeconds`: Delay in seconds (positive = late, negative = early) +- `headsign`: Destination shown on bus +- `isRealtime`: Whether prediction is based on real-time data +- `realtimeState`: `SCHEDULED`, `UPDATED`, or `CANCELED` + +**Example:** +```bash +curl "http://localhost:3000/api/arrivals?stopId=1563&routeId=125" +``` + +**Notes:** +- Returns arrivals from 2 minutes ago up to configured minutes ahead +- Results are sorted by arrival time +- Data is also logged to database for historical analysis + +--- + +### Get Real-Time Vehicle Positions + +Get current positions of all vehicles on a specific route. + +**Endpoint:** `GET /api/vehicles` + +**Query Parameters:** +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `routeId` | string | No | Route ID to filter vehicles (defaults to configured default) | + +**Response:** +```json +[ + { + "id": 12345, + "vehicleId": "MK-SK-1234", + "label": "1234", + "lat": 41.9981, + "lon": 21.4254, + "bearing": 87.5, + "speed": 15.3, + "timestamp": "2026-02-07T14:27:15Z", + "tripId": "trip_789", + "currentStopSequence": 0, + "currentStatus": "IN_TRANSIT_TO" + } +] +``` + +**Fields:** +- `id`: Internal vehicle database ID +- `vehicleId`: Vehicle identification number (license plate) +- `label`: Bus inventory number (shown on bus) +- `lat`/`lon`: Current GPS coordinates +- `bearing`: Direction of travel (0-360 degrees, 0=North) +- `speed`: Current speed in km/h +- `timestamp`: Last position update time +- `tripId`: GTFS trip ID vehicle is currently serving +- `currentStatus`: `IN_TRANSIT_TO`, `STOPPED_AT`, or `UNKNOWN` + +**Example:** +```bash +curl "http://localhost:3000/api/vehicles?routeId=125" +``` + +**Notes:** +- Combines GTFS-RT trip updates with JSON vehicle API +- Vehicle positions are logged to database +- Updates typically every 10-30 seconds from source + +--- + +## Historical Data Endpoints + +### Get Database Statistics + +Get overview statistics about stored historical data. + +**Endpoint:** `GET /api/stats/db` + +**Query Parameters:** None + +**Response:** +```json +{ + "vehiclePositions": { + "totalRecords": 1523847, + "oldestRecord": "2026-01-15T08:00:00.000Z", + "newestRecord": "2026-02-07T14:27:30.000Z", + "dataRetentionDays": 90 + }, + "arrivalRecords": { + "totalRecords": 456231, + "oldestRecord": "2026-01-15T08:00:00.000Z", + "newestRecord": "2026-02-07T14:27:30.000Z", + "dataRetentionDays": 90 + }, + "databaseSize": "2.3 GB", + "status": "healthy" +} +``` + +**Example:** +```bash +curl "http://localhost:3000/api/stats/db" +``` + +--- + +### Get Vehicle Position History + +Retrieve historical GPS positions for a specific vehicle. + +**Endpoint:** `GET /api/history/vehicle/:vehicleId` + +**Path Parameters:** +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `vehicleId` | string | Yes | Vehicle identification number or label | + +**Query Parameters:** +| Parameter | Type | Required | Default | Description | +|-----------|------|----------|---------|-------------| +| `hours` | integer | No | 24 | Number of hours to look back | + +**Response:** +```json +[ + { + "time": "2026-02-07T10:00:00.000Z", + "vehicleId": "MK-SK-1234", + "routeId": "125", + "tripId": "trip_789", + "latitude": 41.9981, + "longitude": 21.4254, + "speed": 15.3, + "bearing": 87.5, + "currentStatus": "IN_TRANSIT_TO" + } +] +``` + +**Example:** +```bash +curl "http://localhost:3000/api/history/vehicle/1234?hours=12" +``` + +**Use Cases:** +- Visualize vehicle path on map +- Analyze driving patterns +- Calculate actual travel times + +--- + +### Get Route Vehicle History + +Get historical positions of all vehicles that served a specific route. + +**Endpoint:** `GET /api/history/route/:routeId/vehicles` + +**Path Parameters:** +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `routeId` | string | Yes | GTFS route ID | + +**Query Parameters:** +| Parameter | Type | Required | Default | Description | +|-----------|------|----------|---------|-------------| +| `hours` | integer | No | 24 | Number of hours to look back | + +**Response:** +```json +[ + { + "time": "2026-02-07T10:00:00.000Z", + "vehicleId": "1234", + "routeId": "125", + "tripId": "trip_789", + "latitude": 41.9981, + "longitude": 21.4254, + "speed": 15.3, + "bearing": 87.5 + } +] +``` + +**Example:** +```bash +curl "http://localhost:3000/api/history/route/125/vehicles?hours=6" +``` + +**Use Cases:** +- Service frequency analysis +- Route coverage visualization +- Fleet utilization tracking + +--- + +### Get Stop Arrival History + +Retrieve historical arrival records for a specific stop. + +**Endpoint:** `GET /api/history/stop/:stopId/arrivals` + +**Path Parameters:** +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `stopId` | string | Yes | GTFS stop ID | + +**Query Parameters:** +| Parameter | Type | Required | Default | Description | +|-----------|------|----------|---------|-------------| +| `routeId` | string | No | (all) | Filter by specific route | +| `hours` | integer | No | 24 | Number of hours to look back | + +**Response:** +```json +[ + { + "time": "2026-02-07T10:15:00.000Z", + "stopId": "1563", + "routeId": "125", + "scheduledTime": "2026-02-07T10:15:00.000Z", + "predictedTime": "2026-02-07T10:17:30.000Z", + "delaySeconds": 150, + "isRealtime": true, + "headsign": "Karpos 2" + } +] +``` + +**Example:** +```bash +curl "http://localhost:3000/api/history/stop/1563/arrivals?routeId=125&hours=48" +``` + +--- + +## Statistics & Analytics Endpoints + +### Get Route Delay Statistics + +Get delay statistics and performance metrics for a route. + +**Endpoint:** `GET /api/stats/route/:routeId/delays` + +**Path Parameters:** +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `routeId` | string | Yes | GTFS route ID | + +**Query Parameters:** +| Parameter | Type | Required | Default | Description | +|-----------|------|----------|---------|-------------| +| `hours` | integer | No | 24 | Time window for analysis | + +**Response:** +```json +{ + "routeId": "125", + "periodHours": 24, + "statistics": { + "totalArrivals": 287, + "avgDelaySeconds": 62, + "medianDelaySeconds": 45, + "minDelaySeconds": -30, + "maxDelaySeconds": 420, + "stdDeviation": 78.5, + "onTimePercentage": 68.3, + "earlyPercentage": 8.7, + "latePercentage": 23.0, + "veryLatePercentage": 4.2 + }, + "thresholds": { + "onTime": "±5 minutes", + "veryLate": ">10 minutes" + } +} +``` + +**Example:** +```bash +curl "http://localhost:3000/api/stats/route/125/delays?hours=72" +``` + +**Definitions:** +- **On-time**: Within ±5 minutes of schedule +- **Early**: More than 5 minutes early +- **Late**: 5-10 minutes late +- **Very Late**: More than 10 minutes late + +--- + +### Get Stop Delay Statistics + +Get delay statistics for a specific stop. + +**Endpoint:** `GET /api/stats/stop/:stopId/delays` + +**Path Parameters:** +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `stopId` | string | Yes | GTFS stop ID | + +**Query Parameters:** +| Parameter | Type | Required | Default | Description | +|-----------|------|----------|---------|-------------| +| `hours` | integer | No | 24 | Time window for analysis | + +**Response:** +```json +{ + "stopId": "1563", + "periodHours": 24, + "statistics": { + "totalArrivals": 145, + "avgDelaySeconds": 72, + "medianDelaySeconds": 60, + "onTimePercentage": 62.1, + "routeBreakdown": [ + { + "routeId": "125", + "routeName": "7", + "arrivals": 89, + "avgDelaySeconds": 65 + }, + { + "routeId": "128", + "routeName": "2", + "arrivals": 56, + "avgDelaySeconds": 82 + } + ] + } +} +``` + +**Example:** +```bash +curl "http://localhost:3000/api/stats/stop/1563/delays?hours=48" +``` + +**Use Cases:** +- Compare reliability across stops +- Identify problematic locations +- Plan transfer reliability + +--- + +### Get Route Hourly Pattern + +Analyze delay patterns by hour of day for a route. + +**Endpoint:** `GET /api/stats/route/:routeId/hourly` + +**Path Parameters:** +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `routeId` | string | Yes | GTFS route ID | + +**Query Parameters:** +| Parameter | Type | Required | Default | Description | +|-----------|------|----------|---------|-------------| +| `days` | integer | No | 7 | Number of days to analyze | + +**Response:** +```json +{ + "routeId": "125", + "periodDays": 7, + "hourlyPattern": [ + { + "hour": 6, + "arrivals": 42, + "avgDelaySeconds": 15, + "onTimePercentage": 85.7 + }, + { + "hour": 7, + "arrivals": 89, + "avgDelaySeconds": 45, + "onTimePercentage": 71.9 + }, + { + "hour": 8, + "arrivals": 124, + "avgDelaySeconds": 125, + "onTimePercentage": 52.4 + } + ] +} +``` + +**Example:** +```bash +curl "http://localhost:3000/api/stats/route/125/hourly?days=14" +``` + +**Use Cases:** +- Identify rush hour patterns +- Compare morning vs evening reliability +- Optimize schedules based on actual performance + +--- + +## Data Models + +### GTFS Stop +```typescript +interface GtfsStop { + stop_id: string; // Unique stop identifier + stop_code: string; // Public-facing stop code + stop_name: string; // Stop name + stop_lat: number; // Latitude (WGS84) + stop_lon: number; // Longitude (WGS84) +} +``` + +### GTFS Route +```typescript +interface GtfsRoute { + route_id: string; // Unique route identifier + route_short_name: string; // Public route number (e.g., "7") + route_long_name: string; // Full route name + route_type?: string; // GTFS route type (0=Tram, 3=Bus, etc.) +} +``` + +### Vehicle Position +```typescript +interface VehiclePosition { + time: Date; // Timestamp (ISO 8601) + vehicle_id: string; // Vehicle identifier + route_id: string; // Route being served + trip_id?: string; // Current trip ID + latitude: number; // GPS latitude + longitude: number; // GPS longitude + speed?: number; // Speed in km/h + bearing?: number; // Direction (0-360°) + current_status: string; // IN_TRANSIT_TO, STOPPED_AT, UNKNOWN +} +``` + +### Arrival Record +```typescript +interface ArrivalRecord { + time: Date; // Record timestamp + stop_id: string; // Stop ID + route_id: string; // Route ID + scheduled_time: Date; // Scheduled arrival time + predicted_time?: Date; // Real-time predicted time + actual_time?: Date; // Actual arrival time (if observed) + delay_seconds: number; // Delay in seconds + is_realtime: boolean; // Based on real-time data + headsign?: string; // Trip headsign +} +``` + +--- + +## Error Handling + +All endpoints return standard HTTP status codes: + +| Status Code | Description | +|-------------|-------------| +| `200` | Success | +| `404` | Resource not found (e.g., invalid stop/route ID) | +| `500` | Server error | + +**Error Response Format:** +```json +{ + "error": "Description of the error", + "details": "Additional error details (when available)" +} +``` + +**Example Error:** +```bash +curl "http://localhost:3000/api/config?stopId=invalid" +``` + +Response (404): +```json +{ + "error": "Stop invalid not found" +} +``` + +--- + +## Rate Limiting + +Currently, there is no rate limiting implemented. For production deployments, consider: + +- Implementing per-IP rate limits +- Using caching for frequently accessed endpoints +- Load balancing for high-traffic scenarios + +**Recommended Polling Intervals:** +- Real-time arrivals: 10-30 seconds +- Vehicle positions: 10-30 seconds +- Historical data: As needed (not time-sensitive) +- Statistics: 5-60 minutes (depending on use case) + +--- + +## GTFS Data Update + +The GTFS static data (stops, routes, schedules) is loaded from the `gtfs/` directory on server startup. + +**To update GTFS data:** + +```bash +bun run setup-gtfs.ts +``` + +This downloads the latest GTFS feed and extracts it. Restart the server to load the new data. + +--- + +## TimescaleDB Setup + +Historical data and analytics require TimescaleDB. See the [README.md](README.md) for setup instructions. + +**Quick start:** +```bash +cd infrastructure +docker compose up -d +``` + +The database automatically: +- Creates hypertables for time-series data +- Sets up 90-day data retention +- Builds continuous aggregates for hourly metrics +- Maintains indexes for fast queries + +--- + +## Example Use Cases + +### Building a Real-Time Tracker + +```javascript +// 1. Get all stops +const stops = await fetch('/api/stops').then(r => r.json()); + +// 2. Get all routes +const routes = await fetch('/api/routes').then(r => r.json()); + +// 3. Poll for arrivals every 15 seconds +setInterval(async () => { + const arrivals = await fetch('/api/arrivals?stopId=1563&routeId=125') + .then(r => r.json()); + updateUI(arrivals); +}, 15000); + +// 4. Poll for vehicle positions +setInterval(async () => { + const vehicles = await fetch('/api/vehicles?routeId=125') + .then(r => r.json()); + updateMap(vehicles); +}, 15000); +``` + +### Building Analytics Dashboard + +```javascript +// Get delay statistics for route +const stats = await fetch('/api/stats/route/125/delays?hours=168') + .then(r => r.json()); + +// Get hourly patterns +const pattern = await fetch('/api/stats/route/125/hourly?days=30') + .then(r => r.json()); + +// Get stop performance comparison +const stopStats = await fetch('/api/stats/stop/1563/delays?hours=168') + .then(r => r.json()); + +// Display charts with the data +renderDelayChart(stats); +renderHourlyPattern(pattern); +renderStopComparison(stopStats); +``` + +--- + +## Additional Resources + +- **OpenAPI Specification**: [openapi.yaml](openapi.yaml) - Machine-readable API spec +- **Upstream API Documentation**: [UPSTREAM-API-DOCUMENTATION.md](UPSTREAM-API-DOCUMENTATION.md) - ModeShift GTFS API docs +- **Swagger UI Demo**: https://editor.swagger.io/ (paste openapi.yaml content) +- **GTFS Specification**: https://gtfs.org/ +- **GTFS-RT Reference**: https://gtfs.org/realtime/ +- **TimescaleDB Documentation**: https://docs.timescale.com/ +- **Project README**: [README.md](README.md) + +--- + +## Support + +For issues or questions: +1. Check the [README.md](README.md) for setup instructions +2. Review this documentation +3. Check the source code in [server.ts](server.ts) and [lib/gtfs.ts](lib/gtfs.ts) + +--- + +**Last Updated:** February 7, 2026 +**API Version:** 1.0.0 diff --git a/README.md b/README.md index 45a2db7..12c1277 100644 --- a/README.md +++ b/README.md @@ -12,12 +12,67 @@ npm run web Open **http://localhost:3000** +Visit **http://localhost:3000/analytics.html** for historical data and performance analytics. + +## TimescaleDB Setup + +The application uses TimescaleDB for storing time-series data (vehicle positions, arrivals, delays). + +### Start the database: + +```bash +cd infrastructure +docker compose up -d +``` + +### Configure environment: + +Create a `.env` file (or use the defaults): + +```bash +POSTGRES_HOST=localhost +POSTGRES_PORT=5432 +POSTGRES_DB=iot_data +POSTGRES_USER=postgres +POSTGRES_PASSWORD=example +``` + +The database will automatically: +- Create hypertables for efficient time-series queries +- Set up compression and retention policies (90 days) +- Build continuous aggregates for hourly metrics +- Index data for fast queries + +### Analytics Features: + +- **Vehicle Position History**: Track individual buses over time +- **Delay Analysis**: On-time performance, average delays, patterns +- **Hourly Patterns**: See when buses are typically late/early +- **Route Statistics**: Reliability scores, service quality metrics +- **Stop Performance**: Compare delays across different stops + +### Background Tracker: + +For continuous data collection without keeping the web interface open: + +```bash +npm run track +``` + +This automatically tracks these popular routes every 30 seconds: +- Routes: 2, 4, 5, 7, 15, 21, 22, 24 +- Private routes: 12П, 19П, 22П, 45П, 52П, 54П, 61П, 9П + +Data is stored in TimescaleDB for historical analysis. The tracker runs indefinitely until stopped with Ctrl+C. + ## Features - **Fully Modular Web Interface**: Select any stop and route via UI controls or URL parameters - **Dynamic Tracking**: Change stops/routes without restarting the server - Interactive map with live vehicle positions - Real-time arrivals with delays +- **Time-Series Data Storage**: Historical tracking with TimescaleDB +- **Analytics Dashboard**: Delay statistics, hourly patterns, performance metrics - 5-second auto-refresh (web), 10-second (terminal) - CLI arguments for terminal tracker - Configurable defaults via [config.ts](config.ts) @@ -32,6 +87,7 @@ npm run find -- --route "7" # Find route IDs by number/name npm run web # Web interface at http://localhost:3000 npm run tracker # Terminal interface (default) npm run tracker -- --stop 1571 --route 125 # Custom stop/route +npm run track # Background tracker for popular routes (30s intervals) npm start # Same as web ``` @@ -67,12 +123,34 @@ npm run tracker -- --help ### API Endpoints +**This Application's API:** +- Complete docs: **[API-DOCUMENTATION.md](API-DOCUMENTATION.md)** +- Interactive docs: http://localhost:3000/api-docs.html (when server is running) +- OpenAPI spec: **[openapi.yaml](openapi.yaml)** + +**Upstream ModeShift GTFS API:** +- Documentation: **[UPSTREAM-API-DOCUMENTATION.md](UPSTREAM-API-DOCUMENTATION.md)** +- Provider: ModeShift (Skopje public transport data) + +#### Quick Reference + Query parameters for custom tracking: ``` GET /api/config?stopId=1571&routeId=125 GET /api/arrivals?stopId=1571&routeId=125 GET /api/vehicles?routeId=125 +GET /api/stops # All stops +GET /api/routes # All routes + +# Historical Data APIs +GET /api/stats/db # Database statistics +GET /api/history/vehicle/:vehicleId?hours=24 +GET /api/history/route/:routeId/vehicles?hours=24 +GET /api/history/stop/:stopId/arrivals?routeId=125&hours=24 +GET /api/stats/route/:routeId/delays?hours=24 +GET /api/stats/stop/:stopId/delays?hours=24 +GET /api/stats/route/:routeId/hourly?days=7 ``` ## Configuration @@ -101,11 +179,17 @@ export const config: AppConfig = { terminal: 10000, // 10 seconds }, minutesAhead: 90, - }, -}; -``` - -## Structure + }, + analytics) +├── bus-tracker-json.ts # Terminal tracker (CLI args) +├── lib/ +│ ├── gtfs.ts # GTFS loader +│ └── database.ts # TimescaleDB time-series storage +├── public/ +│ ├── index.html # Live tracker UI +│ └── analytics.html # Analytics dashboard +├── infrastructure/ +│ └── compose.yml # TimescaleDB Docker setup +└── gtfs/ ure ``` bus/ @@ -116,8 +200,11 @@ bus/ ├── bus-tracker-json.ts # Terminal tracker (CLI args) ├── lib/gtfs.ts # GTFS loader ├── public/index.html # Frontend (modular UI) -└── gtfs/ # Static data (auto-downloaded) -``` +└─**TimescaleDB (PostgreSQL)** for time-series data +- Leaflet.js + OpenStreetMap +- Chart.js for analytics visualizations +- GTFS + GTFS-RT Protocol Buffers +- Docker Compose for database ## Stack diff --git a/UPSTREAM-API-DOCUMENTATION.md b/UPSTREAM-API-DOCUMENTATION.md new file mode 100644 index 0000000..156cea2 --- /dev/null +++ b/UPSTREAM-API-DOCUMENTATION.md @@ -0,0 +1,556 @@ +# ModeShift GTFS API Documentation + +Documentation for the upstream ModeShift GTFS API used by this Skopje Bus Tracker application. + +## Table of Contents + +- [Overview](#overview) +- [OpenAPI Specification](#openapi-specification) +- [Base URL](#base-url) +- [Authentication](#authentication) +- [Endpoints](#endpoints) + - [GTFS-RT Trip Updates](#gtfs-rt-trip-updates) + - [Public Vehicles (JSON)](#public-vehicles-json) + - [Nearby Stop Times](#nearby-stop-times) +- [Data Models](#data-models) +- [Usage Examples](#usage-examples) + +--- + +## Overview + +The ModeShift API provides real-time transit data for Skopje public transport based on the **GTFS (General Transit Feed Specification)** and **GTFS-Realtime** standards. + +**Provider:** ModeShift +**API Type:** GTFS + GTFS-Realtime +**Data Format:** JSON and Protocol Buffers +**Coverage:** Skopje, North Macedonia + +--- + +## OpenAPI Specification + +A complete **OpenAPI 3.0** specification for the ModeShift API is available at [upstream-openapi.yaml](upstream-openapi.yaml). + +This machine-readable specification includes: +- All upstream API endpoints +- Request/response schemas +- Parameter validation +- Examples for each operation + +You can view it interactively using: +- [Swagger Editor](https://editor.swagger.io/) - Paste the YAML content +- [Redoc](https://redocly.github.io/redoc/) - For a clean documentation view +- VS Code with OpenAPI extensions + +--- + +## Base URL + +``` +https://www.modeshift.app/api/v1/9814b106-2afe-47c8-919b-bdec6a5e521e +``` + +The base URL includes a tenant/agency identifier (`9814b106-2afe-47c8-919b-bdec6a5e521e`) specific to Skopje's transit system. + +--- + +## Authentication + +Currently, the API appears to be publicly accessible without authentication tokens. Requests can be made directly to the endpoints. + +--- + +## Endpoints + +### GTFS-RT Trip Updates + +Get real-time trip updates in GTFS-Realtime format (Protocol Buffers). + +**Endpoint:** `GET /transport/gtfsrt/tripupdates.pb` + +**Full URL:** +``` +https://www.modeshift.app/api/v1/9814b106-2afe-47c8-919b-bdec6a5e521e/transport/gtfsrt/tripupdates.pb +``` + +**Response Format:** Protocol Buffers (binary) + +**Content-Type:** `application/octet-stream` + +**Description:** +Returns a GTFS-Realtime FeedMessage containing TripUpdate entities. This follows the official [GTFS-Realtime specification](https://gtfs.org/realtime/). + +**Response Structure (Protobuf):** +```protobuf +message FeedMessage { + FeedHeader header = 1; + repeated FeedEntity entity = 2; +} + +message FeedEntity { + required string id = 1; + optional TripUpdate trip_update = 3; +} + +message TripUpdate { + optional TripDescriptor trip = 1; + optional VehicleDescriptor vehicle = 3; + repeated StopTimeUpdate stop_time_update = 2; +} +``` + +**Example Usage (JavaScript):** +```javascript +import GtfsRealtimeBindings from 'gtfs-realtime-bindings'; + +const response = await fetch( + 'https://www.modeshift.app/api/v1/9814b106-2afe-47c8-919b-bdec6a5e521e/transport/gtfsrt/tripupdates.pb' +); +const buffer = await response.arrayBuffer(); +const feed = GtfsRealtimeBindings.transit_realtime.FeedMessage.decode( + new Uint8Array(buffer) +); + +for (const entity of feed.entity) { + if (entity.tripUpdate) { + console.log('Trip:', entity.tripUpdate.trip.tripId); + console.log('Route:', entity.tripUpdate.trip.routeId); + console.log('Vehicle:', entity.tripUpdate.vehicle?.id); + } +} +``` + +**Key Fields:** +- `trip.tripId`: GTFS trip identifier +- `trip.routeId`: GTFS route identifier +- `vehicle.id`: Vehicle identification number +- `vehicle.label`: Public-facing vehicle number +- `stopTimeUpdate[]`: Array of stop time predictions + +--- + +### Public Vehicles (JSON) + +Get current positions and status of all active vehicles in JSON format. + +**Endpoint:** `GET /transport/public/vehicles` + +**Full URL:** +``` +https://www.modeshift.app/api/v1/9814b106-2afe-47c8-919b-bdec6a5e521e/transport/public/vehicles +``` + +**Response Format:** JSON + +**Description:** +Returns an array of all currently active vehicles with their real-time GPS positions, speed, bearing, and status information. + +**Response Structure:** +```json +[ + { + "id": 12345, + "identificationNumber": "MK-SK-1234", + "inventoryNumber": 1234, + "positionLatitude": 41.9981, + "positionLongitude": 21.4254, + "positionBearing": 87.5, + "positionSpeed": 15.3, + "positionModifiedAt": "2026-02-07T14:27:15Z", + "status": 2 + }, + { + "id": 12346, + "identificationNumber": "MK-SK-5678", + "inventoryNumber": 5678, + "positionLatitude": 42.0012, + "positionLongitude": 21.4301, + "positionBearing": 180.0, + "positionSpeed": 0.0, + "positionModifiedAt": "2026-02-07T14:27:10Z", + "status": 1 + } +] +``` + +**Fields:** + +| Field | Type | Description | +|-------|------|-------------| +| `id` | integer | Internal database identifier for the vehicle | +| `identificationNumber` | string | Vehicle license plate or identification (e.g., "MK-SK-1234") | +| `inventoryNumber` | integer | Public-facing bus number shown on the vehicle | +| `positionLatitude` | number | Current GPS latitude (WGS84) | +| `positionLongitude` | number | Current GPS longitude (WGS84) | +| `positionBearing` | number | Direction of travel in degrees (0-360, 0=North, 90=East) | +| `positionSpeed` | number | Current speed in km/h | +| `positionModifiedAt` | string | ISO 8601 timestamp of last position update | +| `status` | integer | Vehicle status code (1=stopped, 2=in transit) | + +**Status Codes:** +- `1` - Vehicle is stopped +- `2` - Vehicle is in transit + +**Example Usage:** +```bash +curl "https://www.modeshift.app/api/v1/9814b106-2afe-47c8-919b-bdec6a5e521e/transport/public/vehicles" +``` + +**Use Cases:** +- Display all vehicles on a map +- Match vehicles to routes using trip update data +- Track real-time vehicle movements + +--- + +### Nearby Stop Times + +Get arrival predictions for stops near a specific location. + +**Endpoint:** `GET /transport/planner/stops/nearbyTimes` + +**Full URL:** +``` +https://www.modeshift.app/api/v1/9814b106-2afe-47c8-919b-bdec6a5e521e/transport/planner/stops/nearbyTimes +``` + +**Query Parameters:** + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `latitude` | number | Yes | Search center latitude (WGS84) | +| `longitude` | number | Yes | Search center longitude (WGS84) | +| `radius` | integer | Yes | Search radius in meters | + +**Response Format:** JSON + +**Description:** +Returns stops within the specified radius along with their upcoming arrivals, including real-time predictions. + +**Response Structure:** +```json +[ + { + "id": 1563, + "code": "363", + "name": "KARPOS II - STIV NAUMOV", + "latitude": 41.97964, + "longitude": 21.41283, + "patterns": [ + { + "routeId": 125, + "routeShortName": "7", + "routeLongName": "Centar - GTC - Karpos 4 - Karposh 2", + "headsign": "Karpos 2", + "stopTimes": [ + { + "serviceDay": 1738972800, + "scheduledArrival": 52380, + "realtimeArrival": 52500, + "arrivalDelay": 120, + "realtime": true, + "realtimeState": "UPDATED", + "headsign": "Karpos 2" + }, + { + "serviceDay": 1738972800, + "scheduledArrival": 53280, + "realtimeArrival": 53280, + "arrivalDelay": 0, + "realtime": true, + "realtimeState": "SCHEDULED", + "headsign": "Karpos 2" + } + ] + } + ] + } +] +``` + +**Fields:** + +**Stop Level:** +| Field | Type | Description | +|-------|------|-------------| +| `id` | integer | GTFS stop ID | +| `code` | string | Public stop code | +| `name` | string | Stop name | +| `latitude` | number | Stop latitude | +| `longitude` | number | Stop longitude | +| `patterns[]` | array | Array of route patterns serving this stop | + +**Pattern Level:** +| Field | Type | Description | +|-------|------|-------------| +| `routeId` | integer | GTFS route ID | +| `routeShortName` | string | Public route number (e.g., "7") | +| `routeLongName` | string | Full route name | +| `headsign` | string | Direction/destination | +| `stopTimes[]` | array | Array of upcoming arrivals | + +**Stop Time Level:** +| Field | Type | Description | +|-------|------|-------------| +| `serviceDay` | integer | Unix timestamp (seconds) of service day start (midnight) | +| `scheduledArrival` | integer | Seconds since midnight for scheduled arrival time | +| `realtimeArrival` | integer | Seconds since midnight for predicted arrival time | +| `arrivalDelay` | integer | Delay in seconds (positive=late, negative=early) | +| `realtime` | boolean | Whether prediction is based on real-time data | +| `realtimeState` | string | State: `SCHEDULED`, `UPDATED`, `CANCELED` | +| `headsign` | string | Trip headsign (destination) | + +**Calculating Arrival Time:** +```javascript +const arrival = new Date( + (stopTime.serviceDay + stopTime.realtimeArrival) * 1000 +); +``` + +**Example Usage:** +```bash +# Get arrivals near American College +curl "https://www.modeshift.app/api/v1/9814b106-2afe-47c8-919b-bdec6a5e521e/transport/planner/stops/nearbyTimes?latitude=42.00437&longitude=21.39687&radius=50" +``` + +**Use Cases:** +- Display upcoming arrivals at a stop +- Show real-time delay information +- Calculate minutes until arrival +- Filter by specific routes + +--- + +## Data Models + +### GTFS Stop +```typescript +interface Stop { + id: number; // GTFS stop_id (as integer) + code: string; // Public stop code + name: string; // Stop name + latitude: number; // WGS84 latitude + longitude: number; // WGS84 longitude + patterns: Pattern[]; // Routes serving this stop +} +``` + +### Route Pattern +```typescript +interface Pattern { + routeId: number; // GTFS route_id (as integer) + routeShortName: string; // Public route number + routeLongName: string; // Full route name + headsign: string; // Direction/destination + stopTimes: StopTime[]; // Upcoming arrivals +} +``` + +### Stop Time +```typescript +interface StopTime { + serviceDay: number; // Unix timestamp of service day start + scheduledArrival: number; // Seconds since midnight (scheduled) + realtimeArrival: number; // Seconds since midnight (predicted) + arrivalDelay: number; // Delay in seconds + realtime: boolean; // Based on real-time data + realtimeState: string; // SCHEDULED, UPDATED, CANCELED + headsign: string; // Trip headsign +} +``` + +### Vehicle +```typescript +interface Vehicle { + id: number; // Internal ID + identificationNumber: string; // License plate + inventoryNumber: number; // Bus number + positionLatitude: number; // GPS latitude + positionLongitude: number; // GPS longitude + positionBearing: number; // Direction (0-360°) + positionSpeed: number; // Speed in km/h + positionModifiedAt: string; // ISO 8601 timestamp + status: number; // 1=stopped, 2=in_transit +} +``` + +--- + +## Usage Examples + +### Complete Real-Time Tracker Flow + +```javascript +// 1. Get a specific stop's arrivals +const stopLat = 41.97964; +const stopLon = 21.41283; +const radius = 50; + +const nearbyResponse = await fetch( + `https://www.modeshift.app/api/v1/9814b106-2afe-47c8-919b-bdec6a5e521e/transport/planner/stops/nearbyTimes?latitude=${stopLat}&longitude=${stopLon}&radius=${radius}` +); +const stops = await nearbyResponse.json(); + +// 2. Process arrivals for a specific route +const targetRouteId = 125; +const now = new Date(); + +for (const stop of stops) { + for (const pattern of stop.patterns) { + if (pattern.routeId === targetRouteId) { + for (const stopTime of pattern.stopTimes) { + const arrivalTime = new Date( + (stopTime.serviceDay + stopTime.realtimeArrival) * 1000 + ); + const minutesUntil = Math.floor( + (arrivalTime.getTime() - now.getTime()) / 60000 + ); + + console.log(`Route ${pattern.routeShortName} - ${minutesUntil} min`); + console.log(`Delay: ${stopTime.arrivalDelay} seconds`); + } + } + } +} + +// 3. Get all vehicle positions +const vehiclesResponse = await fetch( + 'https://www.modeshift.app/api/v1/9814b106-2afe-47c8-919b-bdec6a5e521e/transport/public/vehicles' +); +const vehicles = await vehiclesResponse.json(); + +// 4. Get trip updates to match vehicles to routes +const tripUpdatesResponse = await fetch( + 'https://www.modeshift.app/api/v1/9814b106-2afe-47c8-919b-bdec6a5e521e/transport/gtfsrt/tripupdates.pb' +); +const buffer = await tripUpdatesResponse.arrayBuffer(); +const feed = GtfsRealtimeBindings.transit_realtime.FeedMessage.decode( + new Uint8Array(buffer) +); + +// 5. Build vehicle-to-route mapping +const vehicleRouteMap = new Map(); +for (const entity of feed.entity) { + if (entity.tripUpdate?.vehicle?.id && entity.tripUpdate?.trip?.routeId) { + vehicleRouteMap.set( + entity.tripUpdate.vehicle.id, + entity.tripUpdate.trip.routeId + ); + } +} + +// 6. Filter vehicles by route +const routeVehicles = vehicles.filter(v => { + const vehicleKey = v.identificationNumber || v.inventoryNumber.toString(); + return vehicleRouteMap.get(vehicleKey) == targetRouteId; +}); + +console.log(`Found ${routeVehicles.length} vehicles on route ${targetRouteId}`); +``` + +### Polling for Updates + +```javascript +// Recommended polling intervals +const VEHICLE_UPDATE_INTERVAL = 10000; // 10 seconds +const ARRIVAL_UPDATE_INTERVAL = 15000; // 15 seconds +const TRIP_UPDATE_INTERVAL = 10000; // 10 seconds + +// Poll for vehicle positions +setInterval(async () => { + const vehicles = await fetch(vehiclesEndpoint).then(r => r.json()); + updateMap(vehicles); +}, VEHICLE_UPDATE_INTERVAL); + +// Poll for arrival predictions +setInterval(async () => { + const arrivals = await fetch(nearbyTimesEndpoint).then(r => r.json()); + updateArrivalBoard(arrivals); +}, ARRIVAL_UPDATE_INTERVAL); +``` + +--- + +## GTFS Static Data + +The API uses GTFS static data which should be downloaded separately. Static data includes: + +- **stops.txt**: All stop locations and names +- **routes.txt**: All route definitions +- **trips.txt**: Trip schedules +- **stop_times.txt**: Stop time schedules +- **calendar.txt**: Service calendar +- **shapes.txt**: Route shapes (geographic paths) + +**Note:** The static GTFS feed download URL should be obtained from ModeShift or the transit agency. + +--- + +## Rate Limiting & Best Practices + +1. **Polling Frequency**: + - Don't poll more frequently than every 10 seconds + - Cache responses when appropriate + - Use conditional requests when possible + +2. **Radius Parameter**: + - Use smallest radius that meets your needs (typically 50-100m) + - Larger radius = more data returned = slower response + +3. **Error Handling**: + - Handle network errors gracefully + - Implement exponential backoff for failures + - Parse protobuf data carefully (can be empty) + +4. **Data Volume**: + - Vehicle positions endpoint returns ALL vehicles (100+) + - Filter client-side rather than making multiple requests + - Consider caching vehicle positions + +--- + +## Common Issues & Solutions + +### Empty GTFS-RT Feed +**Problem:** `tripupdates.pb` returns empty or zero-length buffer +**Solution:** This can happen during off-peak hours or system maintenance. Handle gracefully and retry later. + +### Vehicle Matching +**Problem:** Vehicles in JSON don't match trip updates +**Solution:** Try matching by both `identificationNumber` and `inventoryNumber`. Some vehicles may not have active trips. + +### Time Zone Issues +**Problem:** Arrival times are incorrect +**Solution:** The API uses Europe/Skopje timezone. Service day timestamps are at midnight local time. + +### Delay Calculation +**Problem:** Delays seem wrong +**Solution:** Use `realtimeArrival - scheduledArrival` (already provided as `arrivalDelay` field). + +--- + +## Related Documentation + +- **GTFS Specification**: https://gtfs.org/ +- **GTFS-Realtime**: https://gtfs.org/realtime/ +- **Protocol Buffers**: https://protobuf.dev/ +- **gtfs-realtime-bindings**: https://www.npmjs.com/package/gtfs-realtime-bindings + +--- + +## Support + +For issues with the ModeShift API: +- Contact ModeShift support +- Check ModeShift documentation at https://www.modeshift.app/ + +For issues with this tracker application: +- See [README.md](README.md) +- Check [API-DOCUMENTATION.md](API-DOCUMENTATION.md) for the local API + +--- + +**Last Updated:** February 7, 2026 +**API Provider:** ModeShift +**Coverage:** Skopje, North Macedonia diff --git a/background-tracker.ts b/background-tracker.ts new file mode 100644 index 0000000..12914a0 --- /dev/null +++ b/background-tracker.ts @@ -0,0 +1,406 @@ +#!/usr/bin/env bun +/** + * Background tracker for popular bus routes in Skopje + * Continuously monitors GTFS-RT feeds and stores data to TimescaleDB + */ + +import GtfsRealtimeBindings from 'gtfs-realtime-bindings'; +import { config } from './config'; +import { GtfsRoute, GtfsStop, loadGtfsRoutes, loadGtfsStops } from './lib/gtfs'; +import { + initDatabase, + logVehiclePositions, + logArrival, + VehiclePosition +} from './lib/database'; + +// Popular routes to track +const TRACKED_ROUTES = [ + { id: '245', name: 'Route 2' }, + { id: '121', name: 'Route 4' }, + { id: '123', name: 'Route 5' }, + { id: '125', name: 'Route 7' }, + { id: '129', name: 'Route 15' }, + { id: '134', name: 'Route 21' }, + { id: '136', name: 'Route 22' }, + { id: '138', name: 'Route 24' }, + // Private routes (П) + { id: '203', name: 'Route 22 П' }, + { id: '204', name: 'Route 12 П' }, +]; + +// Key stops to monitor for arrival predictions and delay tracking +// These are the busiest transit points in Skopje +const MONITORED_STOPS = [ + // Central area and major transit hubs + '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '15', + '16', '17', '18', '19', '20', '21', + // Transit centers and bus stations + '47', '48', '49', '50', '51', '52', '53', '54', '55', '56', '57', '58', '59', '60', + '61', '62', '63', '64', '65', '66', '67', '68', + // Kisela Voda area + '174', '175', '176', '177', '178', '179', '180', '181', + // Gjorche Petrov corridor + '246', '247', '248', '249', '250', '251', '252', '253', '254', '255', '256', '257', + '258', '259', '260', '261', '262', '263', '264', '265', '266', '267', '268', '269', + // Karposh areas + '270', '271', '272', '273', '274', '275', '276', '277', '278', '279', '280', '281', + '282', '283', '284', '285', '286', '287', '288', '289', +]; + +const REFRESH_INTERVAL = 30000; // 30 seconds +const ARRIVAL_STOP_CAP = 150; // Max stops to query per cycle + +let stats = { + cycles: 0, + lastUpdate: new Date(), + vehiclesTracked: 0, + arrivalsLogged: 0, + errors: 0, +}; + +// Load GTFS stops data +const stops = loadGtfsStops(); +const stopsByCode = new Map(); + +for (const stop of stops.values()) { + if (stop.stop_code) { + stopsByCode.set(stop.stop_code, stop); + } +} + +// Load GTFS routes data to map route short names back to IDs +const routes = loadGtfsRoutes(); +const trackedRouteIds = new Set(TRACKED_ROUTES.map(r => r.id)); +const routeIdByShortName = new Map(); +const routeIdByShortNameNormalized = new Map(); + +function normalizeRouteKey(value: string): string { + return value.replace(/\s+/g, ' ').trim(); +} + +for (const routeId of trackedRouteIds) { + const route = routes.get(routeId); + if (route?.route_short_name) { + const shortName = route.route_short_name; + routeIdByShortName.set(shortName, routeId); + routeIdByShortNameNormalized.set(normalizeRouteKey(shortName), routeId); + } +} + +// Fetch and process vehicle positions +async function trackVehicles() { + try { + console.log(`[${new Date().toISOString()}] Fetching vehicle data...`); + + // Fetch all vehicles from JSON API + const vehiclesResponse = await fetch(config.apiEndpoints.vehiclesJson); + + if (!vehiclesResponse.ok) { + throw new Error(`HTTP error! status: ${vehiclesResponse.status}`); + } + + const allVehicles = await vehiclesResponse.json() as any[]; + console.log(` Found ${allVehicles.length} total vehicles`); + + // Fetch trip updates to match vehicles to routes + const tripUpdatesResponse = await fetch(config.apiEndpoints.gtfsRtTripUpdates); + + if (!tripUpdatesResponse.ok) { + console.warn(' Could not fetch trip updates'); + return; + } + + const buffer = await tripUpdatesResponse.arrayBuffer(); + + if (buffer.byteLength === 0) { + console.warn(' Empty trip updates feed'); + return; + } + + let feed; + try { + feed = GtfsRealtimeBindings.transit_realtime.FeedMessage.decode( + new Uint8Array(buffer) + ); + } catch (decodeError) { + console.error(' Failed to decode GTFS-RT feed:', decodeError); + stats.errors++; + return; + } + + // Build map of vehicle IDs to route IDs for tracked routes + const vehicleRouteMap = new Map(); + const trackedRouteIds = new Set(TRACKED_ROUTES.map(r => r.id)); + + for (const entity of feed.entity) { + if (!entity.tripUpdate) continue; + + const tripUpdate = entity.tripUpdate; + const routeId = tripUpdate.trip?.routeId; + + // Only process tracked routes + if (!routeId || !trackedRouteIds.has(routeId)) continue; + + // Get vehicle ID and label - store both as keys + const vehicleId = tripUpdate.vehicle?.id; + const vehicleLabel = tripUpdate.vehicle?.label; + const tripInfo = { + routeId: routeId, + tripId: tripUpdate.trip.tripId || '', + }; + + if (vehicleId) { + vehicleRouteMap.set(vehicleId, tripInfo); + } + if (vehicleLabel && vehicleLabel !== vehicleId) { + vehicleRouteMap.set(vehicleLabel, tripInfo); + } + } + + console.log(` Matched ${vehicleRouteMap.size} vehicles to tracked routes`); + + // Debug: Show sample vehicle IDs from both sources + if (vehicleRouteMap.size > 0) { + const sampleGtfsIds = Array.from(vehicleRouteMap.keys()).slice(0, 5); + console.log(` Sample GTFS-RT vehicle IDs: ${sampleGtfsIds.join(', ')}`); + } + + if (allVehicles.length > 0) { + const sampleJsonIds = allVehicles.slice(0, 5).map(v => + `${v.identificationNumber || v.inventoryNumber || 'unknown'}` + ); + console.log(` Sample JSON API vehicle IDs: ${sampleJsonIds.join(', ')}`); + } + + // Filter and prepare vehicle positions for tracked routes + const positions: VehiclePosition[] = []; + const now = Date.now(); + + for (const vehicle of allVehicles) { + const identificationNumber = vehicle.identificationNumber; + const inventoryNumber = vehicle.inventoryNumber?.toString(); + + if (!identificationNumber && !inventoryNumber) continue; + + // Try multiple matching strategies + let routeInfo = vehicleRouteMap.get(identificationNumber) || + vehicleRouteMap.get(inventoryNumber); + + if (routeInfo) { + positions.push({ + timestamp: now, + vehicleId: identificationNumber || inventoryNumber, + routeId: routeInfo.routeId, + tripId: routeInfo.tripId, + latitude: vehicle.positionLatitude, + longitude: vehicle.positionLongitude, + speed: vehicle.positionSpeed, + bearing: vehicle.positionBearing, + currentStatus: vehicle.status === 2 ? 'IN_TRANSIT_TO' : 'UNKNOWN', + }); + } + } + + // Log to database + if (positions.length > 0) { + await logVehiclePositions(positions); + console.log(` [OK] Logged ${positions.length} vehicle positions`); + stats.vehiclesTracked += positions.length; + } else { + console.log(` [WARN] No vehicles found for tracked routes`); + } + + stats.cycles++; + stats.lastUpdate = new Date(); + + } catch (error) { + console.error(' [ERROR] Error tracking vehicles:', error); + stats.errors++; + } +} + +// Track arrivals at monitored stops for traffic pattern analysis +async function trackArrivals() { + try { + const stopsToQuery = MONITORED_STOPS.slice(0, ARRIVAL_STOP_CAP); + console.log(`[${new Date().toISOString()}] Tracking arrivals at ${stopsToQuery.length} key stops...`); + + const now = new Date(); + let arrivalsLogged = 0; + let arrivalsFound = 0; + let duplicates = 0; + const unmatchedRoutes = new Map(); + const matchedRouteCounts = new Map(); + + // Process each monitored stop + for (const stopKey of stopsToQuery) { + const stop = stops.get(stopKey) || stopsByCode.get(stopKey); + if (!stop) continue; + + const stopId = stop.stop_id; + const stopCode = stop.stop_code; + + try { + // Fetch arrivals near this stop + const radius = 50; + const nearbyUrl = `${config.baseUrl}/transport/planner/stops/nearbyTimes?latitude=${stop.stop_lat}&longitude=${stop.stop_lon}&radius=${radius}`; + + const response = await fetch(nearbyUrl); + if (!response.ok) continue; + + const nearbyData = await response.json() as any[]; + + // Process arrivals for tracked routes + for (const stopData of nearbyData) { + const apiStopId = stopData.id?.toString(); + if (apiStopId !== stopId && apiStopId !== stopCode) continue; + + for (const pattern of stopData.patterns) { + const routeKey = pattern.routeId?.toString(); + if (!routeKey) continue; + + const normalizedRouteKey = normalizeRouteKey(routeKey); + + // Only track our monitored routes (match by route_id or short name) + const canonicalRouteId = trackedRouteIds.has(routeKey) + ? routeKey + : routeIdByShortName.get(routeKey) || routeIdByShortNameNormalized.get(normalizedRouteKey); + + if (!canonicalRouteId) { + unmatchedRoutes.set(routeKey, (unmatchedRoutes.get(routeKey) || 0) + 1); + continue; + } + + matchedRouteCounts.set( + canonicalRouteId, + (matchedRouteCounts.get(canonicalRouteId) || 0) + 1 + ); + + for (const stopTime of pattern.stopTimes) { + const serviceDay = new Date(stopTime.serviceDay * 1000); + const arrivalTime = new Date(serviceDay.getTime() + stopTime.realtimeArrival * 1000); + const scheduledTime = new Date(serviceDay.getTime() + stopTime.scheduledArrival * 1000); + + const minutesUntil = Math.floor((arrivalTime.getTime() - now.getTime()) / 60000); + + // Log arrivals in the next 60 minutes + if (minutesUntil >= -2 && minutesUntil <= 60) { + arrivalsFound++; + try { + await logArrival({ + timestamp: scheduledTime.getTime(), + stopId, + routeId: canonicalRouteId, + scheduledTime: scheduledTime.getTime(), + predictedTime: arrivalTime.getTime(), + delaySeconds: stopTime.arrivalDelay, + isRealtime: stopTime.realtime, + headsign: stopTime.headsign, + }); + arrivalsLogged++; + } catch (dbError) { + duplicates++; + } + } + } + } + } + + // Small delay to avoid overwhelming the API + await new Promise(resolve => setTimeout(resolve, 100)); + + } catch (stopError) { + // Skip this stop if there's an error + continue; + } + } + if (arrivalsFound === 0) { + console.log(` [INFO] No arrivals found in time window`); + } else if (arrivalsLogged > 0) { + console.log(` [OK] Logged ${arrivalsLogged} new arrivals (${duplicates} duplicates skipped, ${arrivalsFound} total found)`); + stats.arrivalsLogged += arrivalsLogged; + } else { + console.log(` [INFO] Found ${arrivalsFound} arrivals but all were duplicates (already in database)`); + } + + if (matchedRouteCounts.size > 0) { + const matchedSummary = Array.from(matchedRouteCounts.entries()) + .sort((a, b) => b[1] - a[1]) + .slice(0, 10) + .map(([route, count]) => `${route}:${count}`) + .join(', '); + console.log(` [DEBUG] Matched route IDs: ${matchedSummary}`); + } + + if (unmatchedRoutes.size > 0) { + const topUnmatched = Array.from(unmatchedRoutes.entries()) + .sort((a, b) => b[1] - a[1]) + .slice(0, 8) + .map(([route, count]) => `${route}:${count}`) + .join(', '); + console.log(` [DEBUG] Unmatched route keys: ${topUnmatched}`); + } + + } catch (error) { + console.error(' [ERROR] Error tracking arrivals:', error); + stats.errors++; + } +} + +// Print statistics +function printStats() { + console.log('\n' + '='.repeat(60)); + console.log('Background Tracker Statistics'); + console.log('='.repeat(60)); + console.log(`Tracking ${TRACKED_ROUTES.length} routes:`); + TRACKED_ROUTES.forEach(r => console.log(` - ${r.name} (ID: ${r.id})`)); + console.log(`Monitoring up to ${ARRIVAL_STOP_CAP} key stops for arrival data`); + console.log(`\nCycles completed: ${stats.cycles}`); + console.log(`Vehicle positions tracked: ${stats.vehiclesTracked}`); + console.log(`Arrival predictions logged: ${stats.arrivalsLogged}`); + console.log(`Errors: ${stats.errors}`); + console.log(`Last update: ${stats.lastUpdate.toLocaleString()}`); + console.log(`Refresh interval: ${REFRESH_INTERVAL / 1000}s`); + console.log('='.repeat(60) + '\n'); +} + +// Main loop +async function main() { + console.log('\nStarting Background Bus Tracker for Popular Routes & Stops\n'); + + // Initialize database + try { + await initDatabase(); + } catch (error) { + console.error('Failed to initialize database:', error); + console.log('Continuing without data logging...'); + } + + // Print tracked routes + printStats(); + + // Initial fetch + await trackVehicles(); + await trackArrivals(); + + // Set up recurring fetch + setInterval(async () => { + await trackVehicles(); + await trackArrivals(); + }, REFRESH_INTERVAL); + + // Print stats every 5 minutes + setInterval(() => { + printStats(); + }, 5 * 60 * 1000); + + // Handle graceful shutdown + process.on('SIGINT', () => { + console.log('\n\nShutting down tracker...'); + printStats(); + process.exit(0); + }); +} + +main().catch(console.error); diff --git a/bun.lock b/bun.lock index 3f1e08a..83d80cf 100644 --- a/bun.lock +++ b/bun.lock @@ -6,8 +6,17 @@ "name": "skopje-bus-api-client", "dependencies": { "express": "^4.18.2", + "pg": "^8.18.0", "protobufjs": "^7.5.4", }, + "devDependencies": { + "@types/express": "^4.17.21", + "@types/node": "^20.10.0", + "@types/pg": "^8.11.0", + "protobufjs-cli": "^2.0.0", + "ts-node": "^10.9.1", + "typescript": "^5.9.3", + }, "optionalDependencies": { "gtfs-realtime-bindings": "^1.1.1", }, @@ -22,6 +31,14 @@ "@babel/types": ["@babel/types@7.29.0", "", { "dependencies": { "@babel/helper-string-parser": "^7.27.1", "@babel/helper-validator-identifier": "^7.28.5" } }, "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A=="], + "@cspotcode/source-map-support": ["@cspotcode/source-map-support@0.8.1", "", { "dependencies": { "@jridgewell/trace-mapping": "0.3.9" } }, "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw=="], + + "@jridgewell/resolve-uri": ["@jridgewell/resolve-uri@3.1.2", "", {}, "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw=="], + + "@jridgewell/sourcemap-codec": ["@jridgewell/sourcemap-codec@1.5.5", "", {}, "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og=="], + + "@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.9", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.0.3", "@jridgewell/sourcemap-codec": "^1.4.10" } }, "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ=="], + "@jsdoc/salty": ["@jsdoc/salty@0.2.9", "", { "dependencies": { "lodash": "^4.17.21" } }, "sha512-yYxMVH7Dqw6nO0d5NIV8OQWnitU8k6vXH8NtgqAfIa/IUqRMxRv/NUJJ08VEKbAakwxlgBl5PJdrU0dMPStsnw=="], "@protobufjs/aspromise": ["@protobufjs/aspromise@1.1.2", "", {}, "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ=="], @@ -44,22 +61,56 @@ "@protobufjs/utf8": ["@protobufjs/utf8@1.1.0", "", {}, "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw=="], + "@tsconfig/node10": ["@tsconfig/node10@1.0.12", "", {}, "sha512-UCYBaeFvM11aU2y3YPZ//O5Rhj+xKyzy7mvcIoAjASbigy8mHMryP5cK7dgjlz2hWxh1g5pLw084E0a/wlUSFQ=="], + + "@tsconfig/node12": ["@tsconfig/node12@1.0.11", "", {}, "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag=="], + + "@tsconfig/node14": ["@tsconfig/node14@1.0.3", "", {}, "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow=="], + + "@tsconfig/node16": ["@tsconfig/node16@1.0.4", "", {}, "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA=="], + + "@types/body-parser": ["@types/body-parser@1.19.6", "", { "dependencies": { "@types/connect": "*", "@types/node": "*" } }, "sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g=="], + + "@types/connect": ["@types/connect@3.4.38", "", { "dependencies": { "@types/node": "*" } }, "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug=="], + + "@types/express": ["@types/express@4.17.25", "", { "dependencies": { "@types/body-parser": "*", "@types/express-serve-static-core": "^4.17.33", "@types/qs": "*", "@types/serve-static": "^1" } }, "sha512-dVd04UKsfpINUnK0yBoYHDF3xu7xVH4BuDotC/xGuycx4CgbP48X/KF/586bcObxT0HENHXEU8Nqtu6NR+eKhw=="], + + "@types/express-serve-static-core": ["@types/express-serve-static-core@4.19.8", "", { "dependencies": { "@types/node": "*", "@types/qs": "*", "@types/range-parser": "*", "@types/send": "*" } }, "sha512-02S5fmqeoKzVZCHPZid4b8JH2eM5HzQLZWN2FohQEy/0eXTq8VXZfSN6Pcr3F6N9R/vNrj7cpgbhjie6m/1tCA=="], + + "@types/http-errors": ["@types/http-errors@2.0.5", "", {}, "sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg=="], + "@types/linkify-it": ["@types/linkify-it@5.0.0", "", {}, "sha512-sVDA58zAw4eWAffKOaQH5/5j3XeayukzDk+ewSsnv3p4yJEZHCCzMDiZM8e0OUrRvmpGZ85jf4yDHkHsgBNr9Q=="], "@types/markdown-it": ["@types/markdown-it@14.1.2", "", { "dependencies": { "@types/linkify-it": "^5", "@types/mdurl": "^2" } }, "sha512-promo4eFwuiW+TfGxhi+0x3czqTYJkG8qB17ZUJiVF10Xm7NLVRSLUsfRTU/6h1e24VvRnXCx+hG7li58lkzog=="], "@types/mdurl": ["@types/mdurl@2.0.0", "", {}, "sha512-RGdgjQUZba5p6QEFAVx2OGb8rQDL/cPRG7GiedRzMcJ1tYnUANBncjbSB1NRGwbvjcPeikRABz2nshyPk1bhWg=="], + "@types/mime": ["@types/mime@1.3.5", "", {}, "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w=="], + "@types/node": ["@types/node@20.19.32", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-Ez8QE4DMfhjjTsES9K2dwfV258qBui7qxUsoaixZDiTzbde4U12e1pXGNu/ECsUIOi5/zoCxAQxIhQnaUQ2VvA=="], + "@types/pg": ["@types/pg@8.16.0", "", { "dependencies": { "@types/node": "*", "pg-protocol": "*", "pg-types": "^2.2.0" } }, "sha512-RmhMd/wD+CF8Dfo+cVIy3RR5cl8CyfXQ0tGgW6XBL8L4LM/UTEbNXYRbLwU6w+CgrKBNbrQWt4FUtTfaU5jSYQ=="], + + "@types/qs": ["@types/qs@6.14.0", "", {}, "sha512-eOunJqu0K1923aExK6y8p6fsihYEn/BYuQ4g0CxAAgFc4b/ZLN4CrsRZ55srTdqoiLzU2B2evC+apEIxprEzkQ=="], + + "@types/range-parser": ["@types/range-parser@1.2.7", "", {}, "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ=="], + + "@types/send": ["@types/send@1.2.1", "", { "dependencies": { "@types/node": "*" } }, "sha512-arsCikDvlU99zl1g69TcAB3mzZPpxgw0UQnaHeC1Nwb015xp8bknZv5rIfri9xTOcMuaVgvabfIRA7PSZVuZIQ=="], + + "@types/serve-static": ["@types/serve-static@1.15.10", "", { "dependencies": { "@types/http-errors": "*", "@types/node": "*", "@types/send": "<1" } }, "sha512-tRs1dB+g8Itk72rlSI2ZrW6vZg0YrLI81iQSTkMmOqnqCaNr/8Ek4VwWcN5vZgCYWbg/JJSGBlUaYGAOP73qBw=="], + "accepts": ["accepts@1.3.8", "", { "dependencies": { "mime-types": "~2.1.34", "negotiator": "0.6.3" } }, "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw=="], "acorn": ["acorn@8.15.0", "", { "bin": "bin/acorn" }, "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg=="], "acorn-jsx": ["acorn-jsx@5.3.2", "", { "peerDependencies": { "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ=="], + "acorn-walk": ["acorn-walk@8.3.4", "", { "dependencies": { "acorn": "^8.11.0" } }, "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g=="], + "ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], + "arg": ["arg@4.1.3", "", {}, "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA=="], + "argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], "array-flatten": ["array-flatten@1.1.1", "", {}, "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg=="], @@ -94,6 +145,8 @@ "cookie-signature": ["cookie-signature@1.0.7", "", {}, "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA=="], + "create-require": ["create-require@1.1.1", "", {}, "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ=="], + "debug": ["debug@2.6.9", "", { "dependencies": { "ms": "2.0.0" } }, "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA=="], "deep-is": ["deep-is@0.1.4", "", {}, "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ=="], @@ -102,6 +155,8 @@ "destroy": ["destroy@1.2.0", "", {}, "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg=="], + "diff": ["diff@4.0.4", "", {}, "sha512-X07nttJQkwkfKfvTPG/KSnE2OMdcUCao6+eXF3wmnIQRn2aPAHH3VxDbDOdegkd6JbPsXqShpvEOHfAT+nCNwQ=="], + "dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="], "ee-first": ["ee-first@1.1.1", "", {}, "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow=="], @@ -190,6 +245,8 @@ "long": ["long@5.3.2", "", {}, "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA=="], + "make-error": ["make-error@1.3.6", "", {}, "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw=="], + "markdown-it": ["markdown-it@14.1.0", "", { "dependencies": { "argparse": "^2.0.1", "entities": "^4.4.0", "linkify-it": "^5.0.0", "mdurl": "^2.0.0", "punycode.js": "^2.3.1", "uc.micro": "^2.1.0" }, "bin": "bin/markdown-it.mjs" }, "sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg=="], "markdown-it-anchor": ["markdown-it-anchor@8.6.7", "", { "peerDependencies": { "@types/markdown-it": "*", "markdown-it": "*" } }, "sha512-FlCHFwNnutLgVTflOYHPW2pPcl2AACqVzExlkGQNsi4CJgqOHN7YTgDd4LuhgN1BFO3TS0vLAruV1Td6dwWPJA=="], @@ -234,11 +291,35 @@ "path-to-regexp": ["path-to-regexp@0.1.12", "", {}, "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ=="], + "pg": ["pg@8.18.0", "", { "dependencies": { "pg-connection-string": "^2.11.0", "pg-pool": "^3.11.0", "pg-protocol": "^1.11.0", "pg-types": "2.2.0", "pgpass": "1.0.5" }, "optionalDependencies": { "pg-cloudflare": "^1.3.0" }, "peerDependencies": { "pg-native": ">=3.0.1" }, "optionalPeers": ["pg-native"] }, "sha512-xqrUDL1b9MbkydY/s+VZ6v+xiMUmOUk7SS9d/1kpyQxoJ6U9AO1oIJyUWVZojbfe5Cc/oluutcgFG4L9RDP1iQ=="], + + "pg-cloudflare": ["pg-cloudflare@1.3.0", "", {}, "sha512-6lswVVSztmHiRtD6I8hw4qP/nDm1EJbKMRhf3HCYaqud7frGysPv7FYJ5noZQdhQtN2xJnimfMtvQq21pdbzyQ=="], + + "pg-connection-string": ["pg-connection-string@2.11.0", "", {}, "sha512-kecgoJwhOpxYU21rZjULrmrBJ698U2RxXofKVzOn5UDj61BPj/qMb7diYUR1nLScCDbrztQFl1TaQZT0t1EtzQ=="], + + "pg-int8": ["pg-int8@1.0.1", "", {}, "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw=="], + + "pg-pool": ["pg-pool@3.11.0", "", { "peerDependencies": { "pg": ">=8.0" } }, "sha512-MJYfvHwtGp870aeusDh+hg9apvOe2zmpZJpyt+BMtzUWlVqbhFmMK6bOBXLBUPd7iRtIF9fZplDc7KrPN3PN7w=="], + + "pg-protocol": ["pg-protocol@1.11.0", "", {}, "sha512-pfsxk2M9M3BuGgDOfuy37VNRRX3jmKgMjcvAcWqNDpZSf4cUmv8HSOl5ViRQFsfARFn0KuUQTgLxVMbNq5NW3g=="], + + "pg-types": ["pg-types@2.2.0", "", { "dependencies": { "pg-int8": "1.0.1", "postgres-array": "~2.0.0", "postgres-bytea": "~1.0.0", "postgres-date": "~1.0.4", "postgres-interval": "^1.1.0" } }, "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA=="], + + "pgpass": ["pgpass@1.0.5", "", { "dependencies": { "split2": "^4.1.0" } }, "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug=="], + + "postgres-array": ["postgres-array@2.0.0", "", {}, "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA=="], + + "postgres-bytea": ["postgres-bytea@1.0.1", "", {}, "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ=="], + + "postgres-date": ["postgres-date@1.0.7", "", {}, "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q=="], + + "postgres-interval": ["postgres-interval@1.2.0", "", { "dependencies": { "xtend": "^4.0.0" } }, "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ=="], + "prelude-ls": ["prelude-ls@1.1.2", "", {}, "sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w=="], "protobufjs": ["protobufjs@7.5.4", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", "@protobufjs/codegen": "^2.0.4", "@protobufjs/eventemitter": "^1.1.0", "@protobufjs/fetch": "^1.1.0", "@protobufjs/float": "^1.0.2", "@protobufjs/inquire": "^1.1.0", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.0", "@types/node": ">=13.7.0", "long": "^5.0.0" } }, "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg=="], - "protobufjs-cli": ["protobufjs-cli@1.2.0", "", { "dependencies": { "chalk": "^4.0.0", "escodegen": "^1.13.0", "espree": "^9.0.0", "estraverse": "^5.1.0", "glob": "^8.0.0", "jsdoc": "^4.0.0", "minimist": "^1.2.0", "semver": "^7.1.2", "tmp": "^0.2.1", "uglify-js": "^3.7.7" }, "peerDependencies": { "protobufjs": "^7.0.0" }, "bin": { "pbjs": "bin/pbjs", "pbts": "bin/pbts" } }, "sha512-+YvqJEmsmZHGzE5j0tvEzFeHm0sX7pzRFpyj7+GazhkS4Y0r+jgbioVvFxxSWIlPzUel/lxeOnLChBmV8NmyHA=="], + "protobufjs-cli": ["protobufjs-cli@2.0.0", "", { "dependencies": { "chalk": "^4.0.0", "escodegen": "^1.13.0", "espree": "^9.0.0", "estraverse": "^5.1.0", "glob": "^8.0.0", "jsdoc": "^4.0.0", "minimist": "^1.2.0", "semver": "^7.1.2", "tmp": "^0.2.1", "uglify-js": "^3.7.7" }, "peerDependencies": { "protobufjs": "^7.0.0" }, "bin": { "pbjs": "bin/pbjs", "pbts": "bin/pbts" } }, "sha512-/QGByFXSLY9Was5Oq8yKn5lu1pMywGMHgE0fDR9b4nZFTvaWJonHiJxT3HHu2ZuOQAxd7ufYyQtW8Q1ZgW+gPg=="], "proxy-addr": ["proxy-addr@2.0.7", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg=="], @@ -274,6 +355,8 @@ "source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="], + "split2": ["split2@4.2.0", "", {}, "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg=="], + "statuses": ["statuses@2.0.2", "", {}, "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw=="], "strip-json-comments": ["strip-json-comments@3.1.1", "", {}, "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig=="], @@ -284,10 +367,14 @@ "toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="], + "ts-node": ["ts-node@10.9.2", "", { "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", "@tsconfig/node12": "^1.0.7", "@tsconfig/node14": "^1.0.0", "@tsconfig/node16": "^1.0.2", "acorn": "^8.4.1", "acorn-walk": "^8.1.1", "arg": "^4.1.0", "create-require": "^1.1.0", "diff": "^4.0.1", "make-error": "^1.1.1", "v8-compile-cache-lib": "^3.0.1", "yn": "3.1.1" }, "peerDependencies": { "@swc/core": ">=1.2.50", "@swc/wasm": ">=1.2.50", "@types/node": "*", "typescript": ">=2.7" }, "optionalPeers": ["@swc/core", "@swc/wasm"], "bin": { "ts-node": "dist/bin.js", "ts-script": "dist/bin-script-deprecated.js", "ts-node-cwd": "dist/bin-cwd.js", "ts-node-esm": "dist/bin-esm.js", "ts-node-script": "dist/bin-script.js", "ts-node-transpile-only": "dist/bin-transpile.js" } }, "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ=="], + "type-check": ["type-check@0.3.2", "", { "dependencies": { "prelude-ls": "~1.1.2" } }, "sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg=="], "type-is": ["type-is@1.6.18", "", { "dependencies": { "media-typer": "0.3.0", "mime-types": "~2.1.24" } }, "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g=="], + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + "uc.micro": ["uc.micro@2.1.0", "", {}, "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A=="], "uglify-js": ["uglify-js@3.19.3", "", { "bin": { "uglifyjs": "bin/uglifyjs" } }, "sha512-v3Xu+yuwBXisp6QYTcH4UbH+xYJXqnq2m/LtQVWKWzYc1iehYnLixoQDN9FH6/j9/oybfd6W9Ghwkl8+UMKTKQ=="], @@ -300,6 +387,8 @@ "utils-merge": ["utils-merge@1.0.1", "", {}, "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA=="], + "v8-compile-cache-lib": ["v8-compile-cache-lib@3.0.1", "", {}, "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg=="], + "vary": ["vary@1.1.2", "", {}, "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg=="], "word-wrap": ["word-wrap@1.2.5", "", {}, "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA=="], @@ -308,8 +397,16 @@ "xmlcreate": ["xmlcreate@2.0.4", "", {}, "sha512-nquOebG4sngPmGPICTS5EnxqhKbCmz5Ox5hsszI2T6U5qdrJizBc+0ilYSEjTSzU0yZcmvppztXe/5Al5fUwdg=="], + "xtend": ["xtend@4.0.2", "", {}, "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="], + + "yn": ["yn@3.1.1", "", {}, "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q=="], + + "@types/serve-static/@types/send": ["@types/send@0.17.6", "", { "dependencies": { "@types/mime": "^1", "@types/node": "*" } }, "sha512-Uqt8rPBE8SY0RK8JB1EzVOIZ32uqy8HwdxCnoCOsYrvnswqmFZ/k+9Ikidlk/ImhsdvBsloHbAlewb2IEBV/Og=="], + "escodegen/estraverse": ["estraverse@4.3.0", "", {}, "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw=="], + "gtfs-realtime-bindings/protobufjs-cli": ["protobufjs-cli@1.2.0", "", { "dependencies": { "chalk": "^4.0.0", "escodegen": "^1.13.0", "espree": "^9.0.0", "estraverse": "^5.1.0", "glob": "^8.0.0", "jsdoc": "^4.0.0", "minimist": "^1.2.0", "semver": "^7.1.2", "tmp": "^0.2.1", "uglify-js": "^3.7.7" }, "peerDependencies": { "protobufjs": "^7.0.0" }, "bin": { "pbjs": "bin/pbjs", "pbts": "bin/pbts" } }, "sha512-+YvqJEmsmZHGzE5j0tvEzFeHm0sX7pzRFpyj7+GazhkS4Y0r+jgbioVvFxxSWIlPzUel/lxeOnLChBmV8NmyHA=="], + "send/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], } } diff --git a/infrastructure/compose.yml b/infrastructure/compose.yml new file mode 100644 index 0000000..08b6a79 --- /dev/null +++ b/infrastructure/compose.yml @@ -0,0 +1,24 @@ +services: + timescaledb: + image: timescale/timescaledb:latest-pg17 + environment: + POSTGRES_USER: ${POSTGRES_USER:-postgres} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-example} + POSTGRES_DB: ${POSTGRES_DB:-iot_data} + ports: + - "5432:5432" + volumes: + - timescaledb-data:/var/lib/postgresql/data + restart: unless-stopped + + redis: + image: redis:latest + ports: + - "6379:6379" + restart: unless-stopped + volumes: + - redis-data:/data + +volumes: + timescaledb-data: + redis-data: \ No newline at end of file diff --git a/lib/database.ts b/lib/database.ts new file mode 100644 index 0000000..2d4d07a --- /dev/null +++ b/lib/database.ts @@ -0,0 +1,464 @@ +import { Pool, PoolClient, QueryResult } from 'pg'; + +// PostgreSQL connection pool +const pool = new Pool({ + host: process.env.POSTGRES_HOST || 'localhost', + port: parseInt(process.env.POSTGRES_PORT || '5432'), + database: process.env.POSTGRES_DB || 'iot_data', + user: process.env.POSTGRES_USER || 'postgres', + password: process.env.POSTGRES_PASSWORD || 'example', + max: 20, + idleTimeoutMillis: 30000, + connectionTimeoutMillis: 2000, +}); + +// Test connection +pool.on('error', (err) => { + console.error('Unexpected error on idle client', err); +}); + +// Create tables with TimescaleDB hypertables +export async function initDatabase() { + const client = await pool.connect(); + try { + // Enable TimescaleDB extension + await client.query('CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE'); + + // Vehicle positions time-series + await client.query(` + CREATE TABLE IF NOT EXISTS vehicle_positions ( + time TIMESTAMPTZ NOT NULL, + vehicle_id TEXT NOT NULL, + route_id TEXT NOT NULL, + trip_id TEXT, + latitude DOUBLE PRECISION NOT NULL, + longitude DOUBLE PRECISION NOT NULL, + speed DOUBLE PRECISION, + bearing DOUBLE PRECISION, + current_status TEXT, + PRIMARY KEY (time, vehicle_id) + ) + `); + + // Convert to hypertable if not already + try { + await client.query(` + SELECT create_hypertable('vehicle_positions', 'time', + if_not_exists => TRUE, + chunk_time_interval => INTERVAL '1 day' + ) + `); + } catch (e: any) { + if (!e.message?.includes('already a hypertable')) { + console.warn('Note: vehicle_positions hypertable setup:', e.message); + } + } + + // Create indexes + await client.query(` + CREATE INDEX IF NOT EXISTS idx_vehicle_positions_vehicle + ON vehicle_positions(vehicle_id, time DESC) + `); + await client.query(` + CREATE INDEX IF NOT EXISTS idx_vehicle_positions_route + ON vehicle_positions(route_id, time DESC) + `); + + // Arrival records time-series + await client.query(` + CREATE TABLE IF NOT EXISTS arrival_records ( + time TIMESTAMPTZ NOT NULL, + stop_id TEXT NOT NULL, + route_id TEXT NOT NULL, + trip_id TEXT, + scheduled_time TIMESTAMPTZ NOT NULL, + predicted_time TIMESTAMPTZ, + actual_time TIMESTAMPTZ, + delay_seconds INTEGER, + is_realtime BOOLEAN, + headsign TEXT, + PRIMARY KEY (time, stop_id, route_id, scheduled_time) + ) + `); + + // Convert to hypertable + try { + await client.query(` + SELECT create_hypertable('arrival_records', 'time', + if_not_exists => TRUE, + chunk_time_interval => INTERVAL '1 day' + ) + `); + } catch (e: any) { + if (!e.message?.includes('already a hypertable')) { + console.warn('Note: arrival_records hypertable setup:', e.message); + } + } + + // Create indexes + await client.query(` + CREATE INDEX IF NOT EXISTS idx_arrival_records_stop + ON arrival_records(stop_id, time DESC) + `); + await client.query(` + CREATE INDEX IF NOT EXISTS idx_arrival_records_route + ON arrival_records(route_id, time DESC) + `); + + // Service metrics (aggregated hourly) + await client.query(` + CREATE TABLE IF NOT EXISTS service_metrics ( + hour_timestamp TIMESTAMPTZ NOT NULL, + route_id TEXT NOT NULL, + stop_id TEXT NOT NULL DEFAULT '', + total_arrivals INTEGER DEFAULT 0, + on_time_arrivals INTEGER DEFAULT 0, + late_arrivals INTEGER DEFAULT 0, + avg_delay_seconds DOUBLE PRECISION DEFAULT 0, + max_delay_seconds INTEGER DEFAULT 0, + active_vehicles INTEGER DEFAULT 0, + PRIMARY KEY (hour_timestamp, route_id, stop_id) + ) + `); + + // Route performance summary + await client.query(` + CREATE TABLE IF NOT EXISTS route_stats ( + route_id TEXT PRIMARY KEY, + route_name TEXT, + last_updated TIMESTAMPTZ, + total_trips INTEGER DEFAULT 0, + avg_delay_seconds DOUBLE PRECISION DEFAULT 0, + reliability_score DOUBLE PRECISION DEFAULT 0 + ) + `); + + // Create continuous aggregates for better performance + try { + await client.query(` + CREATE MATERIALIZED VIEW IF NOT EXISTS hourly_route_delays + WITH (timescaledb.continuous) AS + SELECT + time_bucket('1 hour', time) AS bucket, + route_id, + COUNT(*) as arrival_count, + AVG(delay_seconds) as avg_delay, + MAX(delay_seconds) as max_delay, + COUNT(*) FILTER (WHERE ABS(delay_seconds) <= 120) as on_time_count + FROM arrival_records + WHERE delay_seconds IS NOT NULL + GROUP BY bucket, route_id + WITH NO DATA + `); + + // Refresh policy + await client.query(` + SELECT add_continuous_aggregate_policy('hourly_route_delays', + start_offset => INTERVAL '3 hours', + end_offset => INTERVAL '1 hour', + schedule_interval => INTERVAL '1 hour', + if_not_exists => TRUE + ) + `); + } catch (e: any) { + if (!e.message?.includes('already exists')) { + console.warn('Note: continuous aggregate setup:', e.message); + } + } + + // Set up retention policy (keep 90 days) + try { + await client.query(` + SELECT add_retention_policy('vehicle_positions', + INTERVAL '90 days', + if_not_exists => TRUE + ) + `); + await client.query(` + SELECT add_retention_policy('arrival_records', + INTERVAL '90 days', + if_not_exists => TRUE + ) + `); + } catch (e: any) { + console.warn('Note: retention policy setup:', e.message); + } + + console.log('[OK] TimescaleDB initialized successfully'); + } finally { + client.release(); + } +} + +// Insert vehicle position +export interface VehiclePosition { + timestamp: number; + vehicleId: string; + routeId: string; + tripId?: string; + latitude: number; + longitude: number; + speed?: number; + bearing?: number; + currentStatus?: string; +} + +export async function logVehiclePosition(position: VehiclePosition) { + try { + await pool.query(` + INSERT INTO vehicle_positions + (time, vehicle_id, route_id, trip_id, latitude, longitude, speed, bearing, current_status) + VALUES (to_timestamp($1::double precision / 1000), $2, $3, $4, $5, $6, $7, $8, $9) + ON CONFLICT (time, vehicle_id) DO NOTHING + `, [ + position.timestamp, + position.vehicleId, + position.routeId, + position.tripId || null, + position.latitude, + position.longitude, + position.speed || null, + position.bearing || null, + position.currentStatus || null + ]); + } catch (error) { + console.error('Failed to log vehicle position:', error); + } +} + +// Batch insert vehicle positions +export async function logVehiclePositions(positions: VehiclePosition[]) { + if (positions.length === 0) return; + + const client = await pool.connect(); + try { + await client.query('BEGIN'); + + for (const pos of positions) { + await client.query(` + INSERT INTO vehicle_positions + (time, vehicle_id, route_id, trip_id, latitude, longitude, speed, bearing, current_status) + VALUES (to_timestamp($1::double precision / 1000), $2, $3, $4, $5, $6, $7, $8, $9) + ON CONFLICT (time, vehicle_id) DO NOTHING + `, [ + pos.timestamp, + pos.vehicleId, + pos.routeId, + pos.tripId || null, + pos.latitude, + pos.longitude, + pos.speed || null, + pos.bearing || null, + pos.currentStatus || null + ]); + } + + await client.query('COMMIT'); + } catch (error) { + await client.query('ROLLBACK'); + console.error('Failed to batch log vehicle positions:', error); + } finally { + client.release(); + } +} + +// Insert arrival record +export interface ArrivalRecord { + timestamp: number; + stopId: string; + routeId: string; + tripId?: string; + scheduledTime: number; + predictedTime?: number; + actualTime?: number; + delaySeconds?: number; + isRealtime: boolean; + headsign?: string; +} + +export async function logArrival(arrival: ArrivalRecord) { + try { + await pool.query(` + INSERT INTO arrival_records + (time, stop_id, route_id, trip_id, scheduled_time, predicted_time, actual_time, delay_seconds, is_realtime, headsign) + VALUES ( + to_timestamp($1::double precision / 1000), + $2, $3, $4, + to_timestamp($5::double precision / 1000), + to_timestamp($6::double precision / 1000), + to_timestamp($7::double precision / 1000), + $8, $9, $10 + ) + ON CONFLICT (time, stop_id, route_id, scheduled_time) DO UPDATE SET + predicted_time = EXCLUDED.predicted_time, + delay_seconds = EXCLUDED.delay_seconds, + is_realtime = EXCLUDED.is_realtime + `, [ + arrival.timestamp, + arrival.stopId, + arrival.routeId, + arrival.tripId || null, + arrival.scheduledTime, + arrival.predictedTime || null, + arrival.actualTime || null, + arrival.delaySeconds || null, + arrival.isRealtime, + arrival.headsign || null + ]); + } catch (error) { + console.error('Failed to log arrival:', error); + } +} + +// Query historical vehicle positions +export async function getVehicleHistory(vehicleId: string, startTime: number, endTime: number) { + const result = await pool.query(` + SELECT + EXTRACT(EPOCH FROM time) * 1000 as timestamp, + latitude, longitude, speed, bearing, current_status + FROM vehicle_positions + WHERE vehicle_id = $1 + AND time BETWEEN to_timestamp($2::double precision / 1000) AND to_timestamp($3::double precision / 1000) + ORDER BY time ASC + `, [vehicleId, startTime, endTime]); + + return result.rows; +} + +// Query route vehicle positions in time range +export async function getRouteVehiclePositions(routeId: string, startTime: number, endTime: number) { + const result = await pool.query(` + SELECT + EXTRACT(EPOCH FROM time) * 1000 as timestamp, + vehicle_id, latitude, longitude, speed, bearing, current_status + FROM vehicle_positions + WHERE route_id = $1 + AND time BETWEEN to_timestamp($2::double precision / 1000) AND to_timestamp($3::double precision / 1000) + ORDER BY time ASC + `, [routeId, startTime, endTime]); + + return result.rows; +} + +// Query arrival history for a stop +export async function getStopArrivalHistory(stopId: string, routeId: string, startTime: number, endTime: number) { + const result = await pool.query(` + SELECT + EXTRACT(EPOCH FROM time) * 1000 as timestamp, + EXTRACT(EPOCH FROM scheduled_time) * 1000 as scheduled_time, + EXTRACT(EPOCH FROM predicted_time) * 1000 as predicted_time, + EXTRACT(EPOCH FROM actual_time) * 1000 as actual_time, + delay_seconds, is_realtime, headsign + FROM arrival_records + WHERE stop_id = $1 AND route_id = $2 + AND time BETWEEN to_timestamp($3::double precision / 1000) AND to_timestamp($4::double precision / 1000) + ORDER BY time ASC + `, [stopId, routeId, startTime, endTime]); + + return result.rows; +} + +// Get route delay statistics +export async function getRouteDelayStats(routeId: string, hours: number = 24) { + const result = await pool.query(` + SELECT + COUNT(*) as total_arrivals, + AVG(delay_seconds) as avg_delay, + MIN(delay_seconds) as min_delay, + MAX(delay_seconds) as max_delay, + COUNT(*) FILTER (WHERE ABS(delay_seconds) <= 120) as on_time_count, + COUNT(*) FILTER (WHERE delay_seconds > 120) as late_count, + COUNT(*) FILTER (WHERE delay_seconds < -120) as early_count + FROM arrival_records + WHERE route_id = $1 + AND time >= NOW() - INTERVAL '1 hour' * $2 + `, [routeId, hours]); + + return result.rows[0]; +} + +// Get stop delay statistics +export async function getStopDelayStats(stopId: string, hours: number = 24) { + const result = await pool.query(` + SELECT + route_id, + COUNT(*) as total_arrivals, + AVG(delay_seconds) as avg_delay, + COUNT(*) FILTER (WHERE ABS(delay_seconds) <= 120) as on_time_count + FROM arrival_records + WHERE stop_id = $1 + AND time >= NOW() - INTERVAL '1 hour' * $2 + AND delay_seconds IS NOT NULL + GROUP BY route_id + `, [stopId, hours]); + + return result.rows; +} + +// Get hourly pattern for a route +export async function getRouteHourlyPattern(routeId: string, days: number = 7) { + const result = await pool.query(` + SELECT + EXTRACT(HOUR FROM ar.time) as hour, + COUNT(*) as arrival_count, + AVG(ar.delay_seconds) as avg_delay, + COUNT(DISTINCT vp.vehicle_id) as unique_vehicles + FROM arrival_records ar + LEFT JOIN vehicle_positions vp ON ar.trip_id = vp.trip_id + AND ABS(EXTRACT(EPOCH FROM (ar.time - vp.time))) < 60 + WHERE ar.route_id = $1 + AND ar.time >= NOW() - INTERVAL '1 day' * $2 + GROUP BY hour + ORDER BY hour + `, [routeId, days]); + + return result.rows; +} + +// Cleanup old data (already handled by retention policy, but keep for manual cleanup) +export async function cleanupOldData(daysToKeep: number = 90) { + const client = await pool.connect(); + try { + const result1 = await client.query(` + DELETE FROM vehicle_positions + WHERE time < NOW() - INTERVAL '1 day' * $1 + `, [daysToKeep]); + + const result2 = await client.query(` + DELETE FROM arrival_records + WHERE time < NOW() - INTERVAL '1 day' * $1 + `, [daysToKeep]); + + console.log(`Cleaned up old data: ${result1.rowCount} positions, ${result2.rowCount} arrivals`); + } finally { + client.release(); + } +} + +// Get database statistics +export async function getDatabaseStats() { + const client = await pool.connect(); + try { + const positionCount = await client.query('SELECT COUNT(*) as count FROM vehicle_positions'); + const arrivalCount = await client.query('SELECT COUNT(*) as count FROM arrival_records'); + const oldestPosition = await client.query('SELECT MIN(time) as oldest FROM vehicle_positions'); + const newestPosition = await client.query('SELECT MAX(time) as newest FROM vehicle_positions'); + + return { + vehiclePositions: parseInt(positionCount.rows[0].count), + arrivalRecords: parseInt(arrivalCount.rows[0].count), + oldestRecord: oldestPosition.rows[0].oldest, + newestRecord: newestPosition.rows[0].newest, + dbType: 'TimescaleDB', + host: pool.options.host, + database: pool.options.database + }; + } finally { + client.release(); + } +} + +// Close database +export async function closeDatabase() { + await pool.end(); +} diff --git a/package.json b/package.json index 0a234e4..118f9b8 100644 --- a/package.json +++ b/package.json @@ -4,12 +4,8 @@ "description": "Real-time bus tracking for Skopje public transport", "main": "server.ts", "scripts": { - "setup-gtfs": "npx ts-node setup-gtfs.ts", - "find": "npx ts-node find-stops-routes.ts", - "web": "npx ts-node server.ts", - "tracker": "npx ts-node bus-tracker-json.ts", - "build": "tsc", - "start": "npm run web" + "start": "bun run server.ts", + "track": "bun run background-tracker.ts" }, "keywords": [ "skopje", @@ -24,6 +20,7 @@ "license": "MIT", "dependencies": { "express": "^4.18.2", + "pg": "^8.18.0", "protobufjs": "^7.5.4" }, "optionalDependencies": { @@ -32,6 +29,7 @@ "devDependencies": { "@types/express": "^4.17.21", "@types/node": "^20.10.0", + "@types/pg": "^8.11.0", "protobufjs-cli": "^2.0.0", "ts-node": "^10.9.1", "typescript": "^5.9.3" diff --git a/public/analytics.html b/public/analytics.html new file mode 100644 index 0000000..5a0ebf5 --- /dev/null +++ b/public/analytics.html @@ -0,0 +1,500 @@ + + + + + + Bus Analytics - Skopje + + + + +
+

Bus Analytics Dashboard

+
Historical performance and delay analysis
+
+ +
+ ← Back to Live Tracker + Refresh Data +
+ +
+
+ + +
+ +
+ +
+ + +
+ +
+ +
+
+

Delay Distribution

+ +
+ +
+

Hourly Service Pattern (Last 7 Days)

+ +
+ +
+

On-Time Performance

+ +
+
+
+ + + + diff --git a/public/index.html b/public/index.html index 5f6f8f6..306fcaa 100644 --- a/public/index.html +++ b/public/index.html @@ -265,12 +265,31 @@ color: #999; margin-top: 5px; } + + .nav-link { + padding: 8px 20px; + background: rgba(255,255,255,0.2); + border-radius: 4px; + text-decoration: none; + color: white; + font-size: 14px; + white-space: nowrap; + } + + .nav-link:hover { + background: rgba(255,255,255,0.3); + }
-

Skopje Bus Tracker

-
Select stop and route to track
+
+
+

Skopje Bus Tracker

+
Select stop and route to track
+
+ View Analytics +
diff --git a/server.ts b/server.ts index 4f0927a..3e8143c 100644 --- a/server.ts +++ b/server.ts @@ -2,9 +2,32 @@ import express, { Request, Response } from 'express'; import GtfsRealtimeBindings from 'gtfs-realtime-bindings'; import { loadGtfsStops, loadGtfsRoutes } from './lib/gtfs'; import { config, StopConfig, RouteConfig } from './config'; +import { + initDatabase, + logVehiclePosition, + logVehiclePositions, + logArrival, + getDatabaseStats, + cleanupOldData, + getVehicleHistory, + getRouteVehiclePositions, + getStopArrivalHistory, + getRouteDelayStats, + getStopDelayStats, + getRouteHourlyPattern +} from './lib/database'; const app = express(); +// Initialize database +try { + initDatabase(); + console.log('Database initialized successfully'); +} catch (error) { + console.error('Failed to initialize database:', error); + console.log('Continuing without data logging...'); +} + // Load GTFS data const stops = loadGtfsStops(); const routes = loadGtfsRoutes(); @@ -13,6 +36,11 @@ const routes = loadGtfsRoutes(); import * as path from 'path'; app.use(express.static(path.join(__dirname, 'public'))); +// Serve OpenAPI specification +app.get('/openapi.yaml', (req: Request, res: Response) => { + res.sendFile(path.join(__dirname, 'openapi.yaml')); +}); + // API Endpoints app.get('/api/config', (req: Request, res: Response) => { const stopId = (req.query.stopId as string) || config.defaultStop.stopId; @@ -105,6 +133,22 @@ app.get('/api/arrivals', async (req: Request, res: Response) => { isRealtime: stopTime.realtime, realtimeState: stopTime.realtimeState, }); + + // Log arrival to database (use scheduled time as primary timestamp to avoid duplicates) + try { + logArrival({ + timestamp: scheduledTime.getTime(), + stopId, + routeId, + scheduledTime: scheduledTime.getTime(), + predictedTime: arrivalTime.getTime(), + delaySeconds: stopTime.arrivalDelay, + isRealtime: stopTime.realtime, + headsign: stopTime.headsign, + }); + } catch (dbError) { + // Fail silently if database is not available + } } } } @@ -232,6 +276,24 @@ app.get('/api/vehicles', async (req: Request, res: Response) => { console.log(`Found ${routeVehicleIds.size} vehicle IDs for route ${routeId}`); console.log(`Matched ${activeVehicles.length} vehicles from JSON API`); + // Log vehicle positions to database + try { + const positions = activeVehicles.map(v => ({ + timestamp: Date.now(), + vehicleId: v.vehicleId || v.label, + routeId, + tripId: v.tripId, + latitude: v.lat, + longitude: v.lon, + speed: v.speed, + bearing: v.bearing, + currentStatus: v.currentStatus, + })); + logVehiclePositions(positions); + } catch (dbError) { + // Fail silently if database is not available + } + res.json(activeVehicles); } catch (error) { console.error('Error fetching vehicles:', error); @@ -239,6 +301,96 @@ app.get('/api/vehicles', async (req: Request, res: Response) => { } }); +// Historical Data API Endpoints + +app.get('/api/stats/db', async (req: Request, res: Response) => { + try { + const stats = await getDatabaseStats(); + res.json(stats); + } catch (error) { + res.status(500).json({ error: 'Database not available' }); + } +}); + +app.get('/api/history/vehicle/:vehicleId', async (req: Request, res: Response) => { + try { + const vehicleId = req.params.vehicleId; + const hours = parseInt(req.query.hours as string) || 24; + const endTime = Date.now(); + const startTime = endTime - (hours * 60 * 60 * 1000); + + const history = await getVehicleHistory(vehicleId, startTime, endTime); + res.json(history); + } catch (error) { + res.status(500).json({ error: 'Failed to fetch vehicle history' }); + } +}); + +app.get('/api/history/route/:routeId/vehicles', async (req: Request, res: Response) => { + try { + const routeId = req.params.routeId; + const hours = parseInt(req.query.hours as string) || 24; + const endTime = Date.now(); + const startTime = endTime - (hours * 60 * 60 * 1000); + + const positions = await getRouteVehiclePositions(routeId, startTime, endTime); + res.json(positions); + } catch (error) { + res.status(500).json({ error: 'Failed to fetch route vehicle history' }); + } +}); + +app.get('/api/history/stop/:stopId/arrivals', async (req: Request, res: Response) => { + try { + const stopId = req.params.stopId; + const routeId = (req.query.routeId as string) || ''; + const hours = parseInt(req.query.hours as string) || 24; + const endTime = Date.now(); + const startTime = endTime - (hours * 60 * 60 * 1000); + + const arrivals = await getStopArrivalHistory(stopId, routeId, startTime, endTime); + res.json(arrivals); + } catch (error) { + res.status(500).json({ error: 'Failed to fetch arrival history' }); + } +}); + +app.get('/api/stats/route/:routeId/delays', async (req: Request, res: Response) => { + try { + const routeId = req.params.routeId; + const hours = parseInt(req.query.hours as string) || 24; + + const stats = await getRouteDelayStats(routeId, hours); + res.json(stats); + } catch (error) { + res.status(500).json({ error: 'Failed to fetch delay stats' }); + } +}); + +app.get('/api/stats/stop/:stopId/delays', async (req: Request, res: Response) => { + try { + const stopId = req.params.stopId; + const hours = parseInt(req.query.hours as string) || 24; + + const stats = await getStopDelayStats(stopId, hours); + res.json(stats); + } catch (error) { + res.status(500).json({ error: 'Failed to fetch delay stats' }); + } +}); + +app.get('/api/stats/route/:routeId/hourly', async (req: Request, res: Response) => { + try { + const routeId = req.params.routeId; + const days = parseInt(req.query.days as string) || 7; + + const pattern = await getRouteHourlyPattern(routeId, days); + res.json(pattern); + } catch (error) { + res.status(500).json({ error: 'Failed to fetch hourly pattern' }); + } +}); + // Start server app.listen(config.server.port, () => { console.log(` @@ -247,12 +399,16 @@ app.listen(config.server.port, () => { =========================================================================== Server running at: http://localhost:${config.server.port} - Open your browser and navigate to the URL above to view the map. + Pages: + - Main Tracker: http://localhost:${config.server.port} + - Analytics: http://localhost:${config.server.port}/analytics.html + - API Docs: http://localhost:${config.server.port}/api-docs.html Features: - - Real-time bus arrivals - - Live vehicle locations on map + - Real-time bus arrivals and vehicle tracking - Interactive map interface + - Historical analytics and delay statistics + - Complete REST API with OpenAPI specification Press Ctrl+C to stop the server ===========================================================================