fs/metrics/metrics.go

package metrics

import (
	"fmt"
	"os"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"syscall"
	"time"
)

var defaultBuckets = []float64{
	0.0005, 0.001, 0.0025, 0.005, 0.01,
	0.025, 0.05, 0.1, 0.25, 0.5,
	1, 2.5, 5, 10,
}

var lockBuckets = []float64{
	0.000001, 0.000005, 0.00001, 0.00005,
	0.0001, 0.0005, 0.001, 0.005, 0.01,
	0.025, 0.05, 0.1, 0.25, 0.5, 1,
}

var batchBuckets = []float64{1, 2, 4, 8, 16, 32, 64, 100, 128, 256, 512, 1000, 5000}

var Default = NewRegistry()

type histogram struct {
	bounds []float64
	counts []uint64
	sum    float64
	count  uint64
}

func newHistogram(bounds []float64) *histogram {
	cloned := make([]float64, len(bounds))
	copy(cloned, bounds)
	return &histogram{
		bounds: cloned,
		counts: make([]uint64, len(bounds)+1),
	}
}

func (h *histogram) observe(v float64) {
	h.count++
	h.sum += v
	for i, bound := range h.bounds {
		if v <= bound {
			h.counts[i]++
			return
		}
	}
	h.counts[len(h.counts)-1]++
}

func (h *histogram) snapshot() (bounds []float64, counts []uint64, sum float64, count uint64) {
	bounds = make([]float64, len(h.bounds))
	copy(bounds, h.bounds)
	counts = make([]uint64, len(h.counts))
	copy(counts, h.counts)
	return bounds, counts, h.sum, h.count
}

type Registry struct {
	startedAt time.Time

	httpInFlight atomic.Int64

	connectionPoolActive atomic.Int64
	connectionPoolMax    atomic.Int64
	connectionPoolWaits  atomic.Uint64

	requestQueueLength atomic.Int64

	mu sync.Mutex

	httpRequestsRoute      map[string]uint64
	httpResponseBytesRoute map[string]uint64
	httpDurationRoute      map[string]*histogram

	httpRequestsOp map[string]uint64
	httpDurationOp map[string]*histogram
	httpInFlightOp map[string]int64

	authRequests map[string]uint64

	serviceOps      map[string]uint64
	serviceDuration map[string]*histogram

	dbTxTotal    map[string]uint64
	dbTxDuration map[string]*histogram

	blobOps      map[string]uint64
	blobBytes    map[string]uint64
	blobDuration map[string]*histogram

	lockWait map[string]*histogram
	lockHold map[string]*histogram

	cacheHits   map[string]uint64
	cacheMisses map[string]uint64

	batchSize *histogram

	retries map[string]uint64
	errors  map[string]uint64

	gcRuns          map[string]uint64
	gcDuration      *histogram
	gcDeletedChunks uint64
	gcDeleteErrors  uint64
	gcCleanedUpload uint64
}

func NewRegistry() *Registry {
	return &Registry{
		startedAt:              time.Now(),
		httpRequestsRoute:      make(map[string]uint64),
		httpResponseBytesRoute: make(map[string]uint64),
		httpDurationRoute:      make(map[string]*histogram),
		httpRequestsOp:         make(map[string]uint64),
		httpDurationOp:         make(map[string]*histogram),
		httpInFlightOp:         make(map[string]int64),
		authRequests:           make(map[string]uint64),
		serviceOps:             make(map[string]uint64),
		serviceDuration:        make(map[string]*histogram),
		dbTxTotal:              make(map[string]uint64),
		dbTxDuration:           make(map[string]*histogram),
		blobOps:                make(map[string]uint64),
		blobBytes:              make(map[string]uint64),
		blobDuration:           make(map[string]*histogram),
		lockWait:               make(map[string]*histogram),
		lockHold:               make(map[string]*histogram),
		cacheHits:              make(map[string]uint64),
		cacheMisses:            make(map[string]uint64),
		batchSize:              newHistogram(batchBuckets),
		retries:                make(map[string]uint64),
		errors:                 make(map[string]uint64),
		gcRuns:                 make(map[string]uint64),
		gcDuration:             newHistogram(defaultBuckets),
	}
}

func NormalizeHTTPOperation(method string, isDeletePost bool) string {
	switch strings.ToUpper(strings.TrimSpace(method)) {
	case "GET":
		return "get"
	case "PUT":
		return "put"
	case "DELETE":
		return "delete"
	case "HEAD":
		return "head"
	case "POST":
		if isDeletePost {
			return "delete"
		}
		return "put"
	default:
		return "other"
	}
}

func statusResult(status int) string {
	if status >= 200 && status < 400 {
		return "ok"
	}
	return "error"
}

func normalizeRoute(route string) string {
	route = strings.TrimSpace(route)
	if route == "" {
		return "/unknown"
	}
	return route
}

func normalizeOp(op string) string {
	op = strings.ToLower(strings.TrimSpace(op))
	if op == "" {
		return "other"
	}
	return op
}

func (r *Registry) IncHTTPInFlight() {
	r.httpInFlight.Add(1)
}

func (r *Registry) DecHTTPInFlight() {
	r.httpInFlight.Add(-1)
}

func (r *Registry) IncHTTPInFlightOp(op string) {
	r.httpInFlight.Add(1)
	op = normalizeOp(op)
	r.mu.Lock()
	r.httpInFlightOp[op]++
	r.mu.Unlock()
}

func (r *Registry) DecHTTPInFlightOp(op string) {
	r.httpInFlight.Add(-1)
	op = normalizeOp(op)
	r.mu.Lock()
	r.httpInFlightOp[op]--
	if r.httpInFlightOp[op] < 0 {
		r.httpInFlightOp[op] = 0
	}
	r.mu.Unlock()
}

func (r *Registry) ObserveHTTPRequest(method, route string, status int, d time.Duration, responseBytes int) {
	op := NormalizeHTTPOperation(method, false)
	r.ObserveHTTPRequestDetailed(method, route, op, status, d, responseBytes)
}

func (r *Registry) ObserveHTTPRequestDetailed(method, route, op string, status int, d time.Duration, responseBytes int) {
	route = normalizeRoute(route)
	op = normalizeOp(op)
	result := statusResult(status)

	routeKey := method + "|" + route + "|" + strconv.Itoa(status)
	routeDurKey := method + "|" + route
	opKey := op + "|" + result

	r.mu.Lock()
	r.httpRequestsRoute[routeKey]++
	if responseBytes > 0 {
		r.httpResponseBytesRoute[routeKey] += uint64(responseBytes)
	}
	hRoute := r.httpDurationRoute[routeDurKey]
	if hRoute == nil {
		hRoute = newHistogram(defaultBuckets)
		r.httpDurationRoute[routeDurKey] = hRoute
	}
	hRoute.observe(d.Seconds())

	r.httpRequestsOp[opKey]++
	hOp := r.httpDurationOp[opKey]
	if hOp == nil {
		hOp = newHistogram(defaultBuckets)
		r.httpDurationOp[opKey] = hOp
	}
	hOp.observe(d.Seconds())
	r.mu.Unlock()
}

func (r *Registry) ObserveAuth(result, authType, reason string) {
	authType = strings.TrimSpace(authType)
	if authType == "" {
		authType = "unknown"
	}
	reason = strings.TrimSpace(reason)
	if reason == "" {
		reason = "none"
	}
	key := result + "|" + authType + "|" + reason
	r.mu.Lock()
	r.authRequests[key]++
	r.mu.Unlock()
}

func (r *Registry) ObserveService(operation string, d time.Duration, ok bool) {
	result := "error"
	if ok {
		result = "ok"
	}
	key := operation + "|" + result
	r.mu.Lock()
	r.serviceOps[key]++
	h := r.serviceDuration[operation]
	if h == nil {
		h = newHistogram(defaultBuckets)
		r.serviceDuration[operation] = h
	}
	h.observe(d.Seconds())
	r.mu.Unlock()
}

func (r *Registry) ObserveMetadataTx(txType string, d time.Duration, ok bool) {
	result := "error"
	if ok {
		result = "ok"
	}
	key := txType + "|" + result
	r.mu.Lock()
	r.dbTxTotal[key]++
	h := r.dbTxDuration[txType]
	if h == nil {
		h = newHistogram(defaultBuckets)
		r.dbTxDuration[txType] = h
	}
	h.observe(d.Seconds())
	r.mu.Unlock()
}

func (r *Registry) ObserveBlob(operation string, d time.Duration, bytes int64, ok bool, backend ...string) {
	be := "disk"
	if len(backend) > 0 {
		candidate := strings.TrimSpace(backend[0])
		if candidate != "" {
			be = strings.ToLower(candidate)
		}
	}
	result := "error"
	if ok {
		result = "ok"
	}
	op := strings.ToLower(strings.TrimSpace(operation))
	if op == "" {
		op = "unknown"
	}

	histKey := op + "|" + be + "|" + result
	opsKey := histKey

	r.mu.Lock()
	r.blobOps[opsKey]++
	h := r.blobDuration[histKey]
	if h == nil {
		h = newHistogram(defaultBuckets)
		r.blobDuration[histKey] = h
	}
	h.observe(d.Seconds())

	if bytes > 0 {
		r.blobBytes[op] += uint64(bytes)
	}
	r.mu.Unlock()
}

func (r *Registry) SetConnectionPoolMax(max int) {
	if max < 0 {
		max = 0
	}
	r.connectionPoolMax.Store(int64(max))
}

func (r *Registry) IncConnectionPoolActive() {
	r.connectionPoolActive.Add(1)
}

func (r *Registry) DecConnectionPoolActive() {
	r.connectionPoolActive.Add(-1)
}

func (r *Registry) IncConnectionPoolWait() {
	r.connectionPoolWaits.Add(1)
}

func (r *Registry) IncRequestQueueLength() {
	r.requestQueueLength.Add(1)
}

func (r *Registry) DecRequestQueueLength() {
	r.requestQueueLength.Add(-1)
}

func (r *Registry) ObserveLockWait(lockName string, d time.Duration) {
	lockName = strings.TrimSpace(lockName)
	if lockName == "" {
		lockName = "unknown"
	}
	r.mu.Lock()
	h := r.lockWait[lockName]
	if h == nil {
		h = newHistogram(lockBuckets)
		r.lockWait[lockName] = h
	}
	h.observe(d.Seconds())
	r.mu.Unlock()
}

func (r *Registry) ObserveLockHold(lockName string, d time.Duration) {
	lockName = strings.TrimSpace(lockName)
	if lockName == "" {
		lockName = "unknown"
	}
	r.mu.Lock()
	h := r.lockHold[lockName]
	if h == nil {
		h = newHistogram(lockBuckets)
		r.lockHold[lockName] = h
	}
	h.observe(d.Seconds())
	r.mu.Unlock()
}

func (r *Registry) ObserveCacheHit(cache string) {
	cache = strings.TrimSpace(cache)
	if cache == "" {
		cache = "unknown"
	}
	r.mu.Lock()
	r.cacheHits[cache]++
	r.mu.Unlock()
}

func (r *Registry) ObserveCacheMiss(cache string) {
	cache = strings.TrimSpace(cache)
	if cache == "" {
		cache = "unknown"
	}
	r.mu.Lock()
	r.cacheMisses[cache]++
	r.mu.Unlock()
}

func (r *Registry) ObserveBatchSize(size int) {
	if size < 0 {
		size = 0
	}
	r.mu.Lock()
	r.batchSize.observe(float64(size))
	r.mu.Unlock()
}

func (r *Registry) ObserveRetry(op, reason string) {
	op = normalizeOp(op)
	reason = strings.TrimSpace(reason)
	if reason == "" {
		reason = "unknown"
	}
	key := op + "|" + reason
	r.mu.Lock()
	r.retries[key]++
	r.mu.Unlock()
}

func (r *Registry) ObserveError(op, reason string) {
	op = normalizeOp(op)
	reason = strings.TrimSpace(reason)
	if reason == "" {
		reason = "unknown"
	}
	key := op + "|" + reason
	r.mu.Lock()
	r.errors[key]++
	r.mu.Unlock()
}

func (r *Registry) ObserveGC(d time.Duration, deletedChunks, deleteErrors, cleanedUploads int, ok bool) {
	result := "error"
	if ok {
		result = "ok"
	}
	r.mu.Lock()
	r.gcRuns[result]++
	r.gcDuration.observe(d.Seconds())
	if deletedChunks > 0 {
		r.gcDeletedChunks += uint64(deletedChunks)
	}
	if deleteErrors > 0 {
		r.gcDeleteErrors += uint64(deleteErrors)
	}
	if cleanedUploads > 0 {
		r.gcCleanedUpload += uint64(cleanedUploads)
	}
	r.mu.Unlock()
}

func (r *Registry) RenderPrometheus() string {
	now := time.Now()
	var mem runtime.MemStats
	runtime.ReadMemStats(&mem)

	r.mu.Lock()
	httpReqRoute := copyCounterMap(r.httpRequestsRoute)
	httpRespRoute := copyCounterMap(r.httpResponseBytesRoute)
	httpDurRoute := copyHistogramMap(r.httpDurationRoute)
	httpReqOp := copyCounterMap(r.httpRequestsOp)
	httpDurOp := copyHistogramMap(r.httpDurationOp)
	httpInFlightOp := copyIntGaugeMap(r.httpInFlightOp)
	authReq := copyCounterMap(r.authRequests)
	serviceOps := copyCounterMap(r.serviceOps)
	serviceDur := copyHistogramMap(r.serviceDuration)
	dbTx := copyCounterMap(r.dbTxTotal)
	dbTxDur := copyHistogramMap(r.dbTxDuration)
	blobOps := copyCounterMap(r.blobOps)
	blobBytes := copyCounterMap(r.blobBytes)
	blobDur := copyHistogramMap(r.blobDuration)
	lockWait := copyHistogramMap(r.lockWait)
	lockHold := copyHistogramMap(r.lockHold)
	cacheHits := copyCounterMap(r.cacheHits)
	cacheMisses := copyCounterMap(r.cacheMisses)
	batchBounds, batchCounts, batchSum, batchCount := r.batchSize.snapshot()
	retries := copyCounterMap(r.retries)
	errorsTotal := copyCounterMap(r.errors)
	gcRuns := copyCounterMap(r.gcRuns)
	gcDurBounds, gcDurCounts, gcDurSum, gcDurCount := r.gcDuration.snapshot()
	gcDeletedChunks := r.gcDeletedChunks
	gcDeleteErrors := r.gcDeleteErrors
	gcCleanedUploads := r.gcCleanedUpload
	r.mu.Unlock()

	connectionActive := float64(r.connectionPoolActive.Load())
	connectionMax := float64(r.connectionPoolMax.Load())
	connectionWaits := r.connectionPoolWaits.Load()
	queueLength := float64(r.requestQueueLength.Load())

	resident, hasResident := readResidentMemoryBytes()
	cpuSeconds, hasCPU := readProcessCPUSeconds()

	var b strings.Builder

	httpInFlightOp["all"] = r.httpInFlight.Load()
	writeGaugeVecFromInt64(&b, "fs_http_inflight_requests", "Current in-flight HTTP requests by operation.", httpInFlightOp, "op")
	writeCounterVecKV(&b, "fs_http_requests_total", "Total HTTP requests by operation and result.", httpReqOp, []string{"op", "result"})
	writeHistogramVecKV(&b, "fs_http_request_duration_seconds", "HTTP request latency by operation and result.", httpDurOp, []string{"op", "result"})

	writeCounterVecKV(&b, "fs_http_requests_by_route_total", "Total HTTP requests by method/route/status.", httpReqRoute, []string{"method", "route", "status"})
	writeCounterVecKV(&b, "fs_http_response_bytes_total", "Total HTTP response bytes written.", httpRespRoute, []string{"method", "route", "status"})
	writeHistogramVecKV(&b, "fs_http_request_duration_by_route_seconds", "HTTP request latency by method/route.", httpDurRoute, []string{"method", "route"})

	writeCounterVecKV(&b, "fs_auth_requests_total", "Authentication attempts by result.", authReq, []string{"result", "auth_type", "reason"})

	writeCounterVecKV(&b, "fs_service_operations_total", "Service-level operation calls.", serviceOps, []string{"operation", "result"})
	writeHistogramVecKV(&b, "fs_service_operation_duration_seconds", "Service-level operation latency.", serviceDur, []string{"operation"})

	writeCounterVecKV(&b, "fs_metadata_tx_total", "Metadata transaction calls.", dbTx, []string{"type", "result"})
	writeHistogramVecKV(&b, "fs_metadata_tx_duration_seconds", "Metadata transaction latency.", dbTxDur, []string{"type"})

	writeHistogramVecKV(&b, "fs_blob_operation_duration_seconds", "Blob backend operation latency.", blobDur, []string{"op", "backend", "result"})
	writeCounterVecKV(&b, "fs_blob_operations_total", "Blob store operations.", blobOps, []string{"op", "backend", "result"})
	writeCounterVecKV(&b, "fs_blob_bytes_total", "Blob bytes processed by operation.", blobBytes, []string{"op"})

	writeGauge(&b, "fs_connection_pool_active", "Active pooled connections.", connectionActive)
	writeGauge(&b, "fs_connection_pool_max", "Maximum pooled connections.", connectionMax)
	writeCounter(&b, "fs_connection_pool_waits_total", "Number of waits due to pool saturation.", connectionWaits)

	writeGauge(&b, "fs_request_queue_length", "Requests waiting for an execution slot.", queueLength)

	writeHistogramVecKV(&b, "fs_lock_wait_seconds", "Time spent waiting for locks.", lockWait, []string{"lock_name"})
	writeHistogramVecKV(&b, "fs_lock_hold_seconds", "Time locks were held.", lockHold, []string{"lock_name"})

	writeCounterVecKV(&b, "fs_cache_hits_total", "Cache hits by cache name.", cacheHits, []string{"cache"})
	writeCounterVecKV(&b, "fs_cache_misses_total", "Cache misses by cache name.", cacheMisses, []string{"cache"})

	writeHistogram(&b, "fs_batch_size_histogram", "Observed batch sizes.", nil, batchBounds, batchCounts, batchSum, batchCount)

	writeCounterVecKV(&b, "fs_retries_total", "Retries by operation and reason.", retries, []string{"op", "reason"})
	writeCounterVecKV(&b, "fs_errors_total", "Errors by operation and reason.", errorsTotal, []string{"op", "reason"})

	writeCounterVecKV(&b, "fs_gc_runs_total", "Garbage collection runs.", gcRuns, []string{"result"})
	writeHistogram(&b, "fs_gc_duration_seconds", "Garbage collection runtime.", nil, gcDurBounds, gcDurCounts, gcDurSum, gcDurCount)
	writeCounter(&b, "fs_gc_deleted_chunks_total", "Deleted chunks during GC.", gcDeletedChunks)
	writeCounter(&b, "fs_gc_delete_errors_total", "Chunk delete errors during GC.", gcDeleteErrors)
	writeCounter(&b, "fs_gc_cleaned_uploads_total", "Cleaned multipart uploads during GC.", gcCleanedUploads)

	writeGauge(&b, "fs_uptime_seconds", "Process uptime in seconds.", now.Sub(r.startedAt).Seconds())
	writeGauge(&b, "fs_runtime_goroutines", "Number of goroutines.", float64(runtime.NumGoroutine()))
	writeGaugeVec(&b, "fs_runtime_memory_bytes", "Runtime memory in bytes.", map[string]float64{
		"alloc":      float64(mem.Alloc),
		"total":      float64(mem.TotalAlloc),
		"sys":        float64(mem.Sys),
		"heap_alloc": float64(mem.HeapAlloc),
		"heap_sys":   float64(mem.HeapSys),
		"stack_sys":  float64(mem.StackSys),
	}, "type")
	writeCounter(&b, "fs_runtime_gc_cycles_total", "Completed GC cycles.", uint64(mem.NumGC))
	writeCounterFloat(&b, "fs_runtime_gc_pause_seconds_total", "Total GC pause time in seconds.", float64(mem.PauseTotalNs)/1e9)

	if hasCPU {
		writeCounterFloat(&b, "process_cpu_seconds_total", "Total user and system CPU time spent in seconds.", cpuSeconds)
	}
	if hasResident {
		writeGauge(&b, "process_resident_memory_bytes", "Resident memory size in bytes.", resident)
	}

	return b.String()
}

type histogramSnapshot struct {
	bounds []float64
	counts []uint64
	sum    float64
	count  uint64
}

func copyCounterMap(src map[string]uint64) map[string]uint64 {
	out := make(map[string]uint64, len(src))
	for k, v := range src {
		out[k] = v
	}
	return out
}

func copyIntGaugeMap(src map[string]int64) map[string]int64 {
	out := make(map[string]int64, len(src))
	for k, v := range src {
		out[k] = v
	}
	return out
}

func copyHistogramMap(src map[string]*histogram) map[string]histogramSnapshot {
	out := make(map[string]histogramSnapshot, len(src))
	for k, h := range src {
		bounds, counts, sum, count := h.snapshot()
		out[k] = histogramSnapshot{bounds: bounds, counts: counts, sum: sum, count: count}
	}
	return out
}

func writeCounter(b *strings.Builder, name, help string, value uint64) {
	fmt.Fprintf(b, "# HELP %s %s\n", name, help)
	fmt.Fprintf(b, "# TYPE %s counter\n", name)
	fmt.Fprintf(b, "%s %d\n", name, value)
}

func writeCounterFloat(b *strings.Builder, name, help string, value float64) {
	fmt.Fprintf(b, "# HELP %s %s\n", name, help)
	fmt.Fprintf(b, "# TYPE %s counter\n", name)
	fmt.Fprintf(b, "%s %.9f\n", name, value)
}

func writeGauge(b *strings.Builder, name, help string, value float64) {
	fmt.Fprintf(b, "# HELP %s %s\n", name, help)
	fmt.Fprintf(b, "# TYPE %s gauge\n", name)
	fmt.Fprintf(b, "%s %.9f\n", name, value)
}

func writeGaugeVec(b *strings.Builder, name, help string, values map[string]float64, labelName string) {
	fmt.Fprintf(b, "# HELP %s %s\n", name, help)
	fmt.Fprintf(b, "# TYPE %s gauge\n", name)
	keys := make([]string, 0, len(values))
	for k := range values {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for _, key := range keys {
		fmt.Fprintf(b, "%s{%s=\"%s\"} %.9f\n", name, labelName, escapeLabelValue(key), values[key])
	}
}

func writeGaugeVecFromInt64(b *strings.Builder, name, help string, values map[string]int64, labelName string) {
	fmt.Fprintf(b, "# HELP %s %s\n", name, help)
	fmt.Fprintf(b, "# TYPE %s gauge\n", name)
	keys := make([]string, 0, len(values))
	for k := range values {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for _, key := range keys {
		fmt.Fprintf(b, "%s{%s=\"%s\"} %.9f\n", name, labelName, escapeLabelValue(key), float64(values[key]))
	}
}

func writeCounterVecKV(b *strings.Builder, name, help string, values map[string]uint64, labels []string) {
	fmt.Fprintf(b, "# HELP %s %s\n", name, help)
	fmt.Fprintf(b, "# TYPE %s counter\n", name)
	keys := make([]string, 0, len(values))
	for k := range values {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for _, key := range keys {
		parts := strings.Split(key, "|")
		fmt.Fprintf(b, "%s{%s} %d\n", name, formatLabels(labels, parts), values[key])
	}
}

func writeHistogramVecKV(b *strings.Builder, name, help string, values map[string]histogramSnapshot, labels []string) {
	fmt.Fprintf(b, "# HELP %s %s\n", name, help)
	fmt.Fprintf(b, "# TYPE %s histogram\n", name)
	keys := make([]string, 0, len(values))
	for k := range values {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for _, key := range keys {
		parts := strings.Split(key, "|")
		labelsMap := make(map[string]string, len(labels))
		for i, label := range labels {
			if i < len(parts) {
				labelsMap[label] = parts[i]
			} else {
				labelsMap[label] = ""
			}
		}
		writeHistogramWithLabelsMap(b, name, labelsMap, values[key])
	}
}

func writeHistogram(b *strings.Builder, name, help string, labels map[string]string, bounds []float64, counts []uint64, sum float64, count uint64) {
	fmt.Fprintf(b, "# HELP %s %s\n", name, help)
	fmt.Fprintf(b, "# TYPE %s histogram\n", name)
	writeHistogramWithLabelsMap(b, name, labels, histogramSnapshot{bounds: bounds, counts: counts, sum: sum, count: count})
}

func writeHistogramWithLabelsMap(b *strings.Builder, name string, labels map[string]string, s histogramSnapshot) {
	var cumulative uint64
	for i, bucketCount := range s.counts {
		cumulative += bucketCount
		bucketLabels := cloneLabels(labels)
		if i < len(s.bounds) {
			bucketLabels["le"] = trimFloat(s.bounds[i])
		} else {
			bucketLabels["le"] = "+Inf"
		}
		fmt.Fprintf(b, "%s_bucket{%s} %d\n", name, labelsToString(bucketLabels), cumulative)
	}
	labelsSuffix := formatLabelsSuffix(labels)
	fmt.Fprintf(b, "%s_sum%s %.9f\n", name, labelsSuffix, s.sum)
	fmt.Fprintf(b, "%s_count%s %d\n", name, labelsSuffix, s.count)
}

func formatLabelsSuffix(labels map[string]string) string {
	if len(labels) == 0 {
		return ""
	}
	return "{" + labelsToString(labels) + "}"
}

func formatLabels(keys, values []string) string {
	parts := make([]string, 0, len(keys))
	for i, key := range keys {
		value := ""
		if i < len(values) {
			value = values[i]
		}
		parts = append(parts, fmt.Sprintf("%s=\"%s\"", key, escapeLabelValue(value)))
	}
	return strings.Join(parts, ",")
}

func labelsToString(labels map[string]string) string {
	if len(labels) == 0 {
		return ""
	}
	keys := make([]string, 0, len(labels))
	for k := range labels {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	parts := make([]string, 0, len(keys))
	for _, key := range keys {
		parts = append(parts, fmt.Sprintf("%s=\"%s\"", key, escapeLabelValue(labels[key])))
	}
	return strings.Join(parts, ",")
}

func cloneLabels(in map[string]string) map[string]string {
	if len(in) == 0 {
		return map[string]string{}
	}
	out := make(map[string]string, len(in)+1)
	for k, v := range in {
		out[k] = v
	}
	return out
}

func trimFloat(v float64) string {
	return strconv.FormatFloat(v, 'f', -1, 64)
}

func escapeLabelValue(value string) string {
	value = strings.ReplaceAll(value, `\\`, `\\\\`)
	value = strings.ReplaceAll(value, "\n", `\\n`)
	value = strings.ReplaceAll(value, `"`, `\\"`)
	return value
}

func readResidentMemoryBytes() (float64, bool) {
	data, err := os.ReadFile("/proc/self/statm")
	if err != nil {
		return 0, false
	}
	fields := strings.Fields(string(data))
	if len(fields) < 2 {
		return 0, false
	}
	rssPages, err := strconv.ParseInt(fields[1], 10, 64)
	if err != nil || rssPages < 0 {
		return 0, false
	}
	return float64(rssPages * int64(os.Getpagesize())), true
}

func readProcessCPUSeconds() (float64, bool) {
	var usage syscall.Rusage
	if err := syscall.Getrusage(syscall.RUSAGE_SELF, &usage); err != nil {
		return 0, false
	}
	user := float64(usage.Utime.Sec) + float64(usage.Utime.Usec)/1e6
	sys := float64(usage.Stime.Sec) + float64(usage.Stime.Usec)/1e6
	return user + sys, true
}