From c989037160affa61dd612abd0348a26160cd09b8 Mon Sep 17 00:00:00 2001 From: Andrej Mickov Date: Sun, 22 Feb 2026 23:00:33 +0100 Subject: [PATCH] Finialized multipart upload and graceful shutdown. Added Dockerfile. --- .dockerignore | 3 +++ Dockerfile | 16 +++++++++++ README.md | 33 ++++++++++++++++++++++- api/api.go | 63 +++++++++++++++++++++++++++++++++++++++----- metadata/metadata.go | 4 +++ service/service.go | 4 +++ 6 files changed, 115 insertions(+), 8 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..ad0a5f5 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +*.md +.gocache/ +blobs/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..843bf24 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM golang:1.25-alpine AS build + +WORKDIR /app + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . +RUN CGO_ENABLED=0 GOOS=linux go build -o /app/fs . + +FROM scratch AS runner + +COPY --from=build /app/fs /app/fs + +WORKDIR /app +CMD ["/app/fs"] diff --git a/README.md b/README.md index cca2a29..6512bc1 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,34 @@ # fs -An experimental Object Storage written in Go that should be compatible with S3 \ No newline at end of file +An experimental Object Storage written in Go that should be partially compatible with S3 + +## Features + +- Bucket operations: +- `PUT /{bucket}` +- `HEAD /{bucket}` +- `DELETE /{bucket}` +- `GET /` (list buckets) +- Object operations: +- `PUT /{bucket}/{key}` +- `GET /{bucket}/{key}` +- `HEAD /{bucket}/{key}` +- `DELETE /{bucket}/{key}` +- `GET /{bucket}?list-type=2&prefix=...` (ListObjectsV2-style) +- Multipart upload: +- `POST /{bucket}/{key}?uploads` (initiate) +- `PUT /{bucket}/{key}?uploadId=...&partNumber=N` (upload part) +- `GET /{bucket}/{key}?uploadId=...` (list parts) +- `POST /{bucket}/{key}?uploadId=...` (complete) +- `DELETE /{bucket}/{key}?uploadId=...` (abort) +- Multi-object delete: +- `POST /{bucket}?delete` with S3-style XML body +- AWS SigV4 streaming payload decoding for uploads (`aws-chunked` request bodies) + +## Limitations + +- No authentication/authorization yet. +- Not full S3 API coverage. +- No garbage collection of unreferenced blob chunks yet. +- No versioning or lifecycle policies. +- Error and edge-case behavior is still being refined for client compatibility. \ No newline at end of file diff --git a/api/api.go b/api/api.go index eff2c9d..e8b2c8c 100644 --- a/api/api.go +++ b/api/api.go @@ -2,6 +2,7 @@ package api import ( "bufio" + "context" "encoding/xml" "errors" "fmt" @@ -10,9 +11,13 @@ import ( "fs/service" "fs/utils" "io" + "log" "net/http" + "os" + "os/signal" "strconv" "strings" + "syscall" "time" "github.com/go-chi/chi/v5" @@ -58,7 +63,7 @@ func (h *Handler) setupRoutes() { h.router.Delete("/{bucket}/*", h.handleDeleteObject) } -func (h *Handler) handleWelcome(w http.ResponseWriter, r *http.Request) { +func (h *Handler) handleWelcome(w http.ResponseWriter) { w.WriteHeader(http.StatusOK) _, err := w.Write([]byte("Welcome to the Object Storage API!")) if err != nil { @@ -103,7 +108,12 @@ func (h *Handler) handlePostObject(w http.ResponseWriter, r *http.Request) { writeS3Error(w, r, s3ErrInvalidObjectKey, r.URL.Path) return } - defer r.Body.Close() + defer func(Body io.ReadCloser) { + err := Body.Close() + if err != nil { + + } + }(r.Body) if _, ok := r.URL.Query()["uploads"]; ok { upload, err := h.svc.CreateMultipartUpload(bucket, key) @@ -172,7 +182,12 @@ func (h *Handler) handlePutObject(w http.ResponseWriter, r *http.Request) { writeS3Error(w, r, s3ErrInvalidObjectKey, r.URL.Path) return } - defer r.Body.Close() + defer func(Body io.ReadCloser) { + err := Body.Close() + if err != nil { + + } + }(r.Body) bodyReader := io.Reader(r.Body) if shouldDecodeAWSChunkedPayload(r) { @@ -372,7 +387,12 @@ func (h *Handler) handlePostBucket(w http.ResponseWriter, r *http.Request) { writeS3Error(w, r, s3ErrNotImplemented, r.URL.Path) return } - defer r.Body.Close() + defer func(Body io.ReadCloser) { + err := Body.Close() + if err != nil { + + } + }(r.Body) bodyReader := io.Reader(r.Body) if shouldDecodeAWSChunkedPayload(r) { @@ -467,7 +487,10 @@ func (h *Handler) handleGetBuckets(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/xml") w.WriteHeader(http.StatusOK) for _, bucket := range buckets { - w.Write([]byte(bucket)) + _, err := w.Write([]byte(bucket)) + if err != nil { + return + } } } @@ -489,7 +512,10 @@ func (h *Handler) handleGetBucket(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/xml; charset=utf-8") w.Header().Set("Content-Length", strconv.Itoa(len(xmlResponse))) w.WriteHeader(http.StatusOK) - w.Write([]byte(xmlResponse)) + _, err := w.Write([]byte(xmlResponse)) + if err != nil { + return + } return } writeS3Error(w, r, s3ErrNotImplemented, r.URL.Path) @@ -522,5 +548,28 @@ func (h *Handler) handleListObjectsV2(w http.ResponseWriter, r *http.Request, bu func (h *Handler) Start(address string) error { fmt.Printf("Starting API server on %s\n", address) h.setupRoutes() - return http.ListenAndServe(address, h.router) + stop := make(chan os.Signal, 1) + signal.Notify(stop, os.Interrupt, syscall.SIGTERM) + server := http.Server{ + Addr: address, + Handler: h.router, + } + + go func() { + if err := server.ListenAndServe(); err != nil { + log.Fatal(err) + } + }() + <-stop + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + if err := server.Shutdown(ctx); err != nil { + return err + } + if err := h.svc.Close(); err != nil { + return err + } + + return nil } diff --git a/metadata/metadata.go b/metadata/metadata.go index ce346ac..d6b1719 100644 --- a/metadata/metadata.go +++ b/metadata/metadata.go @@ -70,6 +70,10 @@ func NewMetadataHandler(dbPath string) (*MetadataHandler, error) { return h, nil } +func (h *MetadataHandler) Close() error { + return h.db.Close() +} + func (h *MetadataHandler) CreateBucket(bucketName string) error { if !validBucketName.MatchString(bucketName) { return fmt.Errorf("%w: %s", ErrInvalidBucketName, bucketName) diff --git a/service/service.go b/service/service.go index 755aea1..26eee1b 100644 --- a/service/service.go +++ b/service/service.go @@ -254,3 +254,7 @@ func buildMultipartETag(parts []models.UploadedPart) string { } return fmt.Sprintf("%x-%d", hasher.Sum(nil), len(parts)) } + +func (s *ObjectService) Close() error { + return s.metadataHandler.Close() +}