Finialized multipart upload and graceful shutdown. Added Dockerfile.

This commit is contained in:
2026-02-22 23:00:33 +01:00
parent 5d41ec9e0a
commit c989037160
6 changed files with 115 additions and 8 deletions

3
.dockerignore Normal file
View File

@@ -0,0 +1,3 @@
*.md
.gocache/
blobs/

16
Dockerfile Normal file
View File

@@ -0,0 +1,16 @@
FROM golang:1.25-alpine AS build
WORKDIR /app
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN CGO_ENABLED=0 GOOS=linux go build -o /app/fs .
FROM scratch AS runner
COPY --from=build /app/fs /app/fs
WORKDIR /app
CMD ["/app/fs"]

View File

@@ -1,3 +1,34 @@
# fs
An experimental Object Storage written in Go that should be compatible with S3
An experimental Object Storage written in Go that should be partially compatible with S3
## Features
- Bucket operations:
- `PUT /{bucket}`
- `HEAD /{bucket}`
- `DELETE /{bucket}`
- `GET /` (list buckets)
- Object operations:
- `PUT /{bucket}/{key}`
- `GET /{bucket}/{key}`
- `HEAD /{bucket}/{key}`
- `DELETE /{bucket}/{key}`
- `GET /{bucket}?list-type=2&prefix=...` (ListObjectsV2-style)
- Multipart upload:
- `POST /{bucket}/{key}?uploads` (initiate)
- `PUT /{bucket}/{key}?uploadId=...&partNumber=N` (upload part)
- `GET /{bucket}/{key}?uploadId=...` (list parts)
- `POST /{bucket}/{key}?uploadId=...` (complete)
- `DELETE /{bucket}/{key}?uploadId=...` (abort)
- Multi-object delete:
- `POST /{bucket}?delete` with S3-style XML body
- AWS SigV4 streaming payload decoding for uploads (`aws-chunked` request bodies)
## Limitations
- No authentication/authorization yet.
- Not full S3 API coverage.
- No garbage collection of unreferenced blob chunks yet.
- No versioning or lifecycle policies.
- Error and edge-case behavior is still being refined for client compatibility.

View File

@@ -2,6 +2,7 @@ package api
import (
"bufio"
"context"
"encoding/xml"
"errors"
"fmt"
@@ -10,9 +11,13 @@ import (
"fs/service"
"fs/utils"
"io"
"log"
"net/http"
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
"github.com/go-chi/chi/v5"
@@ -58,7 +63,7 @@ func (h *Handler) setupRoutes() {
h.router.Delete("/{bucket}/*", h.handleDeleteObject)
}
func (h *Handler) handleWelcome(w http.ResponseWriter, r *http.Request) {
func (h *Handler) handleWelcome(w http.ResponseWriter) {
w.WriteHeader(http.StatusOK)
_, err := w.Write([]byte("Welcome to the Object Storage API!"))
if err != nil {
@@ -103,7 +108,12 @@ func (h *Handler) handlePostObject(w http.ResponseWriter, r *http.Request) {
writeS3Error(w, r, s3ErrInvalidObjectKey, r.URL.Path)
return
}
defer r.Body.Close()
defer func(Body io.ReadCloser) {
err := Body.Close()
if err != nil {
}
}(r.Body)
if _, ok := r.URL.Query()["uploads"]; ok {
upload, err := h.svc.CreateMultipartUpload(bucket, key)
@@ -172,7 +182,12 @@ func (h *Handler) handlePutObject(w http.ResponseWriter, r *http.Request) {
writeS3Error(w, r, s3ErrInvalidObjectKey, r.URL.Path)
return
}
defer r.Body.Close()
defer func(Body io.ReadCloser) {
err := Body.Close()
if err != nil {
}
}(r.Body)
bodyReader := io.Reader(r.Body)
if shouldDecodeAWSChunkedPayload(r) {
@@ -372,7 +387,12 @@ func (h *Handler) handlePostBucket(w http.ResponseWriter, r *http.Request) {
writeS3Error(w, r, s3ErrNotImplemented, r.URL.Path)
return
}
defer r.Body.Close()
defer func(Body io.ReadCloser) {
err := Body.Close()
if err != nil {
}
}(r.Body)
bodyReader := io.Reader(r.Body)
if shouldDecodeAWSChunkedPayload(r) {
@@ -467,7 +487,10 @@ func (h *Handler) handleGetBuckets(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/xml")
w.WriteHeader(http.StatusOK)
for _, bucket := range buckets {
w.Write([]byte(bucket))
_, err := w.Write([]byte(bucket))
if err != nil {
return
}
}
}
@@ -489,7 +512,10 @@ func (h *Handler) handleGetBucket(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/xml; charset=utf-8")
w.Header().Set("Content-Length", strconv.Itoa(len(xmlResponse)))
w.WriteHeader(http.StatusOK)
w.Write([]byte(xmlResponse))
_, err := w.Write([]byte(xmlResponse))
if err != nil {
return
}
return
}
writeS3Error(w, r, s3ErrNotImplemented, r.URL.Path)
@@ -522,5 +548,28 @@ func (h *Handler) handleListObjectsV2(w http.ResponseWriter, r *http.Request, bu
func (h *Handler) Start(address string) error {
fmt.Printf("Starting API server on %s\n", address)
h.setupRoutes()
return http.ListenAndServe(address, h.router)
stop := make(chan os.Signal, 1)
signal.Notify(stop, os.Interrupt, syscall.SIGTERM)
server := http.Server{
Addr: address,
Handler: h.router,
}
go func() {
if err := server.ListenAndServe(); err != nil {
log.Fatal(err)
}
}()
<-stop
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := server.Shutdown(ctx); err != nil {
return err
}
if err := h.svc.Close(); err != nil {
return err
}
return nil
}

View File

@@ -70,6 +70,10 @@ func NewMetadataHandler(dbPath string) (*MetadataHandler, error) {
return h, nil
}
func (h *MetadataHandler) Close() error {
return h.db.Close()
}
func (h *MetadataHandler) CreateBucket(bucketName string) error {
if !validBucketName.MatchString(bucketName) {
return fmt.Errorf("%w: %s", ErrInvalidBucketName, bucketName)

View File

@@ -254,3 +254,7 @@ func buildMultipartETag(parts []models.UploadedPart) string {
}
return fmt.Sprintf("%x-%d", hasher.Sum(nil), len(parts))
}
func (s *ObjectService) Close() error {
return s.metadataHandler.Close()
}