mirror of
https://github.com/ferdzo/fs.git
synced 2026-04-05 01:36:25 +00:00
Finialized multipart upload and graceful shutdown. Added Dockerfile.
This commit is contained in:
3
.dockerignore
Normal file
3
.dockerignore
Normal file
@@ -0,0 +1,3 @@
|
||||
*.md
|
||||
.gocache/
|
||||
blobs/
|
||||
16
Dockerfile
Normal file
16
Dockerfile
Normal file
@@ -0,0 +1,16 @@
|
||||
FROM golang:1.25-alpine AS build
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
COPY . .
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -o /app/fs .
|
||||
|
||||
FROM scratch AS runner
|
||||
|
||||
COPY --from=build /app/fs /app/fs
|
||||
|
||||
WORKDIR /app
|
||||
CMD ["/app/fs"]
|
||||
33
README.md
33
README.md
@@ -1,3 +1,34 @@
|
||||
# fs
|
||||
|
||||
An experimental Object Storage written in Go that should be compatible with S3
|
||||
An experimental Object Storage written in Go that should be partially compatible with S3
|
||||
|
||||
## Features
|
||||
|
||||
- Bucket operations:
|
||||
- `PUT /{bucket}`
|
||||
- `HEAD /{bucket}`
|
||||
- `DELETE /{bucket}`
|
||||
- `GET /` (list buckets)
|
||||
- Object operations:
|
||||
- `PUT /{bucket}/{key}`
|
||||
- `GET /{bucket}/{key}`
|
||||
- `HEAD /{bucket}/{key}`
|
||||
- `DELETE /{bucket}/{key}`
|
||||
- `GET /{bucket}?list-type=2&prefix=...` (ListObjectsV2-style)
|
||||
- Multipart upload:
|
||||
- `POST /{bucket}/{key}?uploads` (initiate)
|
||||
- `PUT /{bucket}/{key}?uploadId=...&partNumber=N` (upload part)
|
||||
- `GET /{bucket}/{key}?uploadId=...` (list parts)
|
||||
- `POST /{bucket}/{key}?uploadId=...` (complete)
|
||||
- `DELETE /{bucket}/{key}?uploadId=...` (abort)
|
||||
- Multi-object delete:
|
||||
- `POST /{bucket}?delete` with S3-style XML body
|
||||
- AWS SigV4 streaming payload decoding for uploads (`aws-chunked` request bodies)
|
||||
|
||||
## Limitations
|
||||
|
||||
- No authentication/authorization yet.
|
||||
- Not full S3 API coverage.
|
||||
- No garbage collection of unreferenced blob chunks yet.
|
||||
- No versioning or lifecycle policies.
|
||||
- Error and edge-case behavior is still being refined for client compatibility.
|
||||
63
api/api.go
63
api/api.go
@@ -2,6 +2,7 @@ package api
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -10,9 +11,13 @@ import (
|
||||
"fs/service"
|
||||
"fs/utils"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
@@ -58,7 +63,7 @@ func (h *Handler) setupRoutes() {
|
||||
h.router.Delete("/{bucket}/*", h.handleDeleteObject)
|
||||
}
|
||||
|
||||
func (h *Handler) handleWelcome(w http.ResponseWriter, r *http.Request) {
|
||||
func (h *Handler) handleWelcome(w http.ResponseWriter) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, err := w.Write([]byte("Welcome to the Object Storage API!"))
|
||||
if err != nil {
|
||||
@@ -103,7 +108,12 @@ func (h *Handler) handlePostObject(w http.ResponseWriter, r *http.Request) {
|
||||
writeS3Error(w, r, s3ErrInvalidObjectKey, r.URL.Path)
|
||||
return
|
||||
}
|
||||
defer r.Body.Close()
|
||||
defer func(Body io.ReadCloser) {
|
||||
err := Body.Close()
|
||||
if err != nil {
|
||||
|
||||
}
|
||||
}(r.Body)
|
||||
|
||||
if _, ok := r.URL.Query()["uploads"]; ok {
|
||||
upload, err := h.svc.CreateMultipartUpload(bucket, key)
|
||||
@@ -172,7 +182,12 @@ func (h *Handler) handlePutObject(w http.ResponseWriter, r *http.Request) {
|
||||
writeS3Error(w, r, s3ErrInvalidObjectKey, r.URL.Path)
|
||||
return
|
||||
}
|
||||
defer r.Body.Close()
|
||||
defer func(Body io.ReadCloser) {
|
||||
err := Body.Close()
|
||||
if err != nil {
|
||||
|
||||
}
|
||||
}(r.Body)
|
||||
|
||||
bodyReader := io.Reader(r.Body)
|
||||
if shouldDecodeAWSChunkedPayload(r) {
|
||||
@@ -372,7 +387,12 @@ func (h *Handler) handlePostBucket(w http.ResponseWriter, r *http.Request) {
|
||||
writeS3Error(w, r, s3ErrNotImplemented, r.URL.Path)
|
||||
return
|
||||
}
|
||||
defer r.Body.Close()
|
||||
defer func(Body io.ReadCloser) {
|
||||
err := Body.Close()
|
||||
if err != nil {
|
||||
|
||||
}
|
||||
}(r.Body)
|
||||
|
||||
bodyReader := io.Reader(r.Body)
|
||||
if shouldDecodeAWSChunkedPayload(r) {
|
||||
@@ -467,7 +487,10 @@ func (h *Handler) handleGetBuckets(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
for _, bucket := range buckets {
|
||||
w.Write([]byte(bucket))
|
||||
_, err := w.Write([]byte(bucket))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -489,7 +512,10 @@ func (h *Handler) handleGetBucket(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/xml; charset=utf-8")
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(xmlResponse)))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte(xmlResponse))
|
||||
_, err := w.Write([]byte(xmlResponse))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
writeS3Error(w, r, s3ErrNotImplemented, r.URL.Path)
|
||||
@@ -522,5 +548,28 @@ func (h *Handler) handleListObjectsV2(w http.ResponseWriter, r *http.Request, bu
|
||||
func (h *Handler) Start(address string) error {
|
||||
fmt.Printf("Starting API server on %s\n", address)
|
||||
h.setupRoutes()
|
||||
return http.ListenAndServe(address, h.router)
|
||||
stop := make(chan os.Signal, 1)
|
||||
signal.Notify(stop, os.Interrupt, syscall.SIGTERM)
|
||||
server := http.Server{
|
||||
Addr: address,
|
||||
Handler: h.router,
|
||||
}
|
||||
|
||||
go func() {
|
||||
if err := server.ListenAndServe(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}()
|
||||
<-stop
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if err := server.Shutdown(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := h.svc.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -70,6 +70,10 @@ func NewMetadataHandler(dbPath string) (*MetadataHandler, error) {
|
||||
return h, nil
|
||||
}
|
||||
|
||||
func (h *MetadataHandler) Close() error {
|
||||
return h.db.Close()
|
||||
}
|
||||
|
||||
func (h *MetadataHandler) CreateBucket(bucketName string) error {
|
||||
if !validBucketName.MatchString(bucketName) {
|
||||
return fmt.Errorf("%w: %s", ErrInvalidBucketName, bucketName)
|
||||
|
||||
@@ -254,3 +254,7 @@ func buildMultipartETag(parts []models.UploadedPart) string {
|
||||
}
|
||||
return fmt.Sprintf("%x-%d", hasher.Sum(nil), len(parts))
|
||||
}
|
||||
|
||||
func (s *ObjectService) Close() error {
|
||||
return s.metadataHandler.Close()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user