mirror of
https://github.com/ferdzo/fs.git
synced 2026-04-05 14:06:25 +00:00
Finialized multipart upload and graceful shutdown. Added Dockerfile.
This commit is contained in:
3
.dockerignore
Normal file
3
.dockerignore
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
*.md
|
||||||
|
.gocache/
|
||||||
|
blobs/
|
||||||
16
Dockerfile
Normal file
16
Dockerfile
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
FROM golang:1.25-alpine AS build
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY go.mod go.sum ./
|
||||||
|
RUN go mod download
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
RUN CGO_ENABLED=0 GOOS=linux go build -o /app/fs .
|
||||||
|
|
||||||
|
FROM scratch AS runner
|
||||||
|
|
||||||
|
COPY --from=build /app/fs /app/fs
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
CMD ["/app/fs"]
|
||||||
33
README.md
33
README.md
@@ -1,3 +1,34 @@
|
|||||||
# fs
|
# fs
|
||||||
|
|
||||||
An experimental Object Storage written in Go that should be compatible with S3
|
An experimental Object Storage written in Go that should be partially compatible with S3
|
||||||
|
|
||||||
|
## Features
|
||||||
|
|
||||||
|
- Bucket operations:
|
||||||
|
- `PUT /{bucket}`
|
||||||
|
- `HEAD /{bucket}`
|
||||||
|
- `DELETE /{bucket}`
|
||||||
|
- `GET /` (list buckets)
|
||||||
|
- Object operations:
|
||||||
|
- `PUT /{bucket}/{key}`
|
||||||
|
- `GET /{bucket}/{key}`
|
||||||
|
- `HEAD /{bucket}/{key}`
|
||||||
|
- `DELETE /{bucket}/{key}`
|
||||||
|
- `GET /{bucket}?list-type=2&prefix=...` (ListObjectsV2-style)
|
||||||
|
- Multipart upload:
|
||||||
|
- `POST /{bucket}/{key}?uploads` (initiate)
|
||||||
|
- `PUT /{bucket}/{key}?uploadId=...&partNumber=N` (upload part)
|
||||||
|
- `GET /{bucket}/{key}?uploadId=...` (list parts)
|
||||||
|
- `POST /{bucket}/{key}?uploadId=...` (complete)
|
||||||
|
- `DELETE /{bucket}/{key}?uploadId=...` (abort)
|
||||||
|
- Multi-object delete:
|
||||||
|
- `POST /{bucket}?delete` with S3-style XML body
|
||||||
|
- AWS SigV4 streaming payload decoding for uploads (`aws-chunked` request bodies)
|
||||||
|
|
||||||
|
## Limitations
|
||||||
|
|
||||||
|
- No authentication/authorization yet.
|
||||||
|
- Not full S3 API coverage.
|
||||||
|
- No garbage collection of unreferenced blob chunks yet.
|
||||||
|
- No versioning or lifecycle policies.
|
||||||
|
- Error and edge-case behavior is still being refined for client compatibility.
|
||||||
63
api/api.go
63
api/api.go
@@ -2,6 +2,7 @@ package api
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
|
"context"
|
||||||
"encoding/xml"
|
"encoding/xml"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -10,9 +11,13 @@ import (
|
|||||||
"fs/service"
|
"fs/service"
|
||||||
"fs/utils"
|
"fs/utils"
|
||||||
"io"
|
"io"
|
||||||
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/go-chi/chi/v5"
|
"github.com/go-chi/chi/v5"
|
||||||
@@ -58,7 +63,7 @@ func (h *Handler) setupRoutes() {
|
|||||||
h.router.Delete("/{bucket}/*", h.handleDeleteObject)
|
h.router.Delete("/{bucket}/*", h.handleDeleteObject)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *Handler) handleWelcome(w http.ResponseWriter, r *http.Request) {
|
func (h *Handler) handleWelcome(w http.ResponseWriter) {
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
_, err := w.Write([]byte("Welcome to the Object Storage API!"))
|
_, err := w.Write([]byte("Welcome to the Object Storage API!"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -103,7 +108,12 @@ func (h *Handler) handlePostObject(w http.ResponseWriter, r *http.Request) {
|
|||||||
writeS3Error(w, r, s3ErrInvalidObjectKey, r.URL.Path)
|
writeS3Error(w, r, s3ErrInvalidObjectKey, r.URL.Path)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer r.Body.Close()
|
defer func(Body io.ReadCloser) {
|
||||||
|
err := Body.Close()
|
||||||
|
if err != nil {
|
||||||
|
|
||||||
|
}
|
||||||
|
}(r.Body)
|
||||||
|
|
||||||
if _, ok := r.URL.Query()["uploads"]; ok {
|
if _, ok := r.URL.Query()["uploads"]; ok {
|
||||||
upload, err := h.svc.CreateMultipartUpload(bucket, key)
|
upload, err := h.svc.CreateMultipartUpload(bucket, key)
|
||||||
@@ -172,7 +182,12 @@ func (h *Handler) handlePutObject(w http.ResponseWriter, r *http.Request) {
|
|||||||
writeS3Error(w, r, s3ErrInvalidObjectKey, r.URL.Path)
|
writeS3Error(w, r, s3ErrInvalidObjectKey, r.URL.Path)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer r.Body.Close()
|
defer func(Body io.ReadCloser) {
|
||||||
|
err := Body.Close()
|
||||||
|
if err != nil {
|
||||||
|
|
||||||
|
}
|
||||||
|
}(r.Body)
|
||||||
|
|
||||||
bodyReader := io.Reader(r.Body)
|
bodyReader := io.Reader(r.Body)
|
||||||
if shouldDecodeAWSChunkedPayload(r) {
|
if shouldDecodeAWSChunkedPayload(r) {
|
||||||
@@ -372,7 +387,12 @@ func (h *Handler) handlePostBucket(w http.ResponseWriter, r *http.Request) {
|
|||||||
writeS3Error(w, r, s3ErrNotImplemented, r.URL.Path)
|
writeS3Error(w, r, s3ErrNotImplemented, r.URL.Path)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer r.Body.Close()
|
defer func(Body io.ReadCloser) {
|
||||||
|
err := Body.Close()
|
||||||
|
if err != nil {
|
||||||
|
|
||||||
|
}
|
||||||
|
}(r.Body)
|
||||||
|
|
||||||
bodyReader := io.Reader(r.Body)
|
bodyReader := io.Reader(r.Body)
|
||||||
if shouldDecodeAWSChunkedPayload(r) {
|
if shouldDecodeAWSChunkedPayload(r) {
|
||||||
@@ -467,7 +487,10 @@ func (h *Handler) handleGetBuckets(w http.ResponseWriter, r *http.Request) {
|
|||||||
w.Header().Set("Content-Type", "application/xml")
|
w.Header().Set("Content-Type", "application/xml")
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
for _, bucket := range buckets {
|
for _, bucket := range buckets {
|
||||||
w.Write([]byte(bucket))
|
_, err := w.Write([]byte(bucket))
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -489,7 +512,10 @@ func (h *Handler) handleGetBucket(w http.ResponseWriter, r *http.Request) {
|
|||||||
w.Header().Set("Content-Type", "application/xml; charset=utf-8")
|
w.Header().Set("Content-Type", "application/xml; charset=utf-8")
|
||||||
w.Header().Set("Content-Length", strconv.Itoa(len(xmlResponse)))
|
w.Header().Set("Content-Length", strconv.Itoa(len(xmlResponse)))
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
w.Write([]byte(xmlResponse))
|
_, err := w.Write([]byte(xmlResponse))
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
writeS3Error(w, r, s3ErrNotImplemented, r.URL.Path)
|
writeS3Error(w, r, s3ErrNotImplemented, r.URL.Path)
|
||||||
@@ -522,5 +548,28 @@ func (h *Handler) handleListObjectsV2(w http.ResponseWriter, r *http.Request, bu
|
|||||||
func (h *Handler) Start(address string) error {
|
func (h *Handler) Start(address string) error {
|
||||||
fmt.Printf("Starting API server on %s\n", address)
|
fmt.Printf("Starting API server on %s\n", address)
|
||||||
h.setupRoutes()
|
h.setupRoutes()
|
||||||
return http.ListenAndServe(address, h.router)
|
stop := make(chan os.Signal, 1)
|
||||||
|
signal.Notify(stop, os.Interrupt, syscall.SIGTERM)
|
||||||
|
server := http.Server{
|
||||||
|
Addr: address,
|
||||||
|
Handler: h.router,
|
||||||
|
}
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
if err := server.ListenAndServe(); err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
<-stop
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
if err := server.Shutdown(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := h.svc.Close(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -70,6 +70,10 @@ func NewMetadataHandler(dbPath string) (*MetadataHandler, error) {
|
|||||||
return h, nil
|
return h, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (h *MetadataHandler) Close() error {
|
||||||
|
return h.db.Close()
|
||||||
|
}
|
||||||
|
|
||||||
func (h *MetadataHandler) CreateBucket(bucketName string) error {
|
func (h *MetadataHandler) CreateBucket(bucketName string) error {
|
||||||
if !validBucketName.MatchString(bucketName) {
|
if !validBucketName.MatchString(bucketName) {
|
||||||
return fmt.Errorf("%w: %s", ErrInvalidBucketName, bucketName)
|
return fmt.Errorf("%w: %s", ErrInvalidBucketName, bucketName)
|
||||||
|
|||||||
@@ -254,3 +254,7 @@ func buildMultipartETag(parts []models.UploadedPart) string {
|
|||||||
}
|
}
|
||||||
return fmt.Sprintf("%x-%d", hasher.Sum(nil), len(parts))
|
return fmt.Sprintf("%x-%d", hasher.Sum(nil), len(parts))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *ObjectService) Close() error {
|
||||||
|
return s.metadataHandler.Close()
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user