Fixed database migration, imporved logging, imporved hashing function,

introduced connection pooling instead of single conncetion.
This commit is contained in:
Andrej Mickov
2025-08-07 20:45:18 +02:00
parent e178613607
commit f27f1fc3d6
7 changed files with 211 additions and 85 deletions

5
go.mod
View File

@@ -10,8 +10,13 @@ require (
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/pgx/v5 v5.7.5 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/joho/godotenv v1.5.1 // indirect
github.com/lmittmann/tint v1.1.2 // indirect
github.com/redis/go-redis/v9 v9.12.0 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
golang.org/x/crypto v0.37.0 // indirect
golang.org/x/sync v0.13.0 // indirect
golang.org/x/sys v0.32.0 // indirect
golang.org/x/text v0.24.0 // indirect
)

12
go.sum
View File

@@ -1,6 +1,7 @@
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/go-chi/chi/v5 v5.2.2 h1:CMwsvRVTbXVytCk1Wd72Zy1LAsAh9GxMmSNWLHCG618=
@@ -11,16 +12,27 @@ github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7Ulw
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs=
github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/lmittmann/tint v1.1.2 h1:2CQzrL6rslrsyjqLDwD11bZ5OpLBPU+g3G/r5LSfS8w=
github.com/lmittmann/tint v1.1.2/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/redis/go-redis/v9 v9.12.0 h1:XlVPGlflh4nxfhsNXPA8Qp6EmEfTo0rp8oaBzPipXnU=
github.com/redis/go-redis/v9 v9.12.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

View File

@@ -7,11 +7,11 @@ import (
"time"
"github.com/ferdzo/ferurl/utils"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
)
type Database struct {
client *pgx.Conn
connectionPool *pgxpool.Pool
}
type URL struct {
@@ -29,19 +29,19 @@ type PageVisit struct {
}
func NewDatabaseClient(config utils.DatabaseConfig) (*Database, error) {
conn, err := pgx.Connect(context.Background(), utils.DatabaseUrl())
pool, err := pgxpool.New(context.Background(), utils.DatabaseUrl())
if err != nil {
fmt.Fprintf(os.Stderr, "Unable to connect to database: %v\n", err)
return nil, err
}
return &Database{client: conn}, nil
return &Database{connectionPool: pool}, nil
}
func (d *Database) InsertNewURL(u URL) error {
fmt.Print("InsertNewURL called with URL: ", u.URL)
timeNow := time.Now()
_, err := d.client.Exec(context.Background(), "INSERT INTO urls (shorturl, url, created_at,expires_at,active) VALUES ($1, $2, $3,$4,$5)", u.ShortURL, u.URL, timeNow, u.ExpiresAt, true)
_, err := d.connectionPool.Exec(context.Background(), "INSERT INTO urls (shorturl, url, created_at,expires_at,active) VALUES ($1, $2, $3,$4,$5)", u.ShortURL, u.URL, timeNow, u.ExpiresAt, true)
if err != nil {
return fmt.Errorf("failed to insert URL into database: %w", err)
}
@@ -50,7 +50,7 @@ func (d *Database) InsertNewURL(u URL) error {
func (d *Database) InsertAnalytics(p PageVisit) error {
timeNow := time.Now()
_, err := d.client.Exec(context.Background(), "INSERT INTO analytics (shorturl, count, ip_address, user_agent, created_at) VALUES ($1, $2, $3, $4, $5)", p.ShortURL, p.Count, p.IP_Address, p.UserAgent, timeNow)
_, err := d.connectionPool.Exec(context.Background(), "INSERT INTO analytics (shorturl, count, ip_address, user_agent, created_at) VALUES ($1, $2, $3, $4, $5)", p.ShortURL, p.Count, p.IP_Address, p.UserAgent, timeNow)
if err != nil {
return fmt.Errorf("failed to insert analytics into database: %w", err)
}
@@ -58,7 +58,7 @@ func (d *Database) InsertAnalytics(p PageVisit) error {
}
func (d *Database) GetAnalytics(shorturl string) ([]PageVisit, error) {
rows, err := d.client.Query(context.Background(), "SELECT shorturl, count, ip_address, user_agent, created_at FROM analytics WHERE shorturl = $1", shorturl)
rows, err := d.connectionPool.Query(context.Background(), "SELECT shorturl, count, ip_address, user_agent, created_at FROM analytics WHERE shorturl = $1", shorturl)
if err != nil {
return nil, fmt.Errorf("failed to retrieve analytics from database: %w", err)
}
@@ -80,7 +80,7 @@ func (d *Database) GetAnalytics(shorturl string) ([]PageVisit, error) {
}
func (d *Database) DeleteURL(shorturl string) error {
_, err := d.client.Exec(context.Background(), "DELETE FROM urls WHERE shorturl = $1", shorturl)
_, err := d.connectionPool.Exec(context.Background(), "DELETE FROM urls WHERE shorturl = $1", shorturl)
if err != nil {
return fmt.Errorf("failed to delete URL from database: %w", err)
}
@@ -89,7 +89,7 @@ func (d *Database) DeleteURL(shorturl string) error {
func (d *Database) GetURL(shorturl string) (string, error) {
var url string
err := d.client.QueryRow(context.Background(), "SELECT url FROM urls WHERE shorturl = $1", shorturl).Scan(&url)
err := d.connectionPool.QueryRow(context.Background(), "SELECT url FROM urls WHERE shorturl = $1", shorturl).Scan(&url)
if err != nil {
return "", fmt.Errorf("failed to retrieve URL from database: %w", err)
}

View File

@@ -1,20 +1,20 @@
CREATE TABLE urls (
shorturl VARCHAR(7) PRIMARY KEY,
url VARCHAR(100) NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
expires_at TIMESTAMP DEFAULT NULL,
active BOOLEAN DEFAULT TRUE
shorturl VARCHAR(7) PRIMARY KEY,
url VARCHAR(100) NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
expires_at TIMESTAMP DEFAULT NULL,
active BOOLEAN DEFAULT TRUE
);
CREATE TABLE analytics (
id BIGINT AUTO_INCREMENT PRIMARY KEY,
shorturl VARCHAR(7) NOT NULL,
count INTEGER DEFAULT 0,
ip_address VARCHAR(50) DEFAULT NULL,
user_agent VARCHAR(1024) DEFAULT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
id SERIAL PRIMARY KEY,
shorturl VARCHAR(7) NOT NULL,
count INTEGER DEFAULT 0,
ip_address VARCHAR(50) DEFAULT NULL,
user_agent VARCHAR(1024) DEFAULT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (shorturl) REFERENCES urls(shorturl),
FOREIGN KEY (shorturl) REFERENCES urls(shorturl)
);

View File

@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"net/http"
"strings"
"sync"
"time"
@@ -12,8 +13,11 @@ import (
"github.com/ferdzo/ferurl/internal/db"
"github.com/ferdzo/ferurl/utils"
"github.com/go-chi/chi/v5"
"github.com/sirupsen/logrus"
)
var log = logrus.New()
type Service struct {
cache *cache.Cache
database *db.Database
@@ -25,12 +29,17 @@ type Handler struct {
}
func NewService(redisConfig utils.RedisConfig, databaseConfig utils.DatabaseConfig) (*Service, error) {
log.Info("Initializing Redis client")
redisClient, err := cache.NewRedisClient(redisConfig)
if err != nil {
log.WithError(err).Error("Failed to initialize Redis client")
return nil, err
}
log.Info("Initializing Database client")
databaseClient, err := db.NewDatabaseClient(databaseConfig)
if err != nil {
log.WithError(err).Error("Failed to initialize Database client")
return nil, err
}
@@ -63,28 +72,34 @@ func (h *Handler) CreateShortURL(w http.ResponseWriter, r *http.Request) {
decoder := json.NewDecoder(r.Body)
if err := decoder.Decode(&input); err != nil {
log.WithError(err).Warn("Invalid JSON input received")
http.Error(w, "Invalid JSON input", http.StatusBadRequest)
return
}
longUrl := input.URL
if longUrl == "" {
log.Warn("Empty URL received")
http.Error(w, "Long URL is required", http.StatusBadRequest)
return
}
if !utils.IsValidUrl(longUrl) {
log.WithField("url", longUrl).Warn("Invalid URL format received")
http.Error(w, "Invalid URL", http.StatusBadRequest)
return
}
expiresAt := input.ExpiresAt
shortUrl := generateShortUrl(longUrl)
fetchedShortUrl, _ := h.fetchUrl(shortUrl)
if fetchedShortUrl != "" {
fetchedShortUrl, err := h.fetchUrl(shortUrl)
if err == nil && fetchedShortUrl != "" {
fullShortUrl := fmt.Sprintf("%s%s", h.baseURL, shortUrl)
json.NewEncoder(w).Encode(map[string]string{"short_url": fullShortUrl})
return
}
if err != nil && isNotFoundError(err) {
log.WithField("short_url", shortUrl).Debug("Creating new short URL")
}
newUrl := db.URL{
URL: longUrl,
@@ -93,7 +108,7 @@ func (h *Handler) CreateShortURL(w http.ResponseWriter, r *http.Request) {
}
if err := h.service.storeUrl(newUrl); err != nil {
fmt.Println(err)
log.Error("Failed to store URL", "error", err)
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
@@ -113,19 +128,28 @@ func (h *Handler) GetUrl(w http.ResponseWriter, r *http.Request) {
}
shortKey := chi.URLParam(r, "key")
if shortKey == "" {
log.Warn("Empty short URL key received")
http.Error(w, "ID is required", http.StatusBadRequest)
return
}
if !utils.IsValidShortUrl(shortKey) {
log.WithField("key", shortKey).Warn("Invalid short URL format received")
http.Error(w, "Invalid short URL", http.StatusBadRequest)
return
}
longUrl, err := h.fetchUrl(shortKey)
if err != nil {
fmt.Println(err)
http.Error(w, "URL not found", http.StatusInternalServerError)
if isNotFoundError(err) {
http.Error(w, "URL not found", http.StatusNotFound)
} else {
log.WithFields(logrus.Fields{
"short_url": shortKey,
"error": err.Error(),
}).Error("Failed to fetch URL")
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
}
return
}
@@ -137,12 +161,16 @@ func (h *Handler) GetUrl(w http.ResponseWriter, r *http.Request) {
CreatedAt: time.Now(),
}
err = h.service.database.InsertAnalytics(pv)
if err != nil {
fmt.Println(err)
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
go func(pageVisit db.PageVisit) {
defer func() {
if r := recover(); r != nil {
log.WithField("error", r).Error("Panic in analytics insertion")
}
}()
if err := h.service.database.InsertAnalytics(pageVisit); err != nil {
logrus.Error("Failed to store analytics", "error", err)
}
}(pv)
http.Redirect(w, r, longUrl, http.StatusSeeOther)
}
@@ -161,9 +189,13 @@ func (h *Handler) fetchUrl(id string) (string, error) {
}
resultChan := make(chan result, 2)
ctx, cancel := context.WithCancel(context.Background())
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
url, err := h.service.fetchUrlFromCache(id)
select {
case resultChan <- result{url, err, "cache"}:
@@ -173,6 +205,7 @@ func (h *Handler) fetchUrl(id string) (string, error) {
}()
go func() {
defer wg.Done()
url, err := h.service.fetchUrlFromDatabase(id)
select {
case resultChan <- result{url, err, "db"}:
@@ -181,65 +214,118 @@ func (h *Handler) fetchUrl(id string) (string, error) {
}
}()
go func() {
wg.Wait()
close(resultChan)
}()
var foundInDB string
var cacheHit bool
var lastErr error
for range 2 {
res := <-resultChan
if res.err == nil {
if res.source == "db" {
foundInDB = res.url
if !cacheHit {
continue
}
} else if res.source == "cache" {
cacheHit = true
if foundInDB != "" {
return res.url, nil
}
select {
case res, ok := <-resultChan:
if !ok {
break
}
return res.url, nil
if res.err == nil {
if res.source == "db" {
foundInDB = res.url
if !cacheHit {
continue
}
} else if res.source == "cache" {
cacheHit = true
if foundInDB != "" {
return res.url, nil
}
}
return res.url, nil
}
lastErr = res.err
case <-ctx.Done():
return "", fmt.Errorf("timeout while fetching URL: %w", ctx.Err())
}
lastErr = res.err
}
if foundInDB != "" {
go func() {
_ = h.service.cache.Set(id, foundInDB)
fmt.Println("URL updated in Redis")
}()
go func(cacheKey, url string) {
defer func() {
if r := recover(); r != nil {
log.Error("Panic in cache update", "error", r)
}
}()
if err := h.service.cache.Set(cacheKey, url); err != nil {
log.Error("Failed to update URL in cache", "error", err)
}
}(id, foundInDB)
return foundInDB, nil
}
if isNotFoundError(lastErr) {
log.Warn("URL not found")
return "", nil
}
return "", lastErr
}
func (s *Service) fetchUrlFromCache(shortUrl string) (string, error) {
if url, err := s.cache.Get(shortUrl); err == nil {
return url, nil
url, err := s.cache.Get(shortUrl)
if err != nil {
log.WithFields(logrus.Fields{
"short_url": shortUrl,
"error": err,
}).Debug("Cache miss")
return "", err
}
return "", fmt.Errorf("URL not found")
return url, nil
}
func (s *Service) fetchUrlFromDatabase(shortUrl string) (string, error) {
if url, err := s.database.GetURL(shortUrl); err == nil {
return url, nil
url, err := s.database.GetURL(shortUrl)
if err != nil {
log.WithFields(logrus.Fields{
"short_url": shortUrl,
"error": err,
}).Debug("Database lookup failed")
return "", err
}
log.WithField("short_url", shortUrl).Debug("Database hit")
return url, nil
}
return "", fmt.Errorf("URL not found")
func isNotFoundError(err error) bool {
if err == nil {
return false
}
errMsg := err.Error()
return strings.Contains(errMsg, "not found") ||
strings.Contains(errMsg, "redis: nil") ||
strings.Contains(errMsg, "no rows in result set")
}
func (s *Service) storeUrl(u db.URL) error {
var wg sync.WaitGroup
errChan := make(chan error, 2)
wg.Add(1)
wg.Add(2)
go func() {
defer wg.Done()
if err := s.cache.Set(u.ShortURL, u.URL); err != nil {
errChan <- err
log.WithFields(logrus.Fields{
"short_url": u.ShortURL,
"error": err,
}).Error("Failed to store URL in cache")
select {
case errChan <- err:
default:
}
} else {
log.WithField("short_url", u.ShortURL).Debug("Stored URL in cache")
}
}()
@@ -247,18 +333,26 @@ func (s *Service) storeUrl(u db.URL) error {
go func() {
defer wg.Done()
if err := s.database.InsertNewURL(u); err != nil {
errChan <- err
log.WithFields(logrus.Fields{
"short_url": u.ShortURL,
"error": err,
}).Error("Failed to store URL in database")
select {
case errChan <- err:
default:
}
} else {
log.WithField("short_url", u.ShortURL).Debug("Stored URL in database")
}
}()
go func() {
wg.Wait()
close(errChan)
}()
wg.Wait()
close(errChan)
for err := range errChan {
select {
case err := <-errChan:
return err
default:
return nil
}
return nil
}

35
main.go
View File

@@ -8,23 +8,36 @@ import (
"github.com/ferdzo/ferurl/utils"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
"github.com/sirupsen/logrus"
)
func main() {
fmt.Println("Welcome to ferurl, a simple URL shortener!")
var log = logrus.New()
redisConfig := utils.LoadRedisConfig()
dbConfig := utils.LoadDbConfig()
func main() {
log.Info("Starting ferurl URL shortener service")
redisConfig, err := utils.LoadRedisConfig()
if err != nil {
log.WithError(err).Fatal("Failed to load Redis configuration")
return
}
dbConfig, err := utils.LoadDbConfig()
if err != nil {
log.WithError(err).Fatal("Failed to load database configuration")
return
}
s, err := shortner.NewService(redisConfig, dbConfig)
if err != nil {
fmt.Println("Error creating service:", err)
log.WithError(err).Fatal("Failed to initialize service")
return
}
baseUrl := utils.GetEnv("BASE_URL", "https://url.ferdzo.xyz")
h, err := shortner.NewHandler(s, baseUrl)
if err != nil {
fmt.Println("Error creating handler:", err)
log.WithError(err).Fatal("Failed to initialize handler")
return
}
@@ -34,15 +47,21 @@ func main() {
func initServer(h shortner.Handler) {
r := chi.NewRouter()
r.Use(middleware.Logger)
r.Get("/", func(w http.ResponseWriter, r *http.Request) {
http.ServeFile(w, r, "./web/index.html")
})
r.Get("/health", func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, "Welcome to ferurl!")
})
fmt.Println("Server started on port 3000")
r.Post("/create", h.CreateShortURL)
r.Get("/{key}", h.GetUrl)
http.ListenAndServe(":3000", r)
port := utils.GetEnv("PORT", "3000")
log.Info("Starting server on port: " + port)
if err := http.ListenAndServe(":"+port, r); err != nil {
log.WithError(err).Fatal("Server failed to start")
}
}

View File

@@ -2,16 +2,13 @@ package utils
import (
"crypto/sha256"
"fmt"
)
const base62 = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
func GenerateUrlHash(url string) string {
encoded := EncodeToBase62(url)
hash := sha256.Sum256([]byte(encoded))
return fmt.Sprintf("%x", hash)
hash := sha256.Sum256([]byte(url))
return EncodeToBase62(string(hash[:]))
}
func EncodeToBase62(url string) string {
@@ -21,9 +18,8 @@ func EncodeToBase62(url string) string {
for _, b := range hash[:] {
encoded += string(base62[int(b)%62])
}
if len(encoded) > 6 {
encoded = encoded[:6]
if len(encoded) > 7 {
return encoded[:7]
}
return encoded
}