detached 2776772

feat: Enhance rate limiting and storage with Starskey KV store

- Replace memory-based rate limiter with Starskey key-value store for persistent rate limiting
- Modify proxy middleware to use Starskey rate limiter for both HTTP and HTTPS servers
- Add rate limiter cleanup mechanism during proxy shutdown
- Refactor image storage package to use more descriptive `filestore` package
- Update internal references to use new `filestore` package for image handling
- Improve logging and error handling for rate limiting and file storage operations
- Ensure consistent HTTP protocol for internal container connections
commit
277677291013c73d34d4236a123a7a8f87a75e56
author
bamen · · 2025-03-09 20:33 UTC
committer
bamen · · 2025-03-09 20:53 UTC
parent
8a1644c
signature
unsigned
11 files · +485 -103
Unified
UnifiedSplit
M go.mod +9 -0
@@ -19,10 +19,19 @@ require (
1919 github.com/microcosm-cc/bluemonday v1.0.27
2020 github.com/pocketbase/pocketbase v0.22.19
2121 github.com/spf13/cobra v1.8.1
22+ github.com/starskey-io/starskey v0.1.8
2223 gopkg.in/yaml.v3 v3.0.1
2324 modernc.org/sqlite v1.32.0
2425 )
2526
27+require (
28+ github.com/cespare/xxhash/v2 v2.3.0 // indirect
29+ github.com/klauspost/compress v1.16.7 // indirect
30+ github.com/klauspost/cpuid/v2 v2.0.9 // indirect
31+ github.com/zeebo/xxh3 v1.0.2 // indirect
32+ go.mongodb.org/mongo-driver v1.17.2 // indirect
33+)
34+
2635 require (
2736 github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
2837 github.com/charmbracelet/bubbletea v1.1.0 // direct
@@ -19,10 +19,19 @@ require (
19 github.com/microcosm-cc/bluemonday v1.0.2719 github.com/microcosm-cc/bluemonday v1.0.27
20 github.com/pocketbase/pocketbase v0.22.1920 github.com/pocketbase/pocketbase v0.22.19
21 github.com/spf13/cobra v1.8.121 github.com/spf13/cobra v1.8.1
22+ github.com/starskey-io/starskey v0.1.8
22 gopkg.in/yaml.v3 v3.0.123 gopkg.in/yaml.v3 v3.0.1
23 modernc.org/sqlite v1.32.024 modernc.org/sqlite v1.32.0
24 )25 )
25 26
27+require (
28+ github.com/cespare/xxhash/v2 v2.3.0 // indirect
29+ github.com/klauspost/compress v1.16.7 // indirect
30+ github.com/klauspost/cpuid/v2 v2.0.9 // indirect
31+ github.com/zeebo/xxh3 v1.0.2 // indirect
32+ go.mongodb.org/mongo-driver v1.17.2 // indirect
33+)
34+
26 require (35 require (
27 github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect36 github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
28 github.com/charmbracelet/bubbletea v1.1.0 // direct37 github.com/charmbracelet/bubbletea v1.1.0 // direct
M go.sum +14 -0 generated file: collapsed
load diff
M internal/httpserve/handlers/endpoint_helpers.go +5 -5
@@ -17,7 +17,7 @@ import (
1717 "github.com/bnema/gordon/internal/server"
1818 "github.com/bnema/gordon/internal/templating/cmdparams"
1919 "github.com/bnema/gordon/pkg/docker"
20- "github.com/bnema/gordon/pkg/store"
20+ "github.com/bnema/gordon/pkg/filestore"
2121 "github.com/charmbracelet/log"
2222 "github.com/labstack/echo/v4"
2323 )
@@ -143,14 +143,14 @@ func saveAndImportImage(c echo.Context, a *server.App, payload interface{}) (str
143143
144144 imageFileName := sanitizeImageFileName(imageName)
145145
146- imagePath, err := store.SaveImageToStorage(&a.Config, imageFileName, imageReader)
146+ imagePath, err := filestore.SaveImageToStorage(&a.Config, imageFileName, imageReader)
147147 if err != nil {
148148 return "", fmt.Errorf("failed to save image: %v", err)
149149 }
150150
151151 imageID, err = docker.ImportImageToEngine(imagePath)
152152 if err != nil {
153- if removeErr := store.RemoveFromStorage(imagePath); removeErr != nil {
153+ if removeErr := filestore.RemoveFromStorage(imagePath); removeErr != nil {
154154 log.Error("Failed to remove temporary image file after import failure",
155155 "error", removeErr,
156156 "path", imagePath)
@@ -159,7 +159,7 @@ func saveAndImportImage(c echo.Context, a *server.App, payload interface{}) (str
159159 }
160160
161161 if imageID == "" {
162- if removeErr := store.RemoveFromStorage(imagePath); removeErr != nil {
162+ if removeErr := filestore.RemoveFromStorage(imagePath); removeErr != nil {
163163 log.Error("Failed to remove temporary image file after empty ID",
164164 "error", removeErr,
165165 "path", imagePath)
@@ -167,7 +167,7 @@ func saveAndImportImage(c echo.Context, a *server.App, payload interface{}) (str
167167 return "", errors.New("imported image ID is empty")
168168 }
169169
170- err = store.RemoveFromStorage(imagePath)
170+ err = filestore.RemoveFromStorage(imagePath)
171171 if err != nil {
172172 log.Error("Failed to remove temporary image file after successful import",
173173 "error", err,
@@ -17,7 +17,7 @@ import (
17 "github.com/bnema/gordon/internal/server"17 "github.com/bnema/gordon/internal/server"
18 "github.com/bnema/gordon/internal/templating/cmdparams"18 "github.com/bnema/gordon/internal/templating/cmdparams"
19 "github.com/bnema/gordon/pkg/docker"19 "github.com/bnema/gordon/pkg/docker"
20- "github.com/bnema/gordon/pkg/store"20+ "github.com/bnema/gordon/pkg/filestore"
21 "github.com/charmbracelet/log"21 "github.com/charmbracelet/log"
22 "github.com/labstack/echo/v4"22 "github.com/labstack/echo/v4"
23 )23 )
@@ -143,14 +143,14 @@ func saveAndImportImage(c echo.Context, a *server.App, payload interface{}) (str
143 143
144 imageFileName := sanitizeImageFileName(imageName)144 imageFileName := sanitizeImageFileName(imageName)
145 145
146- imagePath, err := store.SaveImageToStorage(&a.Config, imageFileName, imageReader)146+ imagePath, err := filestore.SaveImageToStorage(&a.Config, imageFileName, imageReader)
147 if err != nil {147 if err != nil {
148 return "", fmt.Errorf("failed to save image: %v", err)148 return "", fmt.Errorf("failed to save image: %v", err)
149 }149 }
150 150
151 imageID, err = docker.ImportImageToEngine(imagePath)151 imageID, err = docker.ImportImageToEngine(imagePath)
152 if err != nil {152 if err != nil {
153- if removeErr := store.RemoveFromStorage(imagePath); removeErr != nil {153+ if removeErr := filestore.RemoveFromStorage(imagePath); removeErr != nil {
154 log.Error("Failed to remove temporary image file after import failure",154 log.Error("Failed to remove temporary image file after import failure",
155 "error", removeErr,155 "error", removeErr,
156 "path", imagePath)156 "path", imagePath)
@@ -159,7 +159,7 @@ func saveAndImportImage(c echo.Context, a *server.App, payload interface{}) (str
159 }159 }
160 160
161 if imageID == "" {161 if imageID == "" {
162- if removeErr := store.RemoveFromStorage(imagePath); removeErr != nil {162+ if removeErr := filestore.RemoveFromStorage(imagePath); removeErr != nil {
163 log.Error("Failed to remove temporary image file after empty ID",163 log.Error("Failed to remove temporary image file after empty ID",
164 "error", removeErr,164 "error", removeErr,
165 "path", imagePath)165 "path", imagePath)
@@ -167,7 +167,7 @@ func saveAndImportImage(c echo.Context, a *server.App, payload interface{}) (str
167 return "", errors.New("imported image ID is empty")167 return "", errors.New("imported image ID is empty")
168 }168 }
169 169
170- err = store.RemoveFromStorage(imagePath)170+ err = filestore.RemoveFromStorage(imagePath)
171 if err != nil {171 if err != nil {
172 log.Error("Failed to remove temporary image file after successful import",172 log.Error("Failed to remove temporary image file after successful import",
173 "error", err,173 "error", err,
M internal/httpserve/handlers/upload_image.go +3 -3
@@ -9,7 +9,7 @@ import (
99 "github.com/bnema/gordon/internal/server"
1010 "github.com/bnema/gordon/internal/templating/render"
1111 "github.com/bnema/gordon/pkg/docker"
12- "github.com/bnema/gordon/pkg/store"
12+ "github.com/bnema/gordon/pkg/filestore"
1313 "github.com/labstack/echo/v4"
1414 )
1515
@@ -79,7 +79,7 @@ func UploadImagePOSTHandler(c echo.Context, a *server.App) error {
7979 }
8080
8181 // Save the image to the storage directory
82- saveInPath, err := store.SaveImageToStorage(&a.Config, file.Filename, tempFile)
82+ saveInPath, err := filestore.SaveImageToStorage(&a.Config, file.Filename, tempFile)
8383 if err != nil {
8484 return sendError(c, err)
8585 }
@@ -91,7 +91,7 @@ func UploadImagePOSTHandler(c echo.Context, a *server.App) error {
9191 }
9292
9393 // Remove the image from the storage directory
94- err = store.RemoveFromStorage(saveInPath)
94+ err = filestore.RemoveFromStorage(saveInPath)
9595 if err != nil {
9696 return sendError(c, err)
9797 }
@@ -9,7 +9,7 @@ import (
9 "github.com/bnema/gordon/internal/server"9 "github.com/bnema/gordon/internal/server"
10 "github.com/bnema/gordon/internal/templating/render"10 "github.com/bnema/gordon/internal/templating/render"
11 "github.com/bnema/gordon/pkg/docker"11 "github.com/bnema/gordon/pkg/docker"
12- "github.com/bnema/gordon/pkg/store"12+ "github.com/bnema/gordon/pkg/filestore"
13 "github.com/labstack/echo/v4"13 "github.com/labstack/echo/v4"
14 )14 )
15 15
@@ -79,7 +79,7 @@ func UploadImagePOSTHandler(c echo.Context, a *server.App) error {
79 }79 }
80 80
81 // Save the image to the storage directory81 // Save the image to the storage directory
82- saveInPath, err := store.SaveImageToStorage(&a.Config, file.Filename, tempFile)82+ saveInPath, err := filestore.SaveImageToStorage(&a.Config, file.Filename, tempFile)
83 if err != nil {83 if err != nil {
84 return sendError(c, err)84 return sendError(c, err)
85 }85 }
@@ -91,7 +91,7 @@ func UploadImagePOSTHandler(c echo.Context, a *server.App) error {
91 }91 }
92 92
93 // Remove the image from the storage directory93 // Remove the image from the storage directory
94- err = store.RemoveFromStorage(saveInPath)94+ err = filestore.RemoveFromStorage(saveInPath)
95 if err != nil {95 if err != nil {
96 return sendError(c, err)96 return sendError(c, err)
97 }97 }
M internal/proxy/core.go +3 -0
@@ -51,6 +51,9 @@ type Proxy struct {
5151 func NewProxy(app interfaces.AppInterface) (*Proxy, error) {
5252 log.Debug("Initializing reverse proxy")
5353
54+ // Log important configuration information
55+ log.Debug("All internal connections to containers will use HTTP protocol regardless of external protocol")
56+
5457 // Set up the echo server for HTTPS traffic
5558 httpsServer := echo.New()
5659 httpsServer.HideBanner = true
@@ -51,6 +51,9 @@ type Proxy struct {
51 func NewProxy(app interfaces.AppInterface) (*Proxy, error) {51 func NewProxy(app interfaces.AppInterface) (*Proxy, error) {
52 log.Debug("Initializing reverse proxy")52 log.Debug("Initializing reverse proxy")
53 53
54+ // Log important configuration information
55+ log.Debug("All internal connections to containers will use HTTP protocol regardless of external protocol")
56+
54 // Set up the echo server for HTTPS traffic57 // Set up the echo server for HTTPS traffic
55 httpsServer := echo.New()58 httpsServer := echo.New()
56 httpsServer.HideBanner = true59 httpsServer.HideBanner = true
M internal/proxy/middleware.go +115 -32
@@ -10,8 +10,13 @@ import (
1010 "github.com/labstack/echo/v4"
1111 "github.com/labstack/echo/v4/middleware"
1212 "golang.org/x/time/rate"
13+
14+ "github.com/bnema/gordon/pkg/kv" // Correct import path based on the module name
1315 )
1416
17+// rateLimiterCleanup holds the cleanup functions for rate limiters
18+var rateLimiterCleanup []func()
19+
1520 // Create blacklistedIPs set to track IPs for middleware skipping
1621 type requestContext struct {
1722 blacklisted bool
@@ -22,23 +27,62 @@ func (p *Proxy) setupMiddleware() {
2227 // Create blacklist middleware for HTTPS
2328 p.httpsServer.Use(p.createBlacklistMiddleware())
2429
25- // Add rate limiter middleware to prevent spam attacks
26- p.httpsServer.Use(middleware.RateLimiterWithConfig(middleware.RateLimiterConfig{
27- Store: middleware.NewRateLimiterMemoryStoreWithConfig(
28- middleware.RateLimiterMemoryStoreConfig{
29- Rate: rate.Limit(10), // 10 requests per second
30- Burst: 30, // Burst of 30 requests
31- ExpiresIn: 3 * time.Minute, // Store expiration
30+ // Create a new Starskey rate limiter store for HTTPS server
31+ httpsRateLimiter, err := kv.NewStarskeyRateLimiterStore(
32+ "./data/ratelimiter/https", // Path to store rate limit data
33+ 10, // Rate: 10 requests per second
34+ 30, // Burst: 30 requests
35+ 3*time.Minute, // Expires in 3 minutes
36+ )
37+ if err != nil {
38+ log.Error("Failed to create HTTPS rate limiter store, falling back to memory store", "error", err)
39+
40+ // Fallback to memory store if Starskey store creation fails
41+ p.httpsServer.Use(middleware.RateLimiterWithConfig(middleware.RateLimiterConfig{
42+ Store: middleware.NewRateLimiterMemoryStoreWithConfig(
43+ middleware.RateLimiterMemoryStoreConfig{
44+ Rate: rate.Limit(10), // 10 requests per second
45+ Burst: 30, // Burst of 30 requests
46+ ExpiresIn: 3 * time.Minute, // Store expiration
47+ },
48+ ),
49+ DenyHandler: func(context echo.Context, identifier string, err error) error {
50+ log.Warn("Rate limit exceeded",
51+ "ip", identifier,
52+ "path", context.Request().URL.Path,
53+ "method", context.Request().Method)
54+ return context.String(http.StatusTooManyRequests, "Too many requests")
55+ },
56+ }))
57+ } else {
58+ // Use the Starskey store for HTTPS rate limiting
59+ log.Debug("Using Starskey rate limiter for HTTPS server")
60+
61+ // Add custom Starskey rate limiter middleware
62+ p.httpsServer.Use(middleware.RateLimiterWithConfig(middleware.RateLimiterConfig{
63+ Store: httpsRateLimiter,
64+ IdentifierExtractor: func(ctx echo.Context) (string, error) {
65+ // Use the client's real IP as the identifier
66+ id := ctx.RealIP()
67+ log.Debug("Rate limiting HTTPS request", "ip", id)
68+ return id, nil
3269 },
33- ),
34- DenyHandler: func(context echo.Context, identifier string, err error) error {
35- log.Warn("Rate limit exceeded",
36- "ip", identifier,
37- "path", context.Request().URL.Path,
38- "method", context.Request().Method)
39- return context.String(http.StatusTooManyRequests, "Too many requests")
40- },
41- }))
70+ DenyHandler: func(context echo.Context, identifier string, err error) error {
71+ log.Warn("HTTPS rate limit exceeded",
72+ "ip", identifier,
73+ "path", context.Request().URL.Path,
74+ "method", context.Request().Method)
75+ return context.String(http.StatusTooManyRequests, "Too many requests")
76+ },
77+ }))
78+
79+ // Add cleanup function to global slice
80+ rateLimiterCleanup = append(rateLimiterCleanup, func() {
81+ if err := httpsRateLimiter.Close(); err != nil {
82+ log.Error("Failed to close HTTPS rate limiter store", "error", err)
83+ }
84+ })
85+ }
4286
4387 // Add default Echo logger middleware only if enabled in config
4488 if p.config.EnableLogs {
@@ -51,23 +95,62 @@ func (p *Proxy) setupMiddleware() {
5195 // Create blacklist middleware for HTTP
5296 p.httpServer.Use(p.createBlacklistMiddleware())
5397
54- // Add rate limiter middleware to prevent spam attacks for HTTP server
55- p.httpServer.Use(middleware.RateLimiterWithConfig(middleware.RateLimiterConfig{
56- Store: middleware.NewRateLimiterMemoryStoreWithConfig(
57- middleware.RateLimiterMemoryStoreConfig{
58- Rate: rate.Limit(10), // 10 requests per second
59- Burst: 30, // Burst of 30 requests
60- ExpiresIn: 3 * time.Minute, // Store expiration
98+ // Create a new Starskey rate limiter store for HTTP server
99+ httpRateLimiter, err := kv.NewStarskeyRateLimiterStore(
100+ "./data/ratelimiter/http", // Path to store rate limit data
101+ 10, // Rate: 10 requests per second
102+ 30, // Burst: 30 requests
103+ 3*time.Minute, // Expires in 3 minutes
104+ )
105+ if err != nil {
106+ log.Error("Failed to create HTTP rate limiter store, falling back to memory store", "error", err)
107+
108+ // Fallback to memory store if Starskey store creation fails
109+ p.httpServer.Use(middleware.RateLimiterWithConfig(middleware.RateLimiterConfig{
110+ Store: middleware.NewRateLimiterMemoryStoreWithConfig(
111+ middleware.RateLimiterMemoryStoreConfig{
112+ Rate: rate.Limit(10), // 10 requests per second
113+ Burst: 30, // Burst of 30 requests
114+ ExpiresIn: 3 * time.Minute, // Store expiration
115+ },
116+ ),
117+ DenyHandler: func(context echo.Context, identifier string, err error) error {
118+ log.Warn("Rate limit exceeded",
119+ "ip", identifier,
120+ "path", context.Request().URL.Path,
121+ "method", context.Request().Method)
122+ return context.String(http.StatusTooManyRequests, "Too many requests")
123+ },
124+ }))
125+ } else {
126+ // Use the Starskey store for HTTP rate limiting
127+ log.Debug("Using Starskey rate limiter for HTTP server")
128+
129+ // Add custom Starskey rate limiter middleware
130+ p.httpServer.Use(middleware.RateLimiterWithConfig(middleware.RateLimiterConfig{
131+ Store: httpRateLimiter,
132+ IdentifierExtractor: func(ctx echo.Context) (string, error) {
133+ // Use the client's real IP as the identifier
134+ id := ctx.RealIP()
135+ log.Debug("Rate limiting HTTP request", "ip", id)
136+ return id, nil
137+ },
138+ DenyHandler: func(context echo.Context, identifier string, err error) error {
139+ log.Warn("HTTP rate limit exceeded",
140+ "ip", identifier,
141+ "path", context.Request().URL.Path,
142+ "method", context.Request().Method)
143+ return context.String(http.StatusTooManyRequests, "Too many requests")
61144 },
62- ),
63- DenyHandler: func(context echo.Context, identifier string, err error) error {
64- log.Warn("Rate limit exceeded",
65- "ip", identifier,
66- "path", context.Request().URL.Path,
67- "method", context.Request().Method)
68- return context.String(http.StatusTooManyRequests, "Too many requests")
69- },
70- }))
145+ }))
146+
147+ // Add cleanup function to global slice
148+ rateLimiterCleanup = append(rateLimiterCleanup, func() {
149+ if err := httpRateLimiter.Close(); err != nil {
150+ log.Error("Failed to close HTTP rate limiter store", "error", err)
151+ }
152+ })
153+ }
71154
72155 // Add default Echo logger middleware for HTTP server only if enabled in config
73156 if p.config.EnableLogs {
@@ -10,8 +10,13 @@ import (
10 "github.com/labstack/echo/v4"10 "github.com/labstack/echo/v4"
11 "github.com/labstack/echo/v4/middleware"11 "github.com/labstack/echo/v4/middleware"
12 "golang.org/x/time/rate"12 "golang.org/x/time/rate"
13+
14+ "github.com/bnema/gordon/pkg/kv" // Correct import path based on the module name
13 )15 )
14 16
17+// rateLimiterCleanup holds the cleanup functions for rate limiters
18+var rateLimiterCleanup []func()
19+
15 // Create blacklistedIPs set to track IPs for middleware skipping20 // Create blacklistedIPs set to track IPs for middleware skipping
16 type requestContext struct {21 type requestContext struct {
17 blacklisted bool22 blacklisted bool
@@ -22,23 +27,62 @@ func (p *Proxy) setupMiddleware() {
22 // Create blacklist middleware for HTTPS27 // Create blacklist middleware for HTTPS
23 p.httpsServer.Use(p.createBlacklistMiddleware())28 p.httpsServer.Use(p.createBlacklistMiddleware())
24 29
25- // Add rate limiter middleware to prevent spam attacks30+ // Create a new Starskey rate limiter store for HTTPS server
26- p.httpsServer.Use(middleware.RateLimiterWithConfig(middleware.RateLimiterConfig{31+ httpsRateLimiter, err := kv.NewStarskeyRateLimiterStore(
27- Store: middleware.NewRateLimiterMemoryStoreWithConfig(32+ "./data/ratelimiter/https", // Path to store rate limit data
28- middleware.RateLimiterMemoryStoreConfig{33+ 10, // Rate: 10 requests per second
29- Rate: rate.Limit(10), // 10 requests per second34+ 30, // Burst: 30 requests
30- Burst: 30, // Burst of 30 requests35+ 3*time.Minute, // Expires in 3 minutes
31- ExpiresIn: 3 * time.Minute, // Store expiration36+ )
37+ if err != nil {
38+ log.Error("Failed to create HTTPS rate limiter store, falling back to memory store", "error", err)
39+
40+ // Fallback to memory store if Starskey store creation fails
41+ p.httpsServer.Use(middleware.RateLimiterWithConfig(middleware.RateLimiterConfig{
42+ Store: middleware.NewRateLimiterMemoryStoreWithConfig(
43+ middleware.RateLimiterMemoryStoreConfig{
44+ Rate: rate.Limit(10), // 10 requests per second
45+ Burst: 30, // Burst of 30 requests
46+ ExpiresIn: 3 * time.Minute, // Store expiration
47+ },
48+ ),
49+ DenyHandler: func(context echo.Context, identifier string, err error) error {
50+ log.Warn("Rate limit exceeded",
51+ "ip", identifier,
52+ "path", context.Request().URL.Path,
53+ "method", context.Request().Method)
54+ return context.String(http.StatusTooManyRequests, "Too many requests")
55+ },
56+ }))
57+ } else {
58+ // Use the Starskey store for HTTPS rate limiting
59+ log.Debug("Using Starskey rate limiter for HTTPS server")
60+
61+ // Add custom Starskey rate limiter middleware
62+ p.httpsServer.Use(middleware.RateLimiterWithConfig(middleware.RateLimiterConfig{
63+ Store: httpsRateLimiter,
64+ IdentifierExtractor: func(ctx echo.Context) (string, error) {
65+ // Use the client's real IP as the identifier
66+ id := ctx.RealIP()
67+ log.Debug("Rate limiting HTTPS request", "ip", id)
68+ return id, nil
32 },69 },
33- ),70+ DenyHandler: func(context echo.Context, identifier string, err error) error {
34- DenyHandler: func(context echo.Context, identifier string, err error) error {71+ log.Warn("HTTPS rate limit exceeded",
35- log.Warn("Rate limit exceeded",72+ "ip", identifier,
36- "ip", identifier,73+ "path", context.Request().URL.Path,
37- "path", context.Request().URL.Path,74+ "method", context.Request().Method)
38- "method", context.Request().Method)75+ return context.String(http.StatusTooManyRequests, "Too many requests")
39- return context.String(http.StatusTooManyRequests, "Too many requests")76+ },
40- },77+ }))
41- }))78+
79+ // Add cleanup function to global slice
80+ rateLimiterCleanup = append(rateLimiterCleanup, func() {
81+ if err := httpsRateLimiter.Close(); err != nil {
82+ log.Error("Failed to close HTTPS rate limiter store", "error", err)
83+ }
84+ })
85+ }
42 86
43 // Add default Echo logger middleware only if enabled in config87 // Add default Echo logger middleware only if enabled in config
44 if p.config.EnableLogs {88 if p.config.EnableLogs {
@@ -51,23 +95,62 @@ func (p *Proxy) setupMiddleware() {
51 // Create blacklist middleware for HTTP95 // Create blacklist middleware for HTTP
52 p.httpServer.Use(p.createBlacklistMiddleware())96 p.httpServer.Use(p.createBlacklistMiddleware())
53 97
54- // Add rate limiter middleware to prevent spam attacks for HTTP server98+ // Create a new Starskey rate limiter store for HTTP server
55- p.httpServer.Use(middleware.RateLimiterWithConfig(middleware.RateLimiterConfig{99+ httpRateLimiter, err := kv.NewStarskeyRateLimiterStore(
56- Store: middleware.NewRateLimiterMemoryStoreWithConfig(100+ "./data/ratelimiter/http", // Path to store rate limit data
57- middleware.RateLimiterMemoryStoreConfig{101+ 10, // Rate: 10 requests per second
58- Rate: rate.Limit(10), // 10 requests per second102+ 30, // Burst: 30 requests
59- Burst: 30, // Burst of 30 requests103+ 3*time.Minute, // Expires in 3 minutes
60- ExpiresIn: 3 * time.Minute, // Store expiration104+ )
105+ if err != nil {
106+ log.Error("Failed to create HTTP rate limiter store, falling back to memory store", "error", err)
107+
108+ // Fallback to memory store if Starskey store creation fails
109+ p.httpServer.Use(middleware.RateLimiterWithConfig(middleware.RateLimiterConfig{
110+ Store: middleware.NewRateLimiterMemoryStoreWithConfig(
111+ middleware.RateLimiterMemoryStoreConfig{
112+ Rate: rate.Limit(10), // 10 requests per second
113+ Burst: 30, // Burst of 30 requests
114+ ExpiresIn: 3 * time.Minute, // Store expiration
115+ },
116+ ),
117+ DenyHandler: func(context echo.Context, identifier string, err error) error {
118+ log.Warn("Rate limit exceeded",
119+ "ip", identifier,
120+ "path", context.Request().URL.Path,
121+ "method", context.Request().Method)
122+ return context.String(http.StatusTooManyRequests, "Too many requests")
123+ },
124+ }))
125+ } else {
126+ // Use the Starskey store for HTTP rate limiting
127+ log.Debug("Using Starskey rate limiter for HTTP server")
128+
129+ // Add custom Starskey rate limiter middleware
130+ p.httpServer.Use(middleware.RateLimiterWithConfig(middleware.RateLimiterConfig{
131+ Store: httpRateLimiter,
132+ IdentifierExtractor: func(ctx echo.Context) (string, error) {
133+ // Use the client's real IP as the identifier
134+ id := ctx.RealIP()
135+ log.Debug("Rate limiting HTTP request", "ip", id)
136+ return id, nil
137+ },
138+ DenyHandler: func(context echo.Context, identifier string, err error) error {
139+ log.Warn("HTTP rate limit exceeded",
140+ "ip", identifier,
141+ "path", context.Request().URL.Path,
142+ "method", context.Request().Method)
143+ return context.String(http.StatusTooManyRequests, "Too many requests")
61 },144 },
62- ),145+ }))
63- DenyHandler: func(context echo.Context, identifier string, err error) error {146+
64- log.Warn("Rate limit exceeded",147+ // Add cleanup function to global slice
65- "ip", identifier,148+ rateLimiterCleanup = append(rateLimiterCleanup, func() {
66- "path", context.Request().URL.Path,149+ if err := httpRateLimiter.Close(); err != nil {
67- "method", context.Request().Method)150+ log.Error("Failed to close HTTP rate limiter store", "error", err)
68- return context.String(http.StatusTooManyRequests, "Too many requests")151+ }
69- },152+ })
70- }))153+ }
71 154
72 // Add default Echo logger middleware for HTTP server only if enabled in config155 // Add default Echo logger middleware for HTTP server only if enabled in config
73 if p.config.EnableLogs {156 if p.config.EnableLogs {
M internal/proxy/routes.go +21 -3
@@ -217,7 +217,16 @@ func (p *Proxy) configureRoutes() {
217217
218218 // Create the target URL
219219 targetURL := &url.URL{
220- Scheme: route.Protocol,
220+ Scheme: "http", // Always use HTTP for internal connections
221+ }
222+
223+ // Log protocol conversion if needed
224+ if route.Protocol == "https" {
225+ log.Debug("Converting HTTPS external route to HTTP for internal connection",
226+ "domain", host,
227+ "containerID", route.ContainerID,
228+ "externalProtocol", "https",
229+ "internalProtocol", "http")
221230 }
222231
223232 // Format host properly for both IPv4 and IPv6
@@ -235,7 +244,12 @@ func (p *Proxy) configureRoutes() {
235244 originalDirector := proxy.Director
236245 proxy.Director = func(req *http.Request) {
237246 originalDirector(req)
238- req.Header.Set("X-Forwarded-Proto", "https")
247+ // Use the original protocol for forwarded proto header
248+ forwardedProto := "http"
249+ if route.Protocol == "https" {
250+ forwardedProto = "https"
251+ }
252+ req.Header.Set("X-Forwarded-Proto", forwardedProto)
239253 req.Header.Set("X-Forwarded-Host", host)
240254 req.Header.Set("X-Forwarded-For", c.RealIP())
241255 req.Header.Set("X-Real-IP", c.RealIP())
@@ -245,7 +259,8 @@ func (p *Proxy) configureRoutes() {
245259 "host", host,
246260 "target", targetURL.String(),
247261 "path", req.URL.Path,
248- "clientIP", c.RealIP())
262+ "clientIP", c.RealIP(),
263+ "originalProtocol", route.Protocol)
249264 }
250265
251266 // Add error handling
@@ -274,6 +289,9 @@ func (p *Proxy) configureRoutes() {
274289 }
275290
276291 // AddRoute adds a new route to the database and reloads the proxy
292+// The protocol parameter specifies the external protocol (http or https) for client connections.
293+// Note: Regardless of the external protocol, all internal connections to containers use HTTP.
294+// The HTTPS protocol is only used for external connections and certificates.
277295 func (p *Proxy) AddRoute(domainName, containerID, containerIP, containerPort, protocol, path string) error {
278296 // Begin a transaction
279297 tx, err := p.app.GetDB().Begin()
@@ -217,7 +217,16 @@ func (p *Proxy) configureRoutes() {
217 217
218 // Create the target URL218 // Create the target URL
219 targetURL := &url.URL{219 targetURL := &url.URL{
220- Scheme: route.Protocol,220+ Scheme: "http", // Always use HTTP for internal connections
221+ }
222+
223+ // Log protocol conversion if needed
224+ if route.Protocol == "https" {
225+ log.Debug("Converting HTTPS external route to HTTP for internal connection",
226+ "domain", host,
227+ "containerID", route.ContainerID,
228+ "externalProtocol", "https",
229+ "internalProtocol", "http")
221 }230 }
222 231
223 // Format host properly for both IPv4 and IPv6232 // Format host properly for both IPv4 and IPv6
@@ -235,7 +244,12 @@ func (p *Proxy) configureRoutes() {
235 originalDirector := proxy.Director244 originalDirector := proxy.Director
236 proxy.Director = func(req *http.Request) {245 proxy.Director = func(req *http.Request) {
237 originalDirector(req)246 originalDirector(req)
238- req.Header.Set("X-Forwarded-Proto", "https")247+ // Use the original protocol for forwarded proto header
248+ forwardedProto := "http"
249+ if route.Protocol == "https" {
250+ forwardedProto = "https"
251+ }
252+ req.Header.Set("X-Forwarded-Proto", forwardedProto)
239 req.Header.Set("X-Forwarded-Host", host)253 req.Header.Set("X-Forwarded-Host", host)
240 req.Header.Set("X-Forwarded-For", c.RealIP())254 req.Header.Set("X-Forwarded-For", c.RealIP())
241 req.Header.Set("X-Real-IP", c.RealIP())255 req.Header.Set("X-Real-IP", c.RealIP())
@@ -245,7 +259,8 @@ func (p *Proxy) configureRoutes() {
245 "host", host,259 "host", host,
246 "target", targetURL.String(),260 "target", targetURL.String(),
247 "path", req.URL.Path,261 "path", req.URL.Path,
248- "clientIP", c.RealIP())262+ "clientIP", c.RealIP(),
263+ "originalProtocol", route.Protocol)
249 }264 }
250 265
251 // Add error handling266 // Add error handling
@@ -274,6 +289,9 @@ func (p *Proxy) configureRoutes() {
274 }289 }
275 290
276 // AddRoute adds a new route to the database and reloads the proxy291 // AddRoute adds a new route to the database and reloads the proxy
292+// The protocol parameter specifies the external protocol (http or https) for client connections.
293+// Note: Regardless of the external protocol, all internal connections to containers use HTTP.
294+// The HTTPS protocol is only used for external connections and certificates.
277 func (p *Proxy) AddRoute(domainName, containerID, containerIP, containerPort, protocol, path string) error {295 func (p *Proxy) AddRoute(domainName, containerID, containerIP, containerPort, protocol, path string) error {
278 // Begin a transaction296 // Begin a transaction
279 tx, err := p.app.GetDB().Begin()297 tx, err := p.app.GetDB().Begin()
M internal/proxy/server.go +5 -0
@@ -214,6 +214,11 @@ func (p *Proxy) Start() error {
214214 func (p *Proxy) Stop() error {
215215 log.Info("Stopping reverse proxy")
216216
217+ // Run any rate limiter cleanup functions
218+ for _, cleanup := range rateLimiterCleanup {
219+ cleanup()
220+ }
221+
217222 // Stop the container event listener
218223 docker.StopContainerEventListener()
219224
@@ -214,6 +214,11 @@ func (p *Proxy) Start() error {
214 func (p *Proxy) Stop() error {214 func (p *Proxy) Stop() error {
215 log.Info("Stopping reverse proxy")215 log.Info("Stopping reverse proxy")
216 216
217+ // Run any rate limiter cleanup functions
218+ for _, cleanup := range rateLimiterCleanup {
219+ cleanup()
220+ }
221+
217 // Stop the container event listener222 // Stop the container event listener
218 docker.StopContainerEventListener()223 docker.StopContainerEventListener()
219 224
M pkg/docker/containers.go +117 -59
@@ -750,83 +750,141 @@ func ListenForContainerEvents(networkFilter string, callback func(string, string
750750 // Store cancel func to allow shutdown
751751 containerEventCancel = cancel
752752
753- // Set up filters for the events we're interested in
754- filters := makeEventFilters(networkFilter)
755-
756- // Start listening for events
757- messages, errs := dockerCli.Events(ctx, types.EventsOptions{
758- Filters: filters,
759- })
760-
761- // Process events in a goroutine
753+ // Start the main listener loop in a goroutine
762754 go func() {
755+ // Implement exponential backoff for reconnection
756+ initialDelay := 1 * time.Second
757+ maxDelay := 1 * time.Minute
758+ currentDelay := initialDelay
759+ consecutiveFailures := 0
760+
763761 for {
764- select {
765- case err := <-errs:
766- if err != nil && ctx.Err() == nil {
767- // Only log if this isn't due to context cancellation
768- log.Error("Error in container event stream", "error", err)
769- }
762+ // Check if context was canceled (application is shutting down)
763+ if ctx.Err() != nil {
770764 return
771- case event := <-messages:
772- // Process container events
773- if event.Type == "container" {
774- containerID := event.Actor.ID
775-
776- // Get container name from event attributes
777- containerName := event.Actor.Attributes["name"]
778- if containerName == "" {
779- // If not in attributes, try to get it from the API
780- name, err := GetContainerName(containerID)
781- if err == nil {
782- containerName = strings.TrimPrefix(name, "/")
765+ }
766+
767+ // Set up filters for the events we're interested in
768+ filters := makeEventFilters(networkFilter)
769+
770+ // Start listening for events
771+ messages, errs := dockerCli.Events(ctx, types.EventsOptions{
772+ Filters: filters,
773+ })
774+
775+ log.Info("Started container event listener", "network_filter", networkFilter)
776+
777+ // Process events in an inner loop
778+ listenSuccess := false
779+ eventStreamLoop:
780+ for {
781+ select {
782+ case err := <-errs:
783+ if err != nil && ctx.Err() == nil {
784+ // Only log if this isn't due to context cancellation
785+ log.Error("Error in container event stream", "error", err)
786+ // Increment failure counter for backoff calculation
787+ consecutiveFailures++
788+
789+ // Calculate backoff delay with exponential increase
790+ if listenSuccess {
791+ // If we had successful events before failure, reset the delay
792+ currentDelay = initialDelay
793+ } else {
794+ // Exponential backoff: double the delay each time, up to max
795+ currentDelay = time.Duration(float64(currentDelay) * 1.5)
796+ if currentDelay > maxDelay {
797+ currentDelay = maxDelay
798+ }
783799 }
800+
801+ log.Debug("Will attempt to reconnect to container event stream",
802+ "delay", currentDelay,
803+ "consecutive_failures", consecutiveFailures)
804+
805+ // Break out of the inner event loop to reconnect
806+ break eventStreamLoop
807+ }
808+ // Context was canceled, exit completely
809+ if ctx.Err() != nil {
810+ return
811+ }
812+ case event := <-messages:
813+ // Mark that we've successfully received events
814+ if !listenSuccess {
815+ listenSuccess = true
816+ // Reset failure counter on successful event
817+ consecutiveFailures = 0
818+ log.Info("Successfully receiving container events")
784819 }
785820
786- // Handle container events based on action
787- switch event.Action {
788- case "start":
789- // Container started - need to get its IP and update any proxy routes
790- log.Debug("Container started event",
791- "container_id", containerID,
792- "container_name", containerName)
793-
794- // Get container IP from the network
795- containerIP, err := getContainerIPFromNetwork(containerID, networkFilter)
796- if err != nil {
797- log.Error("Failed to get container IP",
798- "container_id", containerID,
799- "error", err)
800- continue
821+ // Process container events
822+ if event.Type == "container" {
823+ containerID := event.Actor.ID
824+
825+ // Get container name from event attributes
826+ containerName := event.Actor.Attributes["name"]
827+ if containerName == "" {
828+ // If not in attributes, try to get it from the API
829+ name, err := GetContainerName(containerID)
830+ if err == nil {
831+ containerName = strings.TrimPrefix(name, "/")
832+ }
801833 }
802834
803- // Call the callback with the container ID, name, and IP
804- callback(containerID, containerName, containerIP)
805-
806- case "die", "kill", "destroy":
807- // Container stopped - might need to mark routes as inactive
808- log.Debug("Container stopped event",
809- "container_id", containerID,
810- "container_name", containerName,
811- "action", event.Action)
835+ // Handle container events based on action
836+ switch event.Action {
837+ case "start":
838+ // Container started - need to get its IP and update any proxy routes
839+ log.Debug("Container started event",
840+ "container_id", containerID,
841+ "container_name", containerName)
842+
843+ // Get container IP from the network
844+ containerIP, err := getContainerIPFromNetwork(containerID, networkFilter)
845+ if err != nil {
846+ log.Error("Failed to get container IP",
847+ "container_id", containerID,
848+ "error", err)
849+ continue
850+ }
851+
852+ // Call the callback with the container ID, name, and IP
853+ callback(containerID, containerName, containerIP)
854+
855+ case "die", "kill", "destroy":
856+ // Container stopped - might need to mark routes as inactive
857+ log.Debug("Container stopped event",
858+ "container_id", containerID,
859+ "container_name", containerName,
860+ "action", event.Action)
812861
813- case "rename":
814- // Container renamed - update related proxy routes
815- log.Debug("Container renamed event",
816- "container_id", containerID,
817- "container_name", containerName,
818- "new_name", event.Actor.Attributes["name"])
862+ case "rename":
863+ // Container renamed - update related proxy routes
864+ log.Debug("Container renamed event",
865+ "container_id", containerID,
866+ "container_name", containerName,
867+ "new_name", event.Actor.Attributes["name"])
819868
820- // Add other event types as needed
869+ // Add other event types as needed
870+ }
821871 }
872+ case <-ctx.Done():
873+ return
822874 }
875+ }
876+
877+ // If we get here, the event stream failed and we need to reconnect
878+ // Wait for the calculated delay before reconnecting
879+ select {
880+ case <-time.After(currentDelay):
881+ log.Info("Attempting to reconnect to container event stream")
823882 case <-ctx.Done():
824883 return
825884 }
826885 }
827886 }()
828887
829- log.Info("Started container event listener", "network_filter", networkFilter)
830888 return nil
831889 }
832890
@@ -750,83 +750,141 @@ func ListenForContainerEvents(networkFilter string, callback func(string, string
750 // Store cancel func to allow shutdown750 // Store cancel func to allow shutdown
751 containerEventCancel = cancel751 containerEventCancel = cancel
752 752
753- // Set up filters for the events we're interested in753+ // Start the main listener loop in a goroutine
754- filters := makeEventFilters(networkFilter)
755-
756- // Start listening for events
757- messages, errs := dockerCli.Events(ctx, types.EventsOptions{
758- Filters: filters,
759- })
760-
761- // Process events in a goroutine
762 go func() {754 go func() {
755+ // Implement exponential backoff for reconnection
756+ initialDelay := 1 * time.Second
757+ maxDelay := 1 * time.Minute
758+ currentDelay := initialDelay
759+ consecutiveFailures := 0
760+
763 for {761 for {
764- select {762+ // Check if context was canceled (application is shutting down)
765- case err := <-errs:763+ if ctx.Err() != nil {
766- if err != nil && ctx.Err() == nil {
767- // Only log if this isn't due to context cancellation
768- log.Error("Error in container event stream", "error", err)
769- }
770 return764 return
771- case event := <-messages:765+ }
772- // Process container events766+
773- if event.Type == "container" {767+ // Set up filters for the events we're interested in
774- containerID := event.Actor.ID768+ filters := makeEventFilters(networkFilter)
775-769+
776- // Get container name from event attributes770+ // Start listening for events
777- containerName := event.Actor.Attributes["name"]771+ messages, errs := dockerCli.Events(ctx, types.EventsOptions{
778- if containerName == "" {772+ Filters: filters,
779- // If not in attributes, try to get it from the API773+ })
780- name, err := GetContainerName(containerID)774+
781- if err == nil {775+ log.Info("Started container event listener", "network_filter", networkFilter)
782- containerName = strings.TrimPrefix(name, "/")776+
777+ // Process events in an inner loop
778+ listenSuccess := false
779+ eventStreamLoop:
780+ for {
781+ select {
782+ case err := <-errs:
783+ if err != nil && ctx.Err() == nil {
784+ // Only log if this isn't due to context cancellation
785+ log.Error("Error in container event stream", "error", err)
786+ // Increment failure counter for backoff calculation
787+ consecutiveFailures++
788+
789+ // Calculate backoff delay with exponential increase
790+ if listenSuccess {
791+ // If we had successful events before failure, reset the delay
792+ currentDelay = initialDelay
793+ } else {
794+ // Exponential backoff: double the delay each time, up to max
795+ currentDelay = time.Duration(float64(currentDelay) * 1.5)
796+ if currentDelay > maxDelay {
797+ currentDelay = maxDelay
798+ }
783 }799 }
800+
801+ log.Debug("Will attempt to reconnect to container event stream",
802+ "delay", currentDelay,
803+ "consecutive_failures", consecutiveFailures)
804+
805+ // Break out of the inner event loop to reconnect
806+ break eventStreamLoop
807+ }
808+ // Context was canceled, exit completely
809+ if ctx.Err() != nil {
810+ return
811+ }
812+ case event := <-messages:
813+ // Mark that we've successfully received events
814+ if !listenSuccess {
815+ listenSuccess = true
816+ // Reset failure counter on successful event
817+ consecutiveFailures = 0
818+ log.Info("Successfully receiving container events")
784 }819 }
785 820
786- // Handle container events based on action821+ // Process container events
787- switch event.Action {822+ if event.Type == "container" {
788- case "start":823+ containerID := event.Actor.ID
789- // Container started - need to get its IP and update any proxy routes824+
790- log.Debug("Container started event",825+ // Get container name from event attributes
791- "container_id", containerID,826+ containerName := event.Actor.Attributes["name"]
792- "container_name", containerName)827+ if containerName == "" {
793-828+ // If not in attributes, try to get it from the API
794- // Get container IP from the network829+ name, err := GetContainerName(containerID)
795- containerIP, err := getContainerIPFromNetwork(containerID, networkFilter)830+ if err == nil {
796- if err != nil {831+ containerName = strings.TrimPrefix(name, "/")
797- log.Error("Failed to get container IP",832+ }
798- "container_id", containerID,
799- "error", err)
800- continue
801 }833 }
802 834
803- // Call the callback with the container ID, name, and IP835+ // Handle container events based on action
804- callback(containerID, containerName, containerIP)836+ switch event.Action {
805-837+ case "start":
806- case "die", "kill", "destroy":838+ // Container started - need to get its IP and update any proxy routes
807- // Container stopped - might need to mark routes as inactive839+ log.Debug("Container started event",
808- log.Debug("Container stopped event",840+ "container_id", containerID,
809- "container_id", containerID,841+ "container_name", containerName)
810- "container_name", containerName,842+
811- "action", event.Action)843+ // Get container IP from the network
844+ containerIP, err := getContainerIPFromNetwork(containerID, networkFilter)
845+ if err != nil {
846+ log.Error("Failed to get container IP",
847+ "container_id", containerID,
848+ "error", err)
849+ continue
850+ }
851+
852+ // Call the callback with the container ID, name, and IP
853+ callback(containerID, containerName, containerIP)
854+
855+ case "die", "kill", "destroy":
856+ // Container stopped - might need to mark routes as inactive
857+ log.Debug("Container stopped event",
858+ "container_id", containerID,
859+ "container_name", containerName,
860+ "action", event.Action)
812 861
813- case "rename":862+ case "rename":
814- // Container renamed - update related proxy routes863+ // Container renamed - update related proxy routes
815- log.Debug("Container renamed event",864+ log.Debug("Container renamed event",
816- "container_id", containerID,865+ "container_id", containerID,
817- "container_name", containerName,866+ "container_name", containerName,
818- "new_name", event.Actor.Attributes["name"])867+ "new_name", event.Actor.Attributes["name"])
819 868
820- // Add other event types as needed869+ // Add other event types as needed
870+ }
821 }871 }
872+ case <-ctx.Done():
873+ return
822 }874 }
875+ }
876+
877+ // If we get here, the event stream failed and we need to reconnect
878+ // Wait for the calculated delay before reconnecting
879+ select {
880+ case <-time.After(currentDelay):
881+ log.Info("Attempting to reconnect to container event stream")
823 case <-ctx.Done():882 case <-ctx.Done():
824 return883 return
825 }884 }
826 }885 }
827 }()886 }()
828 887
829- log.Info("Started container event listener", "network_filter", networkFilter)
830 return nil888 return nil
831 }889 }
832 890
R pkg/store/store_images.go → pkg/filestore/container_images.go +2 -1 renamed, 95% similar
@@ -1,4 +1,4 @@
1-package store
1+package filestore
22
33 import (
44 "fmt"
@@ -11,6 +11,7 @@ import (
1111 "github.com/google/uuid"
1212 )
1313
14+// SaveImageToStorage saves a container image to the file system
1415 func SaveImageToStorage(config *common.Config, originalFilename string, buf io.Reader) (string, error) {
1516 // Ensure the images directory exists
1617 imagesDir := filepath.Join(config.General.StorageDir, "images")
@@ -1,4 +1,4 @@
1-package store1+package filestore
2 2
3 import (3 import (
4 "fmt"4 "fmt"
@@ -11,6 +11,7 @@ import (
11 "github.com/google/uuid"11 "github.com/google/uuid"
12 )12 )
13 13
14+// SaveImageToStorage saves a container image to the file system
14 func SaveImageToStorage(config *common.Config, originalFilename string, buf io.Reader) (string, error) {15 func SaveImageToStorage(config *common.Config, originalFilename string, buf io.Reader) (string, error) {
15 // Ensure the images directory exists16 // Ensure the images directory exists
16 imagesDir := filepath.Join(config.General.StorageDir, "images")17 imagesDir := filepath.Join(config.General.StorageDir, "images")
A pkg/kv/ratelimiter.go +191 -0
@@ -0,0 +1,191 @@
1+package kv
2+
3+import (
4+ "encoding/json"
5+ "time"
6+
7+ "github.com/charmbracelet/log"
8+ "github.com/starskey-io/starskey"
9+)
10+
11+type RateLimitInfo struct {
12+ Attempts int `json:"attempts"`
13+ ResetTime time.Time `json:"reset_time"`
14+ Jailed bool `json:"jailed"`
15+}
16+
17+type StarskeyRateLimiterStore struct {
18+ db *starskey.Starskey
19+ rate float64
20+ burst int
21+ expiresIn time.Duration
22+}
23+
24+// NewStarskeyRateLimiterStore creates a new rate limiter store backed by Starskey
25+func NewStarskeyRateLimiterStore(dbPath string, rate float64, burst int, expiresIn time.Duration) (*StarskeyRateLimiterStore, error) {
26+ // Configure and open Starskey
27+ db, err := starskey.Open(&starskey.Config{
28+ Permission: 0755,
29+ Directory: dbPath,
30+ FlushThreshold: 64 * 1024 * 1024, // 64MB - adjust based on expected traffic
31+ MaxLevel: 3,
32+ SizeFactor: 10,
33+ BloomFilter: true, // Enable for faster lookups
34+ SuRF: false, // No need for range queries in rate limiting
35+ Logging: true,
36+ Compression: true,
37+ CompressionOption: starskey.SnappyCompression,
38+ })
39+
40+ if err != nil {
41+ return nil, err
42+ }
43+
44+ log.Info("Initialized rate limiter with Starskey backend",
45+ "path", dbPath,
46+ "rate", rate,
47+ "burst", burst,
48+ "expiration", expiresIn)
49+
50+ return &StarskeyRateLimiterStore{
51+ db: db,
52+ rate: rate,
53+ burst: burst,
54+ expiresIn: expiresIn,
55+ }, nil
56+}
57+
58+// Implement Echo's RateLimiterStore interface
59+func (s *StarskeyRateLimiterStore) Allow(identifier string) (bool, error) {
60+ var allowed bool
61+
62+ err := s.db.Update(func(txn *starskey.Txn) error {
63+ now := time.Now()
64+ key := []byte(identifier)
65+
66+ // Default state for new entries
67+ info := RateLimitInfo{
68+ Attempts: 0,
69+ ResetTime: now,
70+ Jailed: false,
71+ }
72+
73+ // Get existing record if any
74+ value, err := txn.Get(key)
75+ if err == nil && value != nil {
76+ if err := json.Unmarshal(value, &info); err != nil {
77+ // Handle corrupted data
78+ info = RateLimitInfo{Attempts: 0, ResetTime: now}
79+ }
80+
81+ // Check if jailed
82+ if info.Jailed && now.Before(info.ResetTime) {
83+ allowed = false
84+ return nil
85+ }
86+
87+ // Calculate time-based allowance
88+ timePassed := now.Sub(info.ResetTime).Seconds()
89+ tokensToAdd := timePassed * s.rate
90+
91+ // Reset if needed
92+ if now.After(info.ResetTime.Add(s.expiresIn)) {
93+ info.Attempts = 0
94+ info.ResetTime = now
95+ } else {
96+ info.Attempts = max(0, info.Attempts-int(tokensToAdd))
97+ }
98+ }
99+
100+ // Check if under rate limit
101+ if info.Attempts < s.burst {
102+ info.Attempts++
103+ allowed = true
104+
105+ // Jail if needed (e.g., if burst exceeded)
106+ if info.Attempts >= s.burst {
107+ info.Jailed = true
108+ info.ResetTime = now.Add(time.Second) // 1 second penalty
109+ log.Debug("IP jailed (burst exceeded)", "ip", identifier, "reset_at", info.ResetTime)
110+ } else {
111+ log.Debug("Request allowed", "ip", identifier, "attempts", info.Attempts)
112+ }
113+
114+ // Save updated info
115+ data, err := json.Marshal(info)
116+ if err != nil {
117+ log.Debug("Failed to marshal rate limit info", "ip", identifier, "error", err)
118+ return err
119+ }
120+
121+ // Fix: txn.Put doesn't return a value in Starskey
122+ // The errors are handled at the transaction level
123+ txn.Put(key, data)
124+ log.Debug("Rate limit info saved", "ip", identifier)
125+ return nil
126+ }
127+
128+ log.Debug("Request blocked (rate limited)", "ip", identifier, "attempts", info.Attempts)
129+ allowed = false
130+ return nil
131+ })
132+
133+ return allowed, err
134+}
135+
136+func (s *StarskeyRateLimiterStore) Reset(identifier string) (bool, error) {
137+ err := s.db.Delete([]byte(identifier))
138+ return err == nil, err
139+}
140+
141+// GetJailedIPs returns all currently jailed IPs
142+func (s *StarskeyRateLimiterStore) GetJailedIPs() (map[string]time.Time, error) {
143+ jailed := make(map[string]time.Time)
144+ now := time.Now()
145+
146+ // Use the FilterKeys method since Starskey doesn't have a direct iterator
147+ compareFunc := func(key []byte) bool {
148+ return true // We'll examine all keys and filter ourselves
149+ }
150+
151+ results, err := s.db.FilterKeys(compareFunc)
152+ if err != nil {
153+ return nil, err
154+ }
155+
156+ // Process results - each pair of entries is key, value
157+ for i := 0; i < len(results); i += 2 {
158+ if i+1 >= len(results) {
159+ break // Safety check
160+ }
161+
162+ key := string(results[i])
163+ value := results[i+1]
164+
165+ var info RateLimitInfo
166+ if err := json.Unmarshal(value, &info); err != nil {
167+ log.Debug("Failed to unmarshal rate limit info", "key", key, "error", err)
168+ continue // Skip invalid entries
169+ }
170+
171+ // Add jailed IPs to the result map
172+ if info.Jailed && now.Before(info.ResetTime) {
173+ jailed[key] = info.ResetTime
174+ }
175+ }
176+
177+ return jailed, nil
178+}
179+
180+// Close closes the underlying Starskey database
181+func (s *StarskeyRateLimiterStore) Close() error {
182+ log.Debug("Closing rate limiter store")
183+ return s.db.Close()
184+}
185+
186+func max(a, b int) int {
187+ if a > b {
188+ return a
189+ }
190+ return b
191+}
@@ -0,0 +1,191 @@
1+package kv
2+
3+import (
4+ "encoding/json"
5+ "time"
6+
7+ "github.com/charmbracelet/log"
8+ "github.com/starskey-io/starskey"
9+)
10+
11+type RateLimitInfo struct {
12+ Attempts int `json:"attempts"`
13+ ResetTime time.Time `json:"reset_time"`
14+ Jailed bool `json:"jailed"`
15+}
16+
17+type StarskeyRateLimiterStore struct {
18+ db *starskey.Starskey
19+ rate float64
20+ burst int
21+ expiresIn time.Duration
22+}
23+
24+// NewStarskeyRateLimiterStore creates a new rate limiter store backed by Starskey
25+func NewStarskeyRateLimiterStore(dbPath string, rate float64, burst int, expiresIn time.Duration) (*StarskeyRateLimiterStore, error) {
26+ // Configure and open Starskey
27+ db, err := starskey.Open(&starskey.Config{
28+ Permission: 0755,
29+ Directory: dbPath,
30+ FlushThreshold: 64 * 1024 * 1024, // 64MB - adjust based on expected traffic
31+ MaxLevel: 3,
32+ SizeFactor: 10,
33+ BloomFilter: true, // Enable for faster lookups
34+ SuRF: false, // No need for range queries in rate limiting
35+ Logging: true,
36+ Compression: true,
37+ CompressionOption: starskey.SnappyCompression,
38+ })
39+
40+ if err != nil {
41+ return nil, err
42+ }
43+
44+ log.Info("Initialized rate limiter with Starskey backend",
45+ "path", dbPath,
46+ "rate", rate,
47+ "burst", burst,
48+ "expiration", expiresIn)
49+
50+ return &StarskeyRateLimiterStore{
51+ db: db,
52+ rate: rate,
53+ burst: burst,
54+ expiresIn: expiresIn,
55+ }, nil
56+}
57+
58+// Implement Echo's RateLimiterStore interface
59+func (s *StarskeyRateLimiterStore) Allow(identifier string) (bool, error) {
60+ var allowed bool
61+
62+ err := s.db.Update(func(txn *starskey.Txn) error {
63+ now := time.Now()
64+ key := []byte(identifier)
65+
66+ // Default state for new entries
67+ info := RateLimitInfo{
68+ Attempts: 0,
69+ ResetTime: now,
70+ Jailed: false,
71+ }
72+
73+ // Get existing record if any
74+ value, err := txn.Get(key)
75+ if err == nil && value != nil {
76+ if err := json.Unmarshal(value, &info); err != nil {
77+ // Handle corrupted data
78+ info = RateLimitInfo{Attempts: 0, ResetTime: now}
79+ }
80+
81+ // Check if jailed
82+ if info.Jailed && now.Before(info.ResetTime) {
83+ allowed = false
84+ return nil
85+ }
86+
87+ // Calculate time-based allowance
88+ timePassed := now.Sub(info.ResetTime).Seconds()
89+ tokensToAdd := timePassed * s.rate
90+
91+ // Reset if needed
92+ if now.After(info.ResetTime.Add(s.expiresIn)) {
93+ info.Attempts = 0
94+ info.ResetTime = now
95+ } else {
96+ info.Attempts = max(0, info.Attempts-int(tokensToAdd))
97+ }
98+ }
99+
100+ // Check if under rate limit
101+ if info.Attempts < s.burst {
102+ info.Attempts++
103+ allowed = true
104+
105+ // Jail if needed (e.g., if burst exceeded)
106+ if info.Attempts >= s.burst {
107+ info.Jailed = true
108+ info.ResetTime = now.Add(time.Second) // 1 second penalty
109+ log.Debug("IP jailed (burst exceeded)", "ip", identifier, "reset_at", info.ResetTime)
110+ } else {
111+ log.Debug("Request allowed", "ip", identifier, "attempts", info.Attempts)
112+ }
113+
114+ // Save updated info
115+ data, err := json.Marshal(info)
116+ if err != nil {
117+ log.Debug("Failed to marshal rate limit info", "ip", identifier, "error", err)
118+ return err
119+ }
120+
121+ // Fix: txn.Put doesn't return a value in Starskey
122+ // The errors are handled at the transaction level
123+ txn.Put(key, data)
124+ log.Debug("Rate limit info saved", "ip", identifier)
125+ return nil
126+ }
127+
128+ log.Debug("Request blocked (rate limited)", "ip", identifier, "attempts", info.Attempts)
129+ allowed = false
130+ return nil
131+ })
132+
133+ return allowed, err
134+}
135+
136+func (s *StarskeyRateLimiterStore) Reset(identifier string) (bool, error) {
137+ err := s.db.Delete([]byte(identifier))
138+ return err == nil, err
139+}
140+
141+// GetJailedIPs returns all currently jailed IPs
142+func (s *StarskeyRateLimiterStore) GetJailedIPs() (map[string]time.Time, error) {
143+ jailed := make(map[string]time.Time)
144+ now := time.Now()
145+
146+ // Use the FilterKeys method since Starskey doesn't have a direct iterator
147+ compareFunc := func(key []byte) bool {
148+ return true // We'll examine all keys and filter ourselves
149+ }
150+
151+ results, err := s.db.FilterKeys(compareFunc)
152+ if err != nil {
153+ return nil, err
154+ }
155+
156+ // Process results - each pair of entries is key, value
157+ for i := 0; i < len(results); i += 2 {
158+ if i+1 >= len(results) {
159+ break // Safety check
160+ }
161+
162+ key := string(results[i])
163+ value := results[i+1]
164+
165+ var info RateLimitInfo
166+ if err := json.Unmarshal(value, &info); err != nil {
167+ log.Debug("Failed to unmarshal rate limit info", "key", key, "error", err)
168+ continue // Skip invalid entries
169+ }
170+
171+ // Add jailed IPs to the result map
172+ if info.Jailed && now.Before(info.ResetTime) {
173+ jailed[key] = info.ResetTime
174+ }
175+ }
176+
177+ return jailed, nil
178+}
179+
180+// Close closes the underlying Starskey database
181+func (s *StarskeyRateLimiterStore) Close() error {
182+ log.Debug("Closing rate limiter store")
183+ return s.db.Close()
184+}
185+
186+func max(a, b int) int {
187+ if a > b {
188+ return a
189+ }
190+ return b
191+}

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 7

Tags 106

↑↓ moveenter openesc close