detached da25f67

feat: Enhance configuration defaults and certificate handling

- Implement checks for explicit configuration of EnableHttpLogs in config.yml.
- Update default behavior to set EnableHttpLogs to true only if not explicitly defined.
- Improve logging for applied default values and explicit configurations.
- Refactor certificate handling to ensure fallback mechanisms are robust and well-logged.
- Introduce methods for preloading certificates and handling rate limits effectively.
commit
da25f67eb0aed8be9a392939e31d80fe4cc8cee5
author
bamen · · 2025-03-25 08:17 UTC
parent
2f01687
signature
unsigned
7 files · +1160 -408
Unified
UnifiedSplit
M internal/common/init_config.go +24 -3
@@ -202,8 +202,29 @@ func applyDefaultsToConfig(config *Config) bool {
202202 logger.Debug("Applied default value for ReverseProxy.GracePeriod", "value", gracePeriod)
203203 defaultsApplied = true
204204 }
205- // Set EnableHttpLogs to true by default if not specified
206- if !config.ReverseProxy.EnableHttpLogs {
205+ // Check if "enableHttpLogs" is explicitly mentioned in the config file
206+ configFilePath := filepath.Join(getConfigDirMustExist(), "config.yml")
207+ if fileExists(configFilePath) {
208+ yamlContent, err := os.ReadFile(configFilePath)
209+ if err == nil {
210+ // Only set default if enableHttpLogs not explicitly specified
211+ if !strings.Contains(string(yamlContent), "enableHttpLogs:") {
212+ config.ReverseProxy.EnableHttpLogs = true
213+ logger.Debug("Applied default value for ReverseProxy.EnableHttpLogs", "value", true)
214+ defaultsApplied = true
215+ } else {
216+ logger.Debug("Keeping explicit value for ReverseProxy.EnableHttpLogs", "value", config.ReverseProxy.EnableHttpLogs)
217+ }
218+ } else {
219+ // If we can't read the file, apply the default only if not set
220+ if !config.ReverseProxy.EnableHttpLogs {
221+ config.ReverseProxy.EnableHttpLogs = true
222+ logger.Debug("Applied default value for ReverseProxy.EnableHttpLogs", "value", true)
223+ defaultsApplied = true
224+ }
225+ }
226+ } else {
227+ // If config file doesn't exist yet, set the default
207228 config.ReverseProxy.EnableHttpLogs = true
208229 logger.Debug("Applied default value for ReverseProxy.EnableHttpLogs", "value", true)
209230 defaultsApplied = true
@@ -230,7 +251,7 @@ func applyDefaultsToConfig(config *Config) bool {
230251 }
231252
232253 // Handle autoRenew
233- configFilePath := filepath.Join(getConfigDirMustExist(), "config.yml")
254+ configFilePath = filepath.Join(getConfigDirMustExist(), "config.yml")
234255 if fileExists(configFilePath) {
235256 yamlContent, err := os.ReadFile(configFilePath)
236257 if err == nil {
@@ -202,8 +202,29 @@ func applyDefaultsToConfig(config *Config) bool {
202 logger.Debug("Applied default value for ReverseProxy.GracePeriod", "value", gracePeriod)202 logger.Debug("Applied default value for ReverseProxy.GracePeriod", "value", gracePeriod)
203 defaultsApplied = true203 defaultsApplied = true
204 }204 }
205- // Set EnableHttpLogs to true by default if not specified205+ // Check if "enableHttpLogs" is explicitly mentioned in the config file
206- if !config.ReverseProxy.EnableHttpLogs {206+ configFilePath := filepath.Join(getConfigDirMustExist(), "config.yml")
207+ if fileExists(configFilePath) {
208+ yamlContent, err := os.ReadFile(configFilePath)
209+ if err == nil {
210+ // Only set default if enableHttpLogs not explicitly specified
211+ if !strings.Contains(string(yamlContent), "enableHttpLogs:") {
212+ config.ReverseProxy.EnableHttpLogs = true
213+ logger.Debug("Applied default value for ReverseProxy.EnableHttpLogs", "value", true)
214+ defaultsApplied = true
215+ } else {
216+ logger.Debug("Keeping explicit value for ReverseProxy.EnableHttpLogs", "value", config.ReverseProxy.EnableHttpLogs)
217+ }
218+ } else {
219+ // If we can't read the file, apply the default only if not set
220+ if !config.ReverseProxy.EnableHttpLogs {
221+ config.ReverseProxy.EnableHttpLogs = true
222+ logger.Debug("Applied default value for ReverseProxy.EnableHttpLogs", "value", true)
223+ defaultsApplied = true
224+ }
225+ }
226+ } else {
227+ // If config file doesn't exist yet, set the default
207 config.ReverseProxy.EnableHttpLogs = true228 config.ReverseProxy.EnableHttpLogs = true
208 logger.Debug("Applied default value for ReverseProxy.EnableHttpLogs", "value", true)229 logger.Debug("Applied default value for ReverseProxy.EnableHttpLogs", "value", true)
209 defaultsApplied = true230 defaultsApplied = true
@@ -230,7 +251,7 @@ func applyDefaultsToConfig(config *Config) bool {
230 }251 }
231 252
232 // Handle autoRenew253 // Handle autoRenew
233- configFilePath := filepath.Join(getConfigDirMustExist(), "config.yml")254+ configFilePath = filepath.Join(getConfigDirMustExist(), "config.yml")
234 if fileExists(configFilePath) {255 if fileExists(configFilePath) {
235 yamlContent, err := os.ReadFile(configFilePath)256 yamlContent, err := os.ReadFile(configFilePath)
236 if err == nil {257 if err == nil {
M internal/proxy/certificates.go +748 -55
@@ -109,36 +109,40 @@ func (p *Proxy) setupCertManager() {
109109 // First check for upstream TLS-terminating proxies if detection is enabled
110110 isProxied := p.detectUpstreamProxy()
111111
112+ // Get admin domain and root domain
113+ adminDomain := p.app.GetConfig().Http.FullDomain()
114+ rootDomain := p.app.GetConfig().Http.Domain
115+
116+ // Generate a fallback certificate for admin domain regardless of proxy status
117+ var fallbackDomains []string
118+ fallbackDomains = append(fallbackDomains, adminDomain)
119+
120+ // Include root domain in fallback certificate if it's different from admin domain
121+ if rootDomain != "" && rootDomain != adminDomain {
122+ fallbackDomains = append(fallbackDomains, rootDomain)
123+ }
124+
125+ // Generate the fallback certificate
126+ fallbackCert, err := generateFallbackCertificates(fallbackDomains)
127+ if err != nil {
128+ logger.Warn("Failed to generate fallback certificate", "error", err)
129+ } else {
130+ logger.Info("Generated fallback self-signed certificate for admin domain",
131+ "domain", adminDomain,
132+ "valid_until", time.Now().Add(24*time.Hour).Format("2006-01-02 15:04:05"))
133+ // Store the fallback certificate
134+ p.fallbackCert = fallbackCert
135+ }
136+
112137 // Skip certificate acquisition if we're behind a TLS-terminating proxy
113138 // and the skip certificates option is enabled
114139 if isProxied && p.config.SkipCertificates {
115140 logger.Info("Upstream TLS-terminating proxy detected, skipping Let's Encrypt certificate acquisition",
116141 "upstream_proxy", "detected",
117142 "certificate_acquisition", "disabled")
118- // We still setup a fallback cert for the admin domain
119- adminDomain := p.app.GetConfig().Http.FullDomain()
120- rootDomain := p.app.GetConfig().Http.Domain
121-
122- var fallbackDomains []string
123- fallbackDomains = append(fallbackDomains, adminDomain)
124-
125- // Include root domain in fallback certificate if it's different from admin domain
126- if rootDomain != "" && rootDomain != adminDomain {
127- fallbackDomains = append(fallbackDomains, rootDomain)
128- }
129-
130- fallbackCert, err := generateFallbackCertificates(fallbackDomains)
131- if err != nil {
132- logger.Warn("Failed to generate fallback certificate", "error", err)
133- } else {
134- logger.Info("Generated fallback self-signed certificate for admin domain behind proxy",
135- "domain", adminDomain,
136- "valid_until", time.Now().Add(24*time.Hour).Format("2006-01-02 15:04:05"))
137- // Store the fallback certificate
138- p.fallbackCert = fallbackCert
139- }
140143
141- return
144+ // We will still initialize the certificate manager but will use it in a more limited way
145+ logger.Info("Initializing minimal Let's Encrypt certificate manager for fallback purposes")
142146 }
143147
144148 // Create a cache directory if it doesn't exist
@@ -181,10 +185,6 @@ func (p *Proxy) setupCertManager() {
181185 "url", acme.LetsEncryptURL)
182186 }
183187
184- // Set HostPolicy to allow the admin domain
185- adminDomain := p.app.GetConfig().Http.FullDomain()
186- rootDomain := p.app.GetConfig().Http.Domain
187-
188188 // Use a stricter hostpolicy that only allows domains in our routes
189189 certManager.HostPolicy = func(_ context.Context, host string) error {
190190 // Always allow the admin domain and root domain
@@ -210,31 +210,14 @@ func (p *Proxy) setupCertManager() {
210210 return fmt.Errorf("host %q not configured in routes", host)
211211 }
212212
213- // Generate a fallback self-signed certificate for the admin domain
214- var fallbackDomains []string
215- fallbackDomains = append(fallbackDomains, adminDomain)
216-
217- // Include root domain in fallback certificate if it's different from admin domain
218- if rootDomain != "" && rootDomain != adminDomain {
219- fallbackDomains = append(fallbackDomains, rootDomain)
220- }
221-
222- fallbackCert, err := generateFallbackCertificates(fallbackDomains)
223- if err != nil {
224- logger.Warn("Failed to generate fallback certificate", "error", err)
225- } else {
226- logger.Info("Generated fallback self-signed certificate for admin domain",
227- "domain", adminDomain,
228- "valid_until", time.Now().Add(24*time.Hour).Format("2006-01-02 15:04:05"))
229- // Store the fallback certificate
230- p.fallbackCert = fallbackCert
231- }
232-
233213 p.certManager = certManager
234214 logger.Debug("Certificate manager setup completed",
235215 "directory", dir,
236216 "mode", p.config.LetsEncryptMode)
237217
218+ // Preload certificates from disk into cache to avoid unnecessary Let's Encrypt requests
219+ p.preloadCertificates()
220+
238221 // Only request admin certificate during server startup, not during specific domain processing
239222 if !p.processingSpecificDomain {
240223 // Request the certificate for the admin domain
@@ -343,13 +326,52 @@ func getCertificateType(cert *x509.Certificate) string {
343326 // Returns whether it's valid and how many days until expiration, and the certificate type
344327 func isCertificateValid(certData []byte, domain string) (bool, float64, string, *x509.Certificate) {
345328 // Parse the certificate to check its validity
346- block, _ := pem.Decode(certData)
329+ block, rest := pem.Decode(certData)
347330 if block == nil || block.Type != "CERTIFICATE" {
348- logger.Warn("Invalid certificate data",
349- "domain", domain)
331+ logger.Warn("Invalid certificate data - not a valid PEM CERTIFICATE block",
332+ "domain", domain,
333+ "dataLength", len(certData))
334+
335+ // Debug: output first few bytes to help diagnose format issues
336+ if len(certData) > 0 {
337+ previewLen := 50
338+ if len(certData) < previewLen {
339+ previewLen = len(certData)
340+ }
341+ preview := string(certData[:previewLen])
342+ logger.Debug("Certificate data preview",
343+ "domain", domain,
344+ "preview", preview+"...")
345+ }
350346 return false, 0, "unknown", nil
351347 }
352348
349+ // Check if there are additional certificates in the chain
350+ var certChain []*x509.Certificate
351+ currentRest := rest
352+
353+ for len(currentRest) > 0 {
354+ var nextBlock *pem.Block
355+ nextBlock, currentRest = pem.Decode(currentRest)
356+ if nextBlock == nil || nextBlock.Type != "CERTIFICATE" {
357+ break
358+ }
359+ cert, err := x509.ParseCertificate(nextBlock.Bytes)
360+ if err != nil {
361+ logger.Debug("Failed to parse certificate in chain",
362+ "domain", domain,
363+ "error", err)
364+ continue
365+ }
366+ certChain = append(certChain, cert)
367+ }
368+
369+ if len(certChain) > 0 {
370+ logger.Debug("Found certificate chain with additional certificates",
371+ "domain", domain,
372+ "chain_length", len(certChain))
373+ }
374+
353375 cert, err := x509.ParseCertificate(block.Bytes)
354376 if err != nil {
355377 logger.Warn("Failed to parse certificate",
@@ -358,8 +380,35 @@ func isCertificateValid(certData []byte, domain string) (bool, float64, string,
358380 return false, 0, "unknown", nil
359381 }
360382
383+ // Verify the certificate includes the domain we're checking for
384+ var domainFound bool
385+ if len(cert.DNSNames) > 0 {
386+ logger.Debug("Certificate DNS names",
387+ "domain", domain,
388+ "certificate_domains", cert.DNSNames)
389+
390+ for _, dnsName := range cert.DNSNames {
391+ if dnsName == domain {
392+ domainFound = true
393+ break
394+ }
395+ }
396+ }
397+
398+ if !domainFound && cert.Subject.CommonName != domain {
399+ logger.Warn("Certificate doesn't match the required domain",
400+ "domain", domain,
401+ "cert_common_name", cert.Subject.CommonName,
402+ "cert_dns_names", cert.DNSNames)
403+ return false, 0, "unknown", cert
404+ }
405+
361406 // Determine the certificate type
362407 certType := getCertificateType(cert)
408+ logger.Debug("Determined certificate type",
409+ "domain", domain,
410+ "type", certType,
411+ "issuer", cert.Issuer.CommonName)
363412
364413 // Check if the certificate is still valid
365414 now := time.Now()
@@ -816,6 +865,46 @@ func (p *Proxy) requestAdminCertificate() {
816865 return
817866 }
818867
868+ // First, check if certificate is already in the autocert cache
869+ // This is the critical step to avoid unnecessary Let's Encrypt requests
870+ cacheKey := "cert-" + adminDomain
871+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
872+ defer cancel()
873+
874+ if p.certManager != nil && p.certManager.Cache != nil {
875+ certData, err := p.certManager.Cache.Get(ctx, cacheKey)
876+ if err == nil && len(certData) > 0 {
877+ // Check if the cached certificate is valid
878+ isValid, expiresInDays, certType, _ := isCertificateValid(certData, adminDomain)
879+ if isValid {
880+ logger.Info("Using valid certificate from cache",
881+ "domain", adminDomain,
882+ "type", certType,
883+ "expires_in_days", expiresInDays)
884+ return
885+ }
886+
887+ // If certificate is invalid or expiring soon, log but continue
888+ logger.Info("Cached certificate is invalid or expiring soon",
889+ "domain", adminDomain,
890+ "expires_in_days", expiresInDays,
891+ "valid", isValid)
892+ }
893+ }
894+
895+ // Additional rate limit safety check
896+ // Check if we've recently hit a rate limit for this domain
897+ // This is a temporary memory-only safeguard
898+ rlCacheKey := adminDomain + "_ratelimit_history"
899+ rateLimitData, _ := p.certManager.Cache.Get(ctx, rlCacheKey)
900+ if len(rateLimitData) > 0 {
901+ logger.Warn("Rate limit history found for admin domain, skipping certificate request",
902+ "domain", adminDomain,
903+ "data", string(rateLimitData),
904+ "solution", "Wait for rate limit to expire")
905+ return
906+ }
907+
819908 // Extract hostname from admin domain, resolving it to see if it's publicly accessible
820909 host := adminDomain
821910 ips, err := net.LookupIP(host)
@@ -907,7 +996,7 @@ func (p *Proxy) requestAdminCertificate() {
907996 "timeout", "1 minute")
908997
909998 // Create a context with timeout for the initial certificate request
910- ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
999+ ctx, cancel = context.WithTimeout(context.Background(), 1*time.Minute)
9111000 defer cancel()
9121001
9131002 // Try to obtain the certificate with timeout
@@ -951,7 +1040,10 @@ func (p *Proxy) requestAdminCertificate() {
9511040 re := regexp.MustCompile(`retry after ([^:]+)`)
9521041 matches := re.FindStringSubmatch(certErr.Error())
9531042 if len(matches) > 1 {
954- retryAfterStr = matches[1]
1043+ retryAfterStr = strings.TrimSpace(matches[1])
1044+ if idx := strings.Index(retryAfterStr, ":"); idx > 0 {
1045+ retryAfterStr = retryAfterStr[:idx]
1046+ }
9551047 }
9561048
9571049 logger.Error("Let's Encrypt rate limit reached",
@@ -960,6 +1052,27 @@ func (p *Proxy) requestAdminCertificate() {
9601052 "retry_after", retryAfterStr,
9611053 "solution", "Wait until the rate limit expires, then restart the server")
9621054
1055+ // Store rate limit history in cache to prevent further requests
1056+ rlCacheKey := adminDomain + "_ratelimit_history"
1057+ rlCtx, rlCancel := context.WithTimeout(context.Background(), 5*time.Second)
1058+ defer rlCancel()
1059+
1060+ // Store the error message and timestamp in the cache
1061+ rlData := []byte(fmt.Sprintf("Rate limited at %s: %s",
1062+ time.Now().Format(time.RFC3339), certErr.Error()))
1063+
1064+ if p.certManager != nil && p.certManager.Cache != nil {
1065+ // Store with a long expiration - Let's Encrypt rate limits are typically for a week
1066+ err := p.certManager.Cache.Put(rlCtx, rlCacheKey, rlData)
1067+ if err != nil {
1068+ logger.Debug("Failed to store rate limit history in cache",
1069+ "domain", adminDomain, "error", err)
1070+ } else {
1071+ logger.Info("Stored rate limit history in cache to prevent further requests",
1072+ "domain", adminDomain, "retry_after", retryAfterStr)
1073+ }
1074+ }
1075+
9631076 // Don't retry on rate limits
9641077 logger.Info("Skipping certificate request retries due to rate limiting",
9651078 "domain", adminDomain)
@@ -989,10 +1102,96 @@ func (p *Proxy) requestAdminCertificate() {
9891102 newCertPath := filepath.Join(certDir, newName)
9901103 newKeyPath := filepath.Join(certDir, newName+"+rsa")
9911104
1105+ // Debug log certificate file paths
1106+ logger.Debug("Certificate file paths",
1107+ "domain", adminDomain,
1108+ "old_cert", oldCertPath,
1109+ "new_cert", newCertPath,
1110+ "old_key", oldKeyPath,
1111+ "new_key", newKeyPath)
1112+
1113+ // Check if files exist first
1114+ oldCertExists := false
1115+ if _, err := os.Stat(oldCertPath); err == nil {
1116+ oldCertExists = true
1117+ logger.Debug("Old certificate file exists", "path", oldCertPath)
1118+ } else {
1119+ logger.Debug("Old certificate file not found", "path", oldCertPath, "error", err)
1120+ }
1121+
1122+ // Read the certificate files directly to verify they contain valid data
1123+ if oldCertExists {
1124+ certBytes, err := os.ReadFile(oldCertPath)
1125+ if err != nil {
1126+ logger.Error("Failed to read certificate file",
1127+ "path", oldCertPath,
1128+ "error", err)
1129+ } else {
1130+ logger.Debug("Successfully read certificate file",
1131+ "path", oldCertPath,
1132+ "size", len(certBytes))
1133+
1134+ // Verify it's a valid PEM format
1135+ block, _ := pem.Decode(certBytes)
1136+ if block == nil {
1137+ logger.Error("Certificate file does not contain valid PEM data",
1138+ "path", oldCertPath)
1139+ } else {
1140+ logger.Debug("Certificate file contains valid PEM data",
1141+ "path", oldCertPath,
1142+ "type", block.Type)
1143+ }
1144+ }
1145+
1146+ // Check permissions on cert and key files
1147+ certFileInfo, err := os.Stat(oldCertPath)
1148+ if err == nil {
1149+ mode := certFileInfo.Mode()
1150+ logger.Debug("Certificate file permissions",
1151+ "path", oldCertPath,
1152+ "mode", mode.String())
1153+
1154+ // Ensure cert file is readable by all
1155+ if mode.Perm()&0444 != 0444 {
1156+ logger.Warn("Certificate file has restrictive permissions, fixing",
1157+ "path", oldCertPath,
1158+ "current_mode", mode.String())
1159+ if err := os.Chmod(oldCertPath, 0644); err != nil {
1160+ logger.Error("Failed to fix certificate file permissions",
1161+ "path", oldCertPath,
1162+ "error", err)
1163+ }
1164+ }
1165+ }
1166+
1167+ // Check key file permissions
1168+ if _, err := os.Stat(oldKeyPath); err == nil {
1169+ keyFileInfo, err := os.Stat(oldKeyPath)
1170+ if err == nil {
1171+ mode := keyFileInfo.Mode()
1172+ logger.Debug("Key file permissions",
1173+ "path", oldKeyPath,
1174+ "mode", mode.String())
1175+
1176+ // Ensure key file is only readable by owner
1177+ if mode.Perm()&0400 != 0400 || mode.Perm()&0077 != 0 {
1178+ logger.Warn("Key file has incorrect permissions, fixing",
1179+ "path", oldKeyPath,
1180+ "current_mode", mode.String())
1181+ if err := os.Chmod(oldKeyPath, 0600); err != nil {
1182+ logger.Error("Failed to fix key file permissions",
1183+ "path", oldKeyPath,
1184+ "error", err)
1185+ }
1186+ }
1187+ }
1188+ }
1189+ }
1190+
9921191 // Only rename if the files exist and the new names are different
9931192 if oldCertPath != newCertPath {
9941193 // Check if old cert exists
995- if _, err := os.Stat(oldCertPath); err == nil {
1194+ if oldCertExists {
9961195 // Rename cert
9971196 if err := os.Rename(oldCertPath, newCertPath); err != nil {
9981197 logger.Warn("Failed to rename certificate file",
@@ -1023,6 +1222,8 @@ func (p *Proxy) requestAdminCertificate() {
10231222 "from", oldKeyPath,
10241223 "to", newKeyPath)
10251224 }
1225+ } else {
1226+ logger.Warn("Old key file not found", "path", oldKeyPath, "error", err)
10261227 }
10271228 }
10281229 }
@@ -1030,6 +1231,75 @@ func (p *Proxy) requestAdminCertificate() {
10301231 logger.Info("Successfully obtained Let's Encrypt certificate",
10311232 "domain", adminDomain,
10321233 "type", certType)
1234+
1235+ // Verify the certificate files are accessible in the cache
1236+ cacheKey := "cert-" + adminDomain
1237+ cacheCtx, cacheCancel := context.WithTimeout(context.Background(), 5*time.Second)
1238+ defer cacheCancel()
1239+
1240+ certData, cacheErr := p.certManager.Cache.Get(cacheCtx, cacheKey)
1241+ if cacheErr != nil {
1242+ logger.Warn("Certificate not found in cache after successful obtainment",
1243+ "domain", adminDomain,
1244+ "error", cacheErr,
1245+ "solution", "Check certificate permissions and file paths")
1246+
1247+ // If certificate isn't in autocert cache but exists in filesystem, try to manually add it
1248+ newCertExists := false
1249+ if _, err := os.Stat(newCertPath); err == nil {
1250+ newCertExists = true
1251+ }
1252+
1253+ if oldCertExists || newCertExists {
1254+ logger.Info("Attempting to manually add certificate to cache from filesystem",
1255+ "domain", adminDomain)
1256+
1257+ sourceCertPath := newCertPath
1258+ sourceKeyPath := newKeyPath
1259+
1260+ // If the file wasn't renamed, use the old paths
1261+ if _, err := os.Stat(sourceCertPath); err != nil {
1262+ sourceCertPath = oldCertPath
1263+ sourceKeyPath = oldKeyPath
1264+ }
1265+
1266+ // Read the certificate and key files
1267+ certBytes, err := os.ReadFile(sourceCertPath)
1268+ if err != nil {
1269+ logger.Error("Failed to read certificate file for manual cache addition",
1270+ "path", sourceCertPath,
1271+ "error", err)
1272+ } else {
1273+ keyBytes, err := os.ReadFile(sourceKeyPath)
1274+ if err != nil {
1275+ logger.Error("Failed to read key file for manual cache addition",
1276+ "path", sourceKeyPath,
1277+ "error", err)
1278+ } else {
1279+ // Add to autocert cache
1280+ if err := p.certManager.Cache.Put(cacheCtx, cacheKey, certBytes); err != nil {
1281+ logger.Error("Failed to manually add certificate to cache",
1282+ "domain", adminDomain,
1283+ "error", err)
1284+ } else {
1285+ keyKey := "key-" + adminDomain
1286+ if err := p.certManager.Cache.Put(cacheCtx, keyKey, keyBytes); err != nil {
1287+ logger.Error("Failed to manually add key to cache",
1288+ "domain", adminDomain,
1289+ "error", err)
1290+ } else {
1291+ logger.Info("Successfully manually added certificate to cache",
1292+ "domain", adminDomain)
1293+ }
1294+ }
1295+ }
1296+ }
1297+ }
1298+ } else {
1299+ logger.Debug("Certificate successfully stored in cache",
1300+ "domain", adminDomain,
1301+ "cert_size", len(certData))
1302+ }
10331303 } else {
10341304 logger.Error("Unexpected error: received nil certificate but no error",
10351305 "domain", adminDomain,
@@ -1056,6 +1326,33 @@ func (p *Proxy) requestDomainCertificate(domain string) {
10561326 return
10571327 }
10581328
1329+ // First, check if certificate is already in the autocert cache
1330+ // This is the critical step to avoid unnecessary Let's Encrypt requests
1331+ cacheKey := "cert-" + domain
1332+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
1333+ defer cancel()
1334+
1335+ if p.certManager != nil && p.certManager.Cache != nil {
1336+ certData, err := p.certManager.Cache.Get(ctx, cacheKey)
1337+ if err == nil && len(certData) > 0 {
1338+ // Check if the cached certificate is valid
1339+ isValid, expiresInDays, certType, _ := isCertificateValid(certData, domain)
1340+ if isValid {
1341+ logger.Info("Using valid certificate from cache",
1342+ "domain", domain,
1343+ "type", certType,
1344+ "expires_in_days", expiresInDays)
1345+ return
1346+ }
1347+
1348+ // If certificate is invalid or expiring soon, log but continue
1349+ logger.Info("Cached certificate is invalid or expiring soon",
1350+ "domain", domain,
1351+ "expires_in_days", expiresInDays,
1352+ "valid", isValid)
1353+ }
1354+ }
1355+
10591356 // Extract hostname from domain, resolving it to see if it's publicly accessible
10601357 ips, err := net.LookupIP(domain)
10611358 if err != nil {
@@ -1100,6 +1397,46 @@ func (p *Proxy) requestDomainCertificate(domain string) {
11001397 validInCache, _ := p.checkCertificateInCache(domain)
11011398 validInFS, _ := p.checkCertificateInFilesystem(domain)
11021399 if !validInCache && validInFS {
1400+ // Try to load the certificate into the cache if it's not there already
1401+ // This helps prevent future Let's Encrypt requests
1402+ hasOnDisk, certType := p.checkCertificateInFilesystem(domain)
1403+ if hasOnDisk {
1404+ logger.Info("Loading filesystem certificate into cache",
1405+ "domain", domain,
1406+ "type", certType)
1407+
1408+ // Get the certificate directory path for finding files
1409+ certDir := p.config.CertDir
1410+ if certDir == "" {
1411+ certDir = p.app.GetConfig().General.StorageDir + "/certs"
1412+ }
1413+
1414+ // Look for production certificate first, then staging
1415+ certFile := getCertificateFileName(domain, "production")
1416+ certPath := filepath.Join(certDir, certFile)
1417+ keyPath := filepath.Join(certDir, certFile+"+rsa")
1418+
1419+ // If production cert doesn't exist, try staging
1420+ if _, err := os.Stat(certPath); os.IsNotExist(err) {
1421+ certFile = getCertificateFileName(domain, "staging")
1422+ certPath = filepath.Join(certDir, certFile)
1423+ keyPath = filepath.Join(certDir, certFile+"+rsa")
1424+ }
1425+
1426+ // If no specific cert found, try the domain name directly
1427+ if _, err := os.Stat(certPath); os.IsNotExist(err) {
1428+ certPath = filepath.Join(certDir, domain)
1429+ keyPath = filepath.Join(certDir, domain+"+rsa")
1430+ }
1431+
1432+ // Add to cache if files exist
1433+ if _, err := os.Stat(certPath); err == nil {
1434+ if _, err := os.Stat(keyPath); err == nil {
1435+ p.addCertificateToCache(domain, certPath, keyPath)
1436+ }
1437+ }
1438+ }
1439+
11031440 if testCertificateWithTLSConnection(domain, p.config.Port) {
11041441 logger.Debug("Certificate verification succeeded via TLS connection",
11051442 "domain", domain,
@@ -1116,6 +1453,18 @@ func (p *Proxy) requestDomainCertificate(domain string) {
11161453 }
11171454 }
11181455
1456+ // Additional rate limit safety check
1457+ // Check if we've recently hit a rate limit for this domain
1458+ // This is a temporary memory-only safeguard
1459+ cacheKey = domain + "_ratelimit_history"
1460+ rateLimitData, _ := p.certManager.Cache.Get(ctx, cacheKey)
1461+ if len(rateLimitData) > 0 {
1462+ logger.Warn("Rate limit history found for this domain, skipping certificate request",
1463+ "domain", domain,
1464+ "solution", "Wait for rate limit to expire")
1465+ return
1466+ }
1467+
11191468 // Get the certificate directory path for naming the output files
11201469 certDir := p.config.CertDir
11211470 if certDir == "" {
@@ -1146,7 +1495,7 @@ func (p *Proxy) requestDomainCertificate(domain string) {
11461495 "timeout", "1 minute")
11471496
11481497 // Create a context with timeout for the initial certificate request
1149- ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
1498+ ctx, cancel = context.WithTimeout(context.Background(), 1*time.Minute)
11501499 defer cancel()
11511500
11521501 // Try to obtain the certificate with timeout
@@ -1190,7 +1539,10 @@ func (p *Proxy) requestDomainCertificate(domain string) {
11901539 re := regexp.MustCompile(`retry after ([^:]+)`)
11911540 matches := re.FindStringSubmatch(certErr.Error())
11921541 if len(matches) > 1 {
1193- retryAfterStr = matches[1]
1542+ retryAfterStr = strings.TrimSpace(matches[1])
1543+ if idx := strings.Index(retryAfterStr, ":"); idx > 0 {
1544+ retryAfterStr = retryAfterStr[:idx]
1545+ }
11941546 }
11951547
11961548 logger.Error("Let's Encrypt rate limit reached",
@@ -1199,6 +1551,27 @@ func (p *Proxy) requestDomainCertificate(domain string) {
11991551 "retry_after", retryAfterStr,
12001552 "solution", "Wait until the rate limit expires, then restart the server")
12011553
1554+ // Store rate limit history in cache to prevent further requests
1555+ rlCacheKey := domain + "_ratelimit_history"
1556+ rlCtx, rlCancel := context.WithTimeout(context.Background(), 5*time.Second)
1557+ defer rlCancel()
1558+
1559+ // Store the error message and timestamp in the cache
1560+ rlData := []byte(fmt.Sprintf("Rate limited at %s: %s",
1561+ time.Now().Format(time.RFC3339), certErr.Error()))
1562+
1563+ if p.certManager != nil && p.certManager.Cache != nil {
1564+ // Store with a long expiration - Let's Encrypt rate limits are typically for a week
1565+ err := p.certManager.Cache.Put(rlCtx, rlCacheKey, rlData)
1566+ if err != nil {
1567+ logger.Debug("Failed to store rate limit history in cache",
1568+ "domain", domain, "error", err)
1569+ } else {
1570+ logger.Info("Stored rate limit history in cache to prevent further requests",
1571+ "domain", domain, "retry_after", retryAfterStr)
1572+ }
1573+ }
1574+
12021575 // Don't retry on rate limits
12031576 logger.Info("Skipping certificate request retries due to rate limiting",
12041577 "domain", domain)
@@ -1262,6 +1635,8 @@ func (p *Proxy) requestDomainCertificate(domain string) {
12621635 "from", oldKeyPath,
12631636 "to", newKeyPath)
12641637 }
1638+ } else {
1639+ logger.Warn("Old key file not found", "path", oldKeyPath, "error", err)
12651640 }
12661641 }
12671642 }
@@ -1393,6 +1768,8 @@ func (p *Proxy) retryCertificateRequest(domain string, maxRetries int, initialBa
13931768 "from", oldKeyPath,
13941769 "to", newKeyPath)
13951770 }
1771+ } else {
1772+ logger.Warn("Old key file not found", "path", oldKeyPath, "error", err)
13961773 }
13971774 }
13981775 }
@@ -1585,3 +1962,319 @@ func generateFallbackCertificates(domains []string) (*tls.Certificate, error) {
15851962
15861963 return &cert, nil
15871964 }
1965+
1966+// preloadCertificates scans the certificate directory and loads all valid certificates
1967+// into the autocert cache to prevent unnecessary requests to Let's Encrypt
1968+func (p *Proxy) preloadCertificates() {
1969+ if p.certManager == nil || p.certManager.Cache == nil {
1970+ logger.Warn("Cannot preload certificates: certificate manager or cache not initialized")
1971+ return
1972+ }
1973+
1974+ // Get the certificate directory path
1975+ certDir := p.config.CertDir
1976+ if certDir == "" {
1977+ certDir = p.app.GetConfig().General.StorageDir + "/certs"
1978+ }
1979+
1980+ logger.Info("Preloading certificates from directory", "dir", certDir)
1981+
1982+ // Ensure directory exists
1983+ if _, err := os.Stat(certDir); os.IsNotExist(err) {
1984+ logger.Warn("Certificate directory does not exist, creating it", "dir", certDir)
1985+ if err := os.MkdirAll(certDir, 0755); err != nil {
1986+ logger.Error("Failed to create certificate directory", "dir", certDir, "error", err)
1987+ return
1988+ }
1989+ }
1990+
1991+ // List all files in the certificate directory
1992+ files, err := os.ReadDir(certDir)
1993+ if err != nil {
1994+ logger.Error("Failed to read certificate directory", "dir", certDir, "error", err)
1995+ return
1996+ }
1997+
1998+ // Log all files in the directory to help troubleshoot
1999+ var fileNames []string
2000+ for _, file := range files {
2001+ fileNames = append(fileNames, file.Name())
2002+
2003+ // Check file permissions and fix if too restrictive
2004+ filePath := filepath.Join(certDir, file.Name())
2005+ info, err := os.Stat(filePath)
2006+ if err != nil {
2007+ logger.Warn("Could not stat file", "path", filePath, "error", err)
2008+ continue
2009+ }
2010+
2011+ mode := info.Mode()
2012+ if strings.HasSuffix(file.Name(), "+rsa") {
2013+ // Key files should be 0600 (owner read/write only)
2014+ if mode.Perm() != 0600 {
2015+ logger.Info("Fixing key file permissions", "file", file.Name(), "current_mode", mode.String(), "target_mode", "0600")
2016+ if err := os.Chmod(filePath, 0600); err != nil {
2017+ logger.Warn("Failed to fix key file permissions", "file", file.Name(), "error", err)
2018+ }
2019+ }
2020+ } else {
2021+ // Certificate files should be 0644 (readable by all)
2022+ if mode.Perm() != 0644 {
2023+ logger.Info("Fixing certificate file permissions", "file", file.Name(), "current_mode", mode.String(), "target_mode", "0644")
2024+ if err := os.Chmod(filePath, 0644); err != nil {
2025+ logger.Warn("Failed to fix certificate file permissions", "file", file.Name(), "error", err)
2026+ }
2027+ }
2028+ }
2029+ }
2030+
2031+ logger.Debug("Files in certificate directory", "dir", certDir, "files", fileNames)
2032+
2033+ // Track the number of certificates loaded
2034+ loadedCount := 0
2035+
2036+ // Create a context for cache operations
2037+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
2038+ defer cancel()
2039+
2040+ // Find and process certificate files (excluding key files with +rsa suffix)
2041+ // Group them by domain for easier processing
2042+ domainCerts := make(map[string][]string)
2043+
2044+ for _, file := range files {
2045+ name := file.Name()
2046+
2047+ // Skip account keys and +rsa key files as we'll handle these separately
2048+ if strings.HasSuffix(name, "+rsa") || strings.Contains(name, "acme_account") {
2049+ continue
2050+ }
2051+
2052+ // Get associated key file name (file+rsa)
2053+ keyName := name + "+rsa"
2054+ keyPath := filepath.Join(certDir, keyName)
2055+
2056+ // Skip certificates without key files
2057+ if _, err := os.Stat(keyPath); os.IsNotExist(err) {
2058+ logger.Debug("Skipping certificate without key file", "cert", name, "key", keyName)
2059+ continue
2060+ }
2061+
2062+ // Try to determine the domain from the filename
2063+ domain := name
2064+
2065+ // Handle special patterns like domain_letsencrypt-production
2066+ for _, pattern := range []string{"_letsencrypt-production", "_letsencrypt-staging", "_self-signed"} {
2067+ if strings.HasSuffix(domain, pattern) {
2068+ domain = strings.TrimSuffix(domain, pattern)
2069+ break
2070+ }
2071+ }
2072+
2073+ // Add to domain certificates map
2074+ domainCerts[domain] = append(domainCerts[domain], name)
2075+ }
2076+
2077+ // Process certificates for each domain
2078+ for domain, certFiles := range domainCerts {
2079+ logger.Debug("Found certificate files for domain", "domain", domain, "files", certFiles)
2080+
2081+ // Prioritize production certificates if multiple exist
2082+ var bestCert string
2083+ bestCertType := ""
2084+
2085+ for _, certFile := range certFiles {
2086+ certPath := filepath.Join(certDir, certFile)
2087+
2088+ // Read certificate to check its validity
2089+ certData, err := os.ReadFile(certPath)
2090+ if err != nil {
2091+ logger.Warn("Failed to read certificate file", "path", certPath, "error", err)
2092+ continue
2093+ }
2094+
2095+ // Debug certificate content
2096+ if len(certData) < 100 {
2097+ logger.Warn("Certificate file too small, may be corrupted",
2098+ "path", certPath, "size", len(certData))
2099+ continue
2100+ }
2101+
2102+ // Validate the certificate
2103+ valid, expiresInDays, certType, _ := isCertificateValid(certData, domain)
2104+ if !valid {
2105+ logger.Debug("Invalid certificate found", "domain", domain, "file", certFile)
2106+ continue
2107+ }
2108+
2109+ logger.Info("Found valid certificate", "domain", domain, "type", certType,
2110+ "expires_in_days", expiresInDays, "file", certFile)
2111+
2112+ // Prioritize production certificates over staging ones
2113+ if bestCertType == "" ||
2114+ (bestCertType != "production" && certType == "production") {
2115+ bestCert = certFile
2116+ bestCertType = certType
2117+ }
2118+ }
2119+
2120+ // If we found a valid certificate for this domain, add it to the cache
2121+ if bestCert != "" {
2122+ certPath := filepath.Join(certDir, bestCert)
2123+ keyPath := filepath.Join(certDir, bestCert+"+rsa")
2124+
2125+ // Read certificate and key data
2126+ certData, err := os.ReadFile(certPath)
2127+ if err != nil {
2128+ logger.Warn("Failed to read certificate file", "path", certPath, "error", err)
2129+ continue
2130+ }
2131+
2132+ keyData, err := os.ReadFile(keyPath)
2133+ if err != nil {
2134+ logger.Warn("Failed to read key file", "path", keyPath, "error", err)
2135+ continue
2136+ }
2137+
2138+ // Add certificate to autocert cache
2139+ cacheKey := "cert-" + domain
2140+ err = p.certManager.Cache.Put(ctx, cacheKey, certData)
2141+ if err != nil {
2142+ logger.Error("Failed to add certificate to cache", "domain", domain, "error", err)
2143+ continue
2144+ }
2145+
2146+ // Add key to cache
2147+ keyKey := "key-" + domain
2148+ err = p.certManager.Cache.Put(ctx, keyKey, keyData)
2149+ if err != nil {
2150+ logger.Error("Failed to add key to cache", "domain", domain, "error", err)
2151+ continue
2152+ }
2153+
2154+ logger.Info("Preloaded certificate into cache", "domain", domain, "type", bestCertType)
2155+ loadedCount++
2156+ } else {
2157+ logger.Warn("No valid certificate found for domain after checking all files",
2158+ "domain", domain,
2159+ "files_checked", certFiles)
2160+ }
2161+ }
2162+
2163+ logger.Info("Certificate preloading complete", "loaded_certificates", loadedCount)
2164+}
2165+
2166+// manuallyLoadCertificate is a fallback method to load certificates directly
2167+// when autocert cache is failing. This directly builds the tls.Certificate from files.
2168+func (p *Proxy) manuallyLoadCertificate(domain string) (*tls.Certificate, error) {
2169+ // Get certificate directory
2170+ certDir := p.config.CertDir
2171+ if certDir == "" {
2172+ certDir = p.app.GetConfig().General.StorageDir + "/certs"
2173+ }
2174+
2175+ // Check preferred file naming patterns
2176+ filePatterns := []string{
2177+ // Try production certificate first
2178+ fmt.Sprintf(LetsEncryptProdPattern, domain),
2179+ // Then staging certificate
2180+ fmt.Sprintf(LetsEncryptStagingPattern, domain),
2181+ // Then self-signed
2182+ fmt.Sprintf(SelfSignedCertPattern, domain),
2183+ // Finally plain domain name
2184+ domain,
2185+ }
2186+
2187+ for _, pattern := range filePatterns {
2188+ certPath := filepath.Join(certDir, pattern)
2189+ keyPath := filepath.Join(certDir, pattern+"+rsa")
2190+
2191+ // Check if both files exist
2192+ if _, err := os.Stat(certPath); os.IsNotExist(err) {
2193+ continue
2194+ }
2195+ if _, err := os.Stat(keyPath); os.IsNotExist(err) {
2196+ continue
2197+ }
2198+
2199+ // Try to load the certificate pair
2200+ cert, err := tls.LoadX509KeyPair(certPath, keyPath)
2201+ if err != nil {
2202+ logger.Warn("Failed to load certificate key pair",
2203+ "domain", domain,
2204+ "certPath", certPath,
2205+ "keyPath", keyPath,
2206+ "error", err)
2207+
2208+ // Additional diagnostics
2209+ if data, readErr := os.ReadFile(certPath); readErr == nil {
2210+ logger.Debug("Certificate file content preview",
2211+ "path", certPath,
2212+ "size", len(data),
2213+ "preview", string(data[:min(100, len(data))]))
2214+
2215+ // Try to parse the certificate - this may help diagnose format issues
2216+ block, _ := pem.Decode(data)
2217+ if block == nil {
2218+ logger.Error("Certificate file does not contain valid PEM data",
2219+ "path", certPath)
2220+ } else {
2221+ logger.Debug("Certificate PEM block info",
2222+ "type", block.Type,
2223+ "headers", block.Headers)
2224+ }
2225+ }
2226+
2227+ continue
2228+ }
2229+
2230+ // Success!
2231+ logger.Info("Manually loaded certificate from filesystem",
2232+ "domain", domain,
2233+ "certPath", certPath)
2234+ return &cert, nil
2235+ }
2236+
2237+ // Also try with glob pattern
2238+ filePaths, err := filepath.Glob(filepath.Join(certDir, domain+"*"))
2239+ if err != nil {
2240+ return nil, fmt.Errorf("glob error: %w", err)
2241+ }
2242+
2243+ for _, certPath := range filePaths {
2244+ // Skip if it's an RSA key
2245+ if strings.HasSuffix(certPath, "+rsa") {
2246+ continue
2247+ }
2248+
2249+ // Check if key exists
2250+ keyPath := certPath + "+rsa"
2251+ if _, err := os.Stat(keyPath); os.IsNotExist(err) {
2252+ continue
2253+ }
2254+
2255+ // Try to load
2256+ cert, err := tls.LoadX509KeyPair(certPath, keyPath)
2257+ if err != nil {
2258+ logger.Debug("Failed to load certificate from glob match",
2259+ "domain", domain,
2260+ "certPath", certPath,
2261+ "error", err)
2262+ continue
2263+ }
2264+
2265+ logger.Info("Manually loaded certificate from glob search",
2266+ "domain", domain,
2267+ "certPath", certPath)
2268+ return &cert, nil
2269+ }
2270+
2271+ return nil, fmt.Errorf("no valid certificate found for domain %s", domain)
2272+}
2273+
2274+// min returns the smaller of x or y.
2275+func min(x, y int) int {
2276+ if x < y {
2277+ return x
2278+ }
2279+ return y
2280+}
@@ -109,36 +109,40 @@ func (p *Proxy) setupCertManager() {
109 // First check for upstream TLS-terminating proxies if detection is enabled109 // First check for upstream TLS-terminating proxies if detection is enabled
110 isProxied := p.detectUpstreamProxy()110 isProxied := p.detectUpstreamProxy()
111 111
112+ // Get admin domain and root domain
113+ adminDomain := p.app.GetConfig().Http.FullDomain()
114+ rootDomain := p.app.GetConfig().Http.Domain
115+
116+ // Generate a fallback certificate for admin domain regardless of proxy status
117+ var fallbackDomains []string
118+ fallbackDomains = append(fallbackDomains, adminDomain)
119+
120+ // Include root domain in fallback certificate if it's different from admin domain
121+ if rootDomain != "" && rootDomain != adminDomain {
122+ fallbackDomains = append(fallbackDomains, rootDomain)
123+ }
124+
125+ // Generate the fallback certificate
126+ fallbackCert, err := generateFallbackCertificates(fallbackDomains)
127+ if err != nil {
128+ logger.Warn("Failed to generate fallback certificate", "error", err)
129+ } else {
130+ logger.Info("Generated fallback self-signed certificate for admin domain",
131+ "domain", adminDomain,
132+ "valid_until", time.Now().Add(24*time.Hour).Format("2006-01-02 15:04:05"))
133+ // Store the fallback certificate
134+ p.fallbackCert = fallbackCert
135+ }
136+
112 // Skip certificate acquisition if we're behind a TLS-terminating proxy137 // Skip certificate acquisition if we're behind a TLS-terminating proxy
113 // and the skip certificates option is enabled138 // and the skip certificates option is enabled
114 if isProxied && p.config.SkipCertificates {139 if isProxied && p.config.SkipCertificates {
115 logger.Info("Upstream TLS-terminating proxy detected, skipping Let's Encrypt certificate acquisition",140 logger.Info("Upstream TLS-terminating proxy detected, skipping Let's Encrypt certificate acquisition",
116 "upstream_proxy", "detected",141 "upstream_proxy", "detected",
117 "certificate_acquisition", "disabled")142 "certificate_acquisition", "disabled")
118- // We still setup a fallback cert for the admin domain
119- adminDomain := p.app.GetConfig().Http.FullDomain()
120- rootDomain := p.app.GetConfig().Http.Domain
121-
122- var fallbackDomains []string
123- fallbackDomains = append(fallbackDomains, adminDomain)
124-
125- // Include root domain in fallback certificate if it's different from admin domain
126- if rootDomain != "" && rootDomain != adminDomain {
127- fallbackDomains = append(fallbackDomains, rootDomain)
128- }
129-
130- fallbackCert, err := generateFallbackCertificates(fallbackDomains)
131- if err != nil {
132- logger.Warn("Failed to generate fallback certificate", "error", err)
133- } else {
134- logger.Info("Generated fallback self-signed certificate for admin domain behind proxy",
135- "domain", adminDomain,
136- "valid_until", time.Now().Add(24*time.Hour).Format("2006-01-02 15:04:05"))
137- // Store the fallback certificate
138- p.fallbackCert = fallbackCert
139- }
140 143
141- return144+ // We will still initialize the certificate manager but will use it in a more limited way
145+ logger.Info("Initializing minimal Let's Encrypt certificate manager for fallback purposes")
142 }146 }
143 147
144 // Create a cache directory if it doesn't exist148 // Create a cache directory if it doesn't exist
@@ -181,10 +185,6 @@ func (p *Proxy) setupCertManager() {
181 "url", acme.LetsEncryptURL)185 "url", acme.LetsEncryptURL)
182 }186 }
183 187
184- // Set HostPolicy to allow the admin domain
185- adminDomain := p.app.GetConfig().Http.FullDomain()
186- rootDomain := p.app.GetConfig().Http.Domain
187-
188 // Use a stricter hostpolicy that only allows domains in our routes188 // Use a stricter hostpolicy that only allows domains in our routes
189 certManager.HostPolicy = func(_ context.Context, host string) error {189 certManager.HostPolicy = func(_ context.Context, host string) error {
190 // Always allow the admin domain and root domain190 // Always allow the admin domain and root domain
@@ -210,31 +210,14 @@ func (p *Proxy) setupCertManager() {
210 return fmt.Errorf("host %q not configured in routes", host)210 return fmt.Errorf("host %q not configured in routes", host)
211 }211 }
212 212
213- // Generate a fallback self-signed certificate for the admin domain
214- var fallbackDomains []string
215- fallbackDomains = append(fallbackDomains, adminDomain)
216-
217- // Include root domain in fallback certificate if it's different from admin domain
218- if rootDomain != "" && rootDomain != adminDomain {
219- fallbackDomains = append(fallbackDomains, rootDomain)
220- }
221-
222- fallbackCert, err := generateFallbackCertificates(fallbackDomains)
223- if err != nil {
224- logger.Warn("Failed to generate fallback certificate", "error", err)
225- } else {
226- logger.Info("Generated fallback self-signed certificate for admin domain",
227- "domain", adminDomain,
228- "valid_until", time.Now().Add(24*time.Hour).Format("2006-01-02 15:04:05"))
229- // Store the fallback certificate
230- p.fallbackCert = fallbackCert
231- }
232-
233 p.certManager = certManager213 p.certManager = certManager
234 logger.Debug("Certificate manager setup completed",214 logger.Debug("Certificate manager setup completed",
235 "directory", dir,215 "directory", dir,
236 "mode", p.config.LetsEncryptMode)216 "mode", p.config.LetsEncryptMode)
237 217
218+ // Preload certificates from disk into cache to avoid unnecessary Let's Encrypt requests
219+ p.preloadCertificates()
220+
238 // Only request admin certificate during server startup, not during specific domain processing221 // Only request admin certificate during server startup, not during specific domain processing
239 if !p.processingSpecificDomain {222 if !p.processingSpecificDomain {
240 // Request the certificate for the admin domain223 // Request the certificate for the admin domain
@@ -343,13 +326,52 @@ func getCertificateType(cert *x509.Certificate) string {
343 // Returns whether it's valid and how many days until expiration, and the certificate type326 // Returns whether it's valid and how many days until expiration, and the certificate type
344 func isCertificateValid(certData []byte, domain string) (bool, float64, string, *x509.Certificate) {327 func isCertificateValid(certData []byte, domain string) (bool, float64, string, *x509.Certificate) {
345 // Parse the certificate to check its validity328 // Parse the certificate to check its validity
346- block, _ := pem.Decode(certData)329+ block, rest := pem.Decode(certData)
347 if block == nil || block.Type != "CERTIFICATE" {330 if block == nil || block.Type != "CERTIFICATE" {
348- logger.Warn("Invalid certificate data",331+ logger.Warn("Invalid certificate data - not a valid PEM CERTIFICATE block",
349- "domain", domain)332+ "domain", domain,
333+ "dataLength", len(certData))
334+
335+ // Debug: output first few bytes to help diagnose format issues
336+ if len(certData) > 0 {
337+ previewLen := 50
338+ if len(certData) < previewLen {
339+ previewLen = len(certData)
340+ }
341+ preview := string(certData[:previewLen])
342+ logger.Debug("Certificate data preview",
343+ "domain", domain,
344+ "preview", preview+"...")
345+ }
350 return false, 0, "unknown", nil346 return false, 0, "unknown", nil
351 }347 }
352 348
349+ // Check if there are additional certificates in the chain
350+ var certChain []*x509.Certificate
351+ currentRest := rest
352+
353+ for len(currentRest) > 0 {
354+ var nextBlock *pem.Block
355+ nextBlock, currentRest = pem.Decode(currentRest)
356+ if nextBlock == nil || nextBlock.Type != "CERTIFICATE" {
357+ break
358+ }
359+ cert, err := x509.ParseCertificate(nextBlock.Bytes)
360+ if err != nil {
361+ logger.Debug("Failed to parse certificate in chain",
362+ "domain", domain,
363+ "error", err)
364+ continue
365+ }
366+ certChain = append(certChain, cert)
367+ }
368+
369+ if len(certChain) > 0 {
370+ logger.Debug("Found certificate chain with additional certificates",
371+ "domain", domain,
372+ "chain_length", len(certChain))
373+ }
374+
353 cert, err := x509.ParseCertificate(block.Bytes)375 cert, err := x509.ParseCertificate(block.Bytes)
354 if err != nil {376 if err != nil {
355 logger.Warn("Failed to parse certificate",377 logger.Warn("Failed to parse certificate",
@@ -358,8 +380,35 @@ func isCertificateValid(certData []byte, domain string) (bool, float64, string,
358 return false, 0, "unknown", nil380 return false, 0, "unknown", nil
359 }381 }
360 382
383+ // Verify the certificate includes the domain we're checking for
384+ var domainFound bool
385+ if len(cert.DNSNames) > 0 {
386+ logger.Debug("Certificate DNS names",
387+ "domain", domain,
388+ "certificate_domains", cert.DNSNames)
389+
390+ for _, dnsName := range cert.DNSNames {
391+ if dnsName == domain {
392+ domainFound = true
393+ break
394+ }
395+ }
396+ }
397+
398+ if !domainFound && cert.Subject.CommonName != domain {
399+ logger.Warn("Certificate doesn't match the required domain",
400+ "domain", domain,
401+ "cert_common_name", cert.Subject.CommonName,
402+ "cert_dns_names", cert.DNSNames)
403+ return false, 0, "unknown", cert
404+ }
405+
361 // Determine the certificate type406 // Determine the certificate type
362 certType := getCertificateType(cert)407 certType := getCertificateType(cert)
408+ logger.Debug("Determined certificate type",
409+ "domain", domain,
410+ "type", certType,
411+ "issuer", cert.Issuer.CommonName)
363 412
364 // Check if the certificate is still valid413 // Check if the certificate is still valid
365 now := time.Now()414 now := time.Now()
@@ -816,6 +865,46 @@ func (p *Proxy) requestAdminCertificate() {
816 return865 return
817 }866 }
818 867
868+ // First, check if certificate is already in the autocert cache
869+ // This is the critical step to avoid unnecessary Let's Encrypt requests
870+ cacheKey := "cert-" + adminDomain
871+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
872+ defer cancel()
873+
874+ if p.certManager != nil && p.certManager.Cache != nil {
875+ certData, err := p.certManager.Cache.Get(ctx, cacheKey)
876+ if err == nil && len(certData) > 0 {
877+ // Check if the cached certificate is valid
878+ isValid, expiresInDays, certType, _ := isCertificateValid(certData, adminDomain)
879+ if isValid {
880+ logger.Info("Using valid certificate from cache",
881+ "domain", adminDomain,
882+ "type", certType,
883+ "expires_in_days", expiresInDays)
884+ return
885+ }
886+
887+ // If certificate is invalid or expiring soon, log but continue
888+ logger.Info("Cached certificate is invalid or expiring soon",
889+ "domain", adminDomain,
890+ "expires_in_days", expiresInDays,
891+ "valid", isValid)
892+ }
893+ }
894+
895+ // Additional rate limit safety check
896+ // Check if we've recently hit a rate limit for this domain
897+ // This is a temporary memory-only safeguard
898+ rlCacheKey := adminDomain + "_ratelimit_history"
899+ rateLimitData, _ := p.certManager.Cache.Get(ctx, rlCacheKey)
900+ if len(rateLimitData) > 0 {
901+ logger.Warn("Rate limit history found for admin domain, skipping certificate request",
902+ "domain", adminDomain,
903+ "data", string(rateLimitData),
904+ "solution", "Wait for rate limit to expire")
905+ return
906+ }
907+
819 // Extract hostname from admin domain, resolving it to see if it's publicly accessible908 // Extract hostname from admin domain, resolving it to see if it's publicly accessible
820 host := adminDomain909 host := adminDomain
821 ips, err := net.LookupIP(host)910 ips, err := net.LookupIP(host)
@@ -907,7 +996,7 @@ func (p *Proxy) requestAdminCertificate() {
907 "timeout", "1 minute")996 "timeout", "1 minute")
908 997
909 // Create a context with timeout for the initial certificate request998 // Create a context with timeout for the initial certificate request
910- ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)999+ ctx, cancel = context.WithTimeout(context.Background(), 1*time.Minute)
911 defer cancel()1000 defer cancel()
912 1001
913 // Try to obtain the certificate with timeout1002 // Try to obtain the certificate with timeout
@@ -951,7 +1040,10 @@ func (p *Proxy) requestAdminCertificate() {
951 re := regexp.MustCompile(`retry after ([^:]+)`)1040 re := regexp.MustCompile(`retry after ([^:]+)`)
952 matches := re.FindStringSubmatch(certErr.Error())1041 matches := re.FindStringSubmatch(certErr.Error())
953 if len(matches) > 1 {1042 if len(matches) > 1 {
954- retryAfterStr = matches[1]1043+ retryAfterStr = strings.TrimSpace(matches[1])
1044+ if idx := strings.Index(retryAfterStr, ":"); idx > 0 {
1045+ retryAfterStr = retryAfterStr[:idx]
1046+ }
955 }1047 }
956 1048
957 logger.Error("Let's Encrypt rate limit reached",1049 logger.Error("Let's Encrypt rate limit reached",
@@ -960,6 +1052,27 @@ func (p *Proxy) requestAdminCertificate() {
960 "retry_after", retryAfterStr,1052 "retry_after", retryAfterStr,
961 "solution", "Wait until the rate limit expires, then restart the server")1053 "solution", "Wait until the rate limit expires, then restart the server")
962 1054
1055+ // Store rate limit history in cache to prevent further requests
1056+ rlCacheKey := adminDomain + "_ratelimit_history"
1057+ rlCtx, rlCancel := context.WithTimeout(context.Background(), 5*time.Second)
1058+ defer rlCancel()
1059+
1060+ // Store the error message and timestamp in the cache
1061+ rlData := []byte(fmt.Sprintf("Rate limited at %s: %s",
1062+ time.Now().Format(time.RFC3339), certErr.Error()))
1063+
1064+ if p.certManager != nil && p.certManager.Cache != nil {
1065+ // Store with a long expiration - Let's Encrypt rate limits are typically for a week
1066+ err := p.certManager.Cache.Put(rlCtx, rlCacheKey, rlData)
1067+ if err != nil {
1068+ logger.Debug("Failed to store rate limit history in cache",
1069+ "domain", adminDomain, "error", err)
1070+ } else {
1071+ logger.Info("Stored rate limit history in cache to prevent further requests",
1072+ "domain", adminDomain, "retry_after", retryAfterStr)
1073+ }
1074+ }
1075+
963 // Don't retry on rate limits1076 // Don't retry on rate limits
964 logger.Info("Skipping certificate request retries due to rate limiting",1077 logger.Info("Skipping certificate request retries due to rate limiting",
965 "domain", adminDomain)1078 "domain", adminDomain)
@@ -989,10 +1102,96 @@ func (p *Proxy) requestAdminCertificate() {
989 newCertPath := filepath.Join(certDir, newName)1102 newCertPath := filepath.Join(certDir, newName)
990 newKeyPath := filepath.Join(certDir, newName+"+rsa")1103 newKeyPath := filepath.Join(certDir, newName+"+rsa")
991 1104
1105+ // Debug log certificate file paths
1106+ logger.Debug("Certificate file paths",
1107+ "domain", adminDomain,
1108+ "old_cert", oldCertPath,
1109+ "new_cert", newCertPath,
1110+ "old_key", oldKeyPath,
1111+ "new_key", newKeyPath)
1112+
1113+ // Check if files exist first
1114+ oldCertExists := false
1115+ if _, err := os.Stat(oldCertPath); err == nil {
1116+ oldCertExists = true
1117+ logger.Debug("Old certificate file exists", "path", oldCertPath)
1118+ } else {
1119+ logger.Debug("Old certificate file not found", "path", oldCertPath, "error", err)
1120+ }
1121+
1122+ // Read the certificate files directly to verify they contain valid data
1123+ if oldCertExists {
1124+ certBytes, err := os.ReadFile(oldCertPath)
1125+ if err != nil {
1126+ logger.Error("Failed to read certificate file",
1127+ "path", oldCertPath,
1128+ "error", err)
1129+ } else {
1130+ logger.Debug("Successfully read certificate file",
1131+ "path", oldCertPath,
1132+ "size", len(certBytes))
1133+
1134+ // Verify it's a valid PEM format
1135+ block, _ := pem.Decode(certBytes)
1136+ if block == nil {
1137+ logger.Error("Certificate file does not contain valid PEM data",
1138+ "path", oldCertPath)
1139+ } else {
1140+ logger.Debug("Certificate file contains valid PEM data",
1141+ "path", oldCertPath,
1142+ "type", block.Type)
1143+ }
1144+ }
1145+
1146+ // Check permissions on cert and key files
1147+ certFileInfo, err := os.Stat(oldCertPath)
1148+ if err == nil {
1149+ mode := certFileInfo.Mode()
1150+ logger.Debug("Certificate file permissions",
1151+ "path", oldCertPath,
1152+ "mode", mode.String())
1153+
1154+ // Ensure cert file is readable by all
1155+ if mode.Perm()&0444 != 0444 {
1156+ logger.Warn("Certificate file has restrictive permissions, fixing",
1157+ "path", oldCertPath,
1158+ "current_mode", mode.String())
1159+ if err := os.Chmod(oldCertPath, 0644); err != nil {
1160+ logger.Error("Failed to fix certificate file permissions",
1161+ "path", oldCertPath,
1162+ "error", err)
1163+ }
1164+ }
1165+ }
1166+
1167+ // Check key file permissions
1168+ if _, err := os.Stat(oldKeyPath); err == nil {
1169+ keyFileInfo, err := os.Stat(oldKeyPath)
1170+ if err == nil {
1171+ mode := keyFileInfo.Mode()
1172+ logger.Debug("Key file permissions",
1173+ "path", oldKeyPath,
1174+ "mode", mode.String())
1175+
1176+ // Ensure key file is only readable by owner
1177+ if mode.Perm()&0400 != 0400 || mode.Perm()&0077 != 0 {
1178+ logger.Warn("Key file has incorrect permissions, fixing",
1179+ "path", oldKeyPath,
1180+ "current_mode", mode.String())
1181+ if err := os.Chmod(oldKeyPath, 0600); err != nil {
1182+ logger.Error("Failed to fix key file permissions",
1183+ "path", oldKeyPath,
1184+ "error", err)
1185+ }
1186+ }
1187+ }
1188+ }
1189+ }
1190+
992 // Only rename if the files exist and the new names are different1191 // Only rename if the files exist and the new names are different
993 if oldCertPath != newCertPath {1192 if oldCertPath != newCertPath {
994 // Check if old cert exists1193 // Check if old cert exists
995- if _, err := os.Stat(oldCertPath); err == nil {1194+ if oldCertExists {
996 // Rename cert1195 // Rename cert
997 if err := os.Rename(oldCertPath, newCertPath); err != nil {1196 if err := os.Rename(oldCertPath, newCertPath); err != nil {
998 logger.Warn("Failed to rename certificate file",1197 logger.Warn("Failed to rename certificate file",
@@ -1023,6 +1222,8 @@ func (p *Proxy) requestAdminCertificate() {
1023 "from", oldKeyPath,1222 "from", oldKeyPath,
1024 "to", newKeyPath)1223 "to", newKeyPath)
1025 }1224 }
1225+ } else {
1226+ logger.Warn("Old key file not found", "path", oldKeyPath, "error", err)
1026 }1227 }
1027 }1228 }
1028 }1229 }
@@ -1030,6 +1231,75 @@ func (p *Proxy) requestAdminCertificate() {
1030 logger.Info("Successfully obtained Let's Encrypt certificate",1231 logger.Info("Successfully obtained Let's Encrypt certificate",
1031 "domain", adminDomain,1232 "domain", adminDomain,
1032 "type", certType)1233 "type", certType)
1234+
1235+ // Verify the certificate files are accessible in the cache
1236+ cacheKey := "cert-" + adminDomain
1237+ cacheCtx, cacheCancel := context.WithTimeout(context.Background(), 5*time.Second)
1238+ defer cacheCancel()
1239+
1240+ certData, cacheErr := p.certManager.Cache.Get(cacheCtx, cacheKey)
1241+ if cacheErr != nil {
1242+ logger.Warn("Certificate not found in cache after successful obtainment",
1243+ "domain", adminDomain,
1244+ "error", cacheErr,
1245+ "solution", "Check certificate permissions and file paths")
1246+
1247+ // If certificate isn't in autocert cache but exists in filesystem, try to manually add it
1248+ newCertExists := false
1249+ if _, err := os.Stat(newCertPath); err == nil {
1250+ newCertExists = true
1251+ }
1252+
1253+ if oldCertExists || newCertExists {
1254+ logger.Info("Attempting to manually add certificate to cache from filesystem",
1255+ "domain", adminDomain)
1256+
1257+ sourceCertPath := newCertPath
1258+ sourceKeyPath := newKeyPath
1259+
1260+ // If the file wasn't renamed, use the old paths
1261+ if _, err := os.Stat(sourceCertPath); err != nil {
1262+ sourceCertPath = oldCertPath
1263+ sourceKeyPath = oldKeyPath
1264+ }
1265+
1266+ // Read the certificate and key files
1267+ certBytes, err := os.ReadFile(sourceCertPath)
1268+ if err != nil {
1269+ logger.Error("Failed to read certificate file for manual cache addition",
1270+ "path", sourceCertPath,
1271+ "error", err)
1272+ } else {
1273+ keyBytes, err := os.ReadFile(sourceKeyPath)
1274+ if err != nil {
1275+ logger.Error("Failed to read key file for manual cache addition",
1276+ "path", sourceKeyPath,
1277+ "error", err)
1278+ } else {
1279+ // Add to autocert cache
1280+ if err := p.certManager.Cache.Put(cacheCtx, cacheKey, certBytes); err != nil {
1281+ logger.Error("Failed to manually add certificate to cache",
1282+ "domain", adminDomain,
1283+ "error", err)
1284+ } else {
1285+ keyKey := "key-" + adminDomain
1286+ if err := p.certManager.Cache.Put(cacheCtx, keyKey, keyBytes); err != nil {
1287+ logger.Error("Failed to manually add key to cache",
1288+ "domain", adminDomain,
1289+ "error", err)
1290+ } else {
1291+ logger.Info("Successfully manually added certificate to cache",
1292+ "domain", adminDomain)
1293+ }
1294+ }
1295+ }
1296+ }
1297+ }
1298+ } else {
1299+ logger.Debug("Certificate successfully stored in cache",
1300+ "domain", adminDomain,
1301+ "cert_size", len(certData))
1302+ }
1033 } else {1303 } else {
1034 logger.Error("Unexpected error: received nil certificate but no error",1304 logger.Error("Unexpected error: received nil certificate but no error",
1035 "domain", adminDomain,1305 "domain", adminDomain,
@@ -1056,6 +1326,33 @@ func (p *Proxy) requestDomainCertificate(domain string) {
1056 return1326 return
1057 }1327 }
1058 1328
1329+ // First, check if certificate is already in the autocert cache
1330+ // This is the critical step to avoid unnecessary Let's Encrypt requests
1331+ cacheKey := "cert-" + domain
1332+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
1333+ defer cancel()
1334+
1335+ if p.certManager != nil && p.certManager.Cache != nil {
1336+ certData, err := p.certManager.Cache.Get(ctx, cacheKey)
1337+ if err == nil && len(certData) > 0 {
1338+ // Check if the cached certificate is valid
1339+ isValid, expiresInDays, certType, _ := isCertificateValid(certData, domain)
1340+ if isValid {
1341+ logger.Info("Using valid certificate from cache",
1342+ "domain", domain,
1343+ "type", certType,
1344+ "expires_in_days", expiresInDays)
1345+ return
1346+ }
1347+
1348+ // If certificate is invalid or expiring soon, log but continue
1349+ logger.Info("Cached certificate is invalid or expiring soon",
1350+ "domain", domain,
1351+ "expires_in_days", expiresInDays,
1352+ "valid", isValid)
1353+ }
1354+ }
1355+
1059 // Extract hostname from domain, resolving it to see if it's publicly accessible1356 // Extract hostname from domain, resolving it to see if it's publicly accessible
1060 ips, err := net.LookupIP(domain)1357 ips, err := net.LookupIP(domain)
1061 if err != nil {1358 if err != nil {
@@ -1100,6 +1397,46 @@ func (p *Proxy) requestDomainCertificate(domain string) {
1100 validInCache, _ := p.checkCertificateInCache(domain)1397 validInCache, _ := p.checkCertificateInCache(domain)
1101 validInFS, _ := p.checkCertificateInFilesystem(domain)1398 validInFS, _ := p.checkCertificateInFilesystem(domain)
1102 if !validInCache && validInFS {1399 if !validInCache && validInFS {
1400+ // Try to load the certificate into the cache if it's not there already
1401+ // This helps prevent future Let's Encrypt requests
1402+ hasOnDisk, certType := p.checkCertificateInFilesystem(domain)
1403+ if hasOnDisk {
1404+ logger.Info("Loading filesystem certificate into cache",
1405+ "domain", domain,
1406+ "type", certType)
1407+
1408+ // Get the certificate directory path for finding files
1409+ certDir := p.config.CertDir
1410+ if certDir == "" {
1411+ certDir = p.app.GetConfig().General.StorageDir + "/certs"
1412+ }
1413+
1414+ // Look for production certificate first, then staging
1415+ certFile := getCertificateFileName(domain, "production")
1416+ certPath := filepath.Join(certDir, certFile)
1417+ keyPath := filepath.Join(certDir, certFile+"+rsa")
1418+
1419+ // If production cert doesn't exist, try staging
1420+ if _, err := os.Stat(certPath); os.IsNotExist(err) {
1421+ certFile = getCertificateFileName(domain, "staging")
1422+ certPath = filepath.Join(certDir, certFile)
1423+ keyPath = filepath.Join(certDir, certFile+"+rsa")
1424+ }
1425+
1426+ // If no specific cert found, try the domain name directly
1427+ if _, err := os.Stat(certPath); os.IsNotExist(err) {
1428+ certPath = filepath.Join(certDir, domain)
1429+ keyPath = filepath.Join(certDir, domain+"+rsa")
1430+ }
1431+
1432+ // Add to cache if files exist
1433+ if _, err := os.Stat(certPath); err == nil {
1434+ if _, err := os.Stat(keyPath); err == nil {
1435+ p.addCertificateToCache(domain, certPath, keyPath)
1436+ }
1437+ }
1438+ }
1439+
1103 if testCertificateWithTLSConnection(domain, p.config.Port) {1440 if testCertificateWithTLSConnection(domain, p.config.Port) {
1104 logger.Debug("Certificate verification succeeded via TLS connection",1441 logger.Debug("Certificate verification succeeded via TLS connection",
1105 "domain", domain,1442 "domain", domain,
@@ -1116,6 +1453,18 @@ func (p *Proxy) requestDomainCertificate(domain string) {
1116 }1453 }
1117 }1454 }
1118 1455
1456+ // Additional rate limit safety check
1457+ // Check if we've recently hit a rate limit for this domain
1458+ // This is a temporary memory-only safeguard
1459+ cacheKey = domain + "_ratelimit_history"
1460+ rateLimitData, _ := p.certManager.Cache.Get(ctx, cacheKey)
1461+ if len(rateLimitData) > 0 {
1462+ logger.Warn("Rate limit history found for this domain, skipping certificate request",
1463+ "domain", domain,
1464+ "solution", "Wait for rate limit to expire")
1465+ return
1466+ }
1467+
1119 // Get the certificate directory path for naming the output files1468 // Get the certificate directory path for naming the output files
1120 certDir := p.config.CertDir1469 certDir := p.config.CertDir
1121 if certDir == "" {1470 if certDir == "" {
@@ -1146,7 +1495,7 @@ func (p *Proxy) requestDomainCertificate(domain string) {
1146 "timeout", "1 minute")1495 "timeout", "1 minute")
1147 1496
1148 // Create a context with timeout for the initial certificate request1497 // Create a context with timeout for the initial certificate request
1149- ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)1498+ ctx, cancel = context.WithTimeout(context.Background(), 1*time.Minute)
1150 defer cancel()1499 defer cancel()
1151 1500
1152 // Try to obtain the certificate with timeout1501 // Try to obtain the certificate with timeout
@@ -1190,7 +1539,10 @@ func (p *Proxy) requestDomainCertificate(domain string) {
1190 re := regexp.MustCompile(`retry after ([^:]+)`)1539 re := regexp.MustCompile(`retry after ([^:]+)`)
1191 matches := re.FindStringSubmatch(certErr.Error())1540 matches := re.FindStringSubmatch(certErr.Error())
1192 if len(matches) > 1 {1541 if len(matches) > 1 {
1193- retryAfterStr = matches[1]1542+ retryAfterStr = strings.TrimSpace(matches[1])
1543+ if idx := strings.Index(retryAfterStr, ":"); idx > 0 {
1544+ retryAfterStr = retryAfterStr[:idx]
1545+ }
1194 }1546 }
1195 1547
1196 logger.Error("Let's Encrypt rate limit reached",1548 logger.Error("Let's Encrypt rate limit reached",
@@ -1199,6 +1551,27 @@ func (p *Proxy) requestDomainCertificate(domain string) {
1199 "retry_after", retryAfterStr,1551 "retry_after", retryAfterStr,
1200 "solution", "Wait until the rate limit expires, then restart the server")1552 "solution", "Wait until the rate limit expires, then restart the server")
1201 1553
1554+ // Store rate limit history in cache to prevent further requests
1555+ rlCacheKey := domain + "_ratelimit_history"
1556+ rlCtx, rlCancel := context.WithTimeout(context.Background(), 5*time.Second)
1557+ defer rlCancel()
1558+
1559+ // Store the error message and timestamp in the cache
1560+ rlData := []byte(fmt.Sprintf("Rate limited at %s: %s",
1561+ time.Now().Format(time.RFC3339), certErr.Error()))
1562+
1563+ if p.certManager != nil && p.certManager.Cache != nil {
1564+ // Store with a long expiration - Let's Encrypt rate limits are typically for a week
1565+ err := p.certManager.Cache.Put(rlCtx, rlCacheKey, rlData)
1566+ if err != nil {
1567+ logger.Debug("Failed to store rate limit history in cache",
1568+ "domain", domain, "error", err)
1569+ } else {
1570+ logger.Info("Stored rate limit history in cache to prevent further requests",
1571+ "domain", domain, "retry_after", retryAfterStr)
1572+ }
1573+ }
1574+
1202 // Don't retry on rate limits1575 // Don't retry on rate limits
1203 logger.Info("Skipping certificate request retries due to rate limiting",1576 logger.Info("Skipping certificate request retries due to rate limiting",
1204 "domain", domain)1577 "domain", domain)
@@ -1262,6 +1635,8 @@ func (p *Proxy) requestDomainCertificate(domain string) {
1262 "from", oldKeyPath,1635 "from", oldKeyPath,
1263 "to", newKeyPath)1636 "to", newKeyPath)
1264 }1637 }
1638+ } else {
1639+ logger.Warn("Old key file not found", "path", oldKeyPath, "error", err)
1265 }1640 }
1266 }1641 }
1267 }1642 }
@@ -1393,6 +1768,8 @@ func (p *Proxy) retryCertificateRequest(domain string, maxRetries int, initialBa
1393 "from", oldKeyPath,1768 "from", oldKeyPath,
1394 "to", newKeyPath)1769 "to", newKeyPath)
1395 }1770 }
1771+ } else {
1772+ logger.Warn("Old key file not found", "path", oldKeyPath, "error", err)
1396 }1773 }
1397 }1774 }
1398 }1775 }
@@ -1585,3 +1962,319 @@ func generateFallbackCertificates(domains []string) (*tls.Certificate, error) {
1585 1962
1586 return &cert, nil1963 return &cert, nil
1587 }1964 }
1965+
1966+// preloadCertificates scans the certificate directory and loads all valid certificates
1967+// into the autocert cache to prevent unnecessary requests to Let's Encrypt
1968+func (p *Proxy) preloadCertificates() {
1969+ if p.certManager == nil || p.certManager.Cache == nil {
1970+ logger.Warn("Cannot preload certificates: certificate manager or cache not initialized")
1971+ return
1972+ }
1973+
1974+ // Get the certificate directory path
1975+ certDir := p.config.CertDir
1976+ if certDir == "" {
1977+ certDir = p.app.GetConfig().General.StorageDir + "/certs"
1978+ }
1979+
1980+ logger.Info("Preloading certificates from directory", "dir", certDir)
1981+
1982+ // Ensure directory exists
1983+ if _, err := os.Stat(certDir); os.IsNotExist(err) {
1984+ logger.Warn("Certificate directory does not exist, creating it", "dir", certDir)
1985+ if err := os.MkdirAll(certDir, 0755); err != nil {
1986+ logger.Error("Failed to create certificate directory", "dir", certDir, "error", err)
1987+ return
1988+ }
1989+ }
1990+
1991+ // List all files in the certificate directory
1992+ files, err := os.ReadDir(certDir)
1993+ if err != nil {
1994+ logger.Error("Failed to read certificate directory", "dir", certDir, "error", err)
1995+ return
1996+ }
1997+
1998+ // Log all files in the directory to help troubleshoot
1999+ var fileNames []string
2000+ for _, file := range files {
2001+ fileNames = append(fileNames, file.Name())
2002+
2003+ // Check file permissions and fix if too restrictive
2004+ filePath := filepath.Join(certDir, file.Name())
2005+ info, err := os.Stat(filePath)
2006+ if err != nil {
2007+ logger.Warn("Could not stat file", "path", filePath, "error", err)
2008+ continue
2009+ }
2010+
2011+ mode := info.Mode()
2012+ if strings.HasSuffix(file.Name(), "+rsa") {
2013+ // Key files should be 0600 (owner read/write only)
2014+ if mode.Perm() != 0600 {
2015+ logger.Info("Fixing key file permissions", "file", file.Name(), "current_mode", mode.String(), "target_mode", "0600")
2016+ if err := os.Chmod(filePath, 0600); err != nil {
2017+ logger.Warn("Failed to fix key file permissions", "file", file.Name(), "error", err)
2018+ }
2019+ }
2020+ } else {
2021+ // Certificate files should be 0644 (readable by all)
2022+ if mode.Perm() != 0644 {
2023+ logger.Info("Fixing certificate file permissions", "file", file.Name(), "current_mode", mode.String(), "target_mode", "0644")
2024+ if err := os.Chmod(filePath, 0644); err != nil {
2025+ logger.Warn("Failed to fix certificate file permissions", "file", file.Name(), "error", err)
2026+ }
2027+ }
2028+ }
2029+ }
2030+
2031+ logger.Debug("Files in certificate directory", "dir", certDir, "files", fileNames)
2032+
2033+ // Track the number of certificates loaded
2034+ loadedCount := 0
2035+
2036+ // Create a context for cache operations
2037+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
2038+ defer cancel()
2039+
2040+ // Find and process certificate files (excluding key files with +rsa suffix)
2041+ // Group them by domain for easier processing
2042+ domainCerts := make(map[string][]string)
2043+
2044+ for _, file := range files {
2045+ name := file.Name()
2046+
2047+ // Skip account keys and +rsa key files as we'll handle these separately
2048+ if strings.HasSuffix(name, "+rsa") || strings.Contains(name, "acme_account") {
2049+ continue
2050+ }
2051+
2052+ // Get associated key file name (file+rsa)
2053+ keyName := name + "+rsa"
2054+ keyPath := filepath.Join(certDir, keyName)
2055+
2056+ // Skip certificates without key files
2057+ if _, err := os.Stat(keyPath); os.IsNotExist(err) {
2058+ logger.Debug("Skipping certificate without key file", "cert", name, "key", keyName)
2059+ continue
2060+ }
2061+
2062+ // Try to determine the domain from the filename
2063+ domain := name
2064+
2065+ // Handle special patterns like domain_letsencrypt-production
2066+ for _, pattern := range []string{"_letsencrypt-production", "_letsencrypt-staging", "_self-signed"} {
2067+ if strings.HasSuffix(domain, pattern) {
2068+ domain = strings.TrimSuffix(domain, pattern)
2069+ break
2070+ }
2071+ }
2072+
2073+ // Add to domain certificates map
2074+ domainCerts[domain] = append(domainCerts[domain], name)
2075+ }
2076+
2077+ // Process certificates for each domain
2078+ for domain, certFiles := range domainCerts {
2079+ logger.Debug("Found certificate files for domain", "domain", domain, "files", certFiles)
2080+
2081+ // Prioritize production certificates if multiple exist
2082+ var bestCert string
2083+ bestCertType := ""
2084+
2085+ for _, certFile := range certFiles {
2086+ certPath := filepath.Join(certDir, certFile)
2087+
2088+ // Read certificate to check its validity
2089+ certData, err := os.ReadFile(certPath)
2090+ if err != nil {
2091+ logger.Warn("Failed to read certificate file", "path", certPath, "error", err)
2092+ continue
2093+ }
2094+
2095+ // Debug certificate content
2096+ if len(certData) < 100 {
2097+ logger.Warn("Certificate file too small, may be corrupted",
2098+ "path", certPath, "size", len(certData))
2099+ continue
2100+ }
2101+
2102+ // Validate the certificate
2103+ valid, expiresInDays, certType, _ := isCertificateValid(certData, domain)
2104+ if !valid {
2105+ logger.Debug("Invalid certificate found", "domain", domain, "file", certFile)
2106+ continue
2107+ }
2108+
2109+ logger.Info("Found valid certificate", "domain", domain, "type", certType,
2110+ "expires_in_days", expiresInDays, "file", certFile)
2111+
2112+ // Prioritize production certificates over staging ones
2113+ if bestCertType == "" ||
2114+ (bestCertType != "production" && certType == "production") {
2115+ bestCert = certFile
2116+ bestCertType = certType
2117+ }
2118+ }
2119+
2120+ // If we found a valid certificate for this domain, add it to the cache
2121+ if bestCert != "" {
2122+ certPath := filepath.Join(certDir, bestCert)
2123+ keyPath := filepath.Join(certDir, bestCert+"+rsa")
2124+
2125+ // Read certificate and key data
2126+ certData, err := os.ReadFile(certPath)
2127+ if err != nil {
2128+ logger.Warn("Failed to read certificate file", "path", certPath, "error", err)
2129+ continue
2130+ }
2131+
2132+ keyData, err := os.ReadFile(keyPath)
2133+ if err != nil {
2134+ logger.Warn("Failed to read key file", "path", keyPath, "error", err)
2135+ continue
2136+ }
2137+
2138+ // Add certificate to autocert cache
2139+ cacheKey := "cert-" + domain
2140+ err = p.certManager.Cache.Put(ctx, cacheKey, certData)
2141+ if err != nil {
2142+ logger.Error("Failed to add certificate to cache", "domain", domain, "error", err)
2143+ continue
2144+ }
2145+
2146+ // Add key to cache
2147+ keyKey := "key-" + domain
2148+ err = p.certManager.Cache.Put(ctx, keyKey, keyData)
2149+ if err != nil {
2150+ logger.Error("Failed to add key to cache", "domain", domain, "error", err)
2151+ continue
2152+ }
2153+
2154+ logger.Info("Preloaded certificate into cache", "domain", domain, "type", bestCertType)
2155+ loadedCount++
2156+ } else {
2157+ logger.Warn("No valid certificate found for domain after checking all files",
2158+ "domain", domain,
2159+ "files_checked", certFiles)
2160+ }
2161+ }
2162+
2163+ logger.Info("Certificate preloading complete", "loaded_certificates", loadedCount)
2164+}
2165+
2166+// manuallyLoadCertificate is a fallback method to load certificates directly
2167+// when autocert cache is failing. This directly builds the tls.Certificate from files.
2168+func (p *Proxy) manuallyLoadCertificate(domain string) (*tls.Certificate, error) {
2169+ // Get certificate directory
2170+ certDir := p.config.CertDir
2171+ if certDir == "" {
2172+ certDir = p.app.GetConfig().General.StorageDir + "/certs"
2173+ }
2174+
2175+ // Check preferred file naming patterns
2176+ filePatterns := []string{
2177+ // Try production certificate first
2178+ fmt.Sprintf(LetsEncryptProdPattern, domain),
2179+ // Then staging certificate
2180+ fmt.Sprintf(LetsEncryptStagingPattern, domain),
2181+ // Then self-signed
2182+ fmt.Sprintf(SelfSignedCertPattern, domain),
2183+ // Finally plain domain name
2184+ domain,
2185+ }
2186+
2187+ for _, pattern := range filePatterns {
2188+ certPath := filepath.Join(certDir, pattern)
2189+ keyPath := filepath.Join(certDir, pattern+"+rsa")
2190+
2191+ // Check if both files exist
2192+ if _, err := os.Stat(certPath); os.IsNotExist(err) {
2193+ continue
2194+ }
2195+ if _, err := os.Stat(keyPath); os.IsNotExist(err) {
2196+ continue
2197+ }
2198+
2199+ // Try to load the certificate pair
2200+ cert, err := tls.LoadX509KeyPair(certPath, keyPath)
2201+ if err != nil {
2202+ logger.Warn("Failed to load certificate key pair",
2203+ "domain", domain,
2204+ "certPath", certPath,
2205+ "keyPath", keyPath,
2206+ "error", err)
2207+
2208+ // Additional diagnostics
2209+ if data, readErr := os.ReadFile(certPath); readErr == nil {
2210+ logger.Debug("Certificate file content preview",
2211+ "path", certPath,
2212+ "size", len(data),
2213+ "preview", string(data[:min(100, len(data))]))
2214+
2215+ // Try to parse the certificate - this may help diagnose format issues
2216+ block, _ := pem.Decode(data)
2217+ if block == nil {
2218+ logger.Error("Certificate file does not contain valid PEM data",
2219+ "path", certPath)
2220+ } else {
2221+ logger.Debug("Certificate PEM block info",
2222+ "type", block.Type,
2223+ "headers", block.Headers)
2224+ }
2225+ }
2226+
2227+ continue
2228+ }
2229+
2230+ // Success!
2231+ logger.Info("Manually loaded certificate from filesystem",
2232+ "domain", domain,
2233+ "certPath", certPath)
2234+ return &cert, nil
2235+ }
2236+
2237+ // Also try with glob pattern
2238+ filePaths, err := filepath.Glob(filepath.Join(certDir, domain+"*"))
2239+ if err != nil {
2240+ return nil, fmt.Errorf("glob error: %w", err)
2241+ }
2242+
2243+ for _, certPath := range filePaths {
2244+ // Skip if it's an RSA key
2245+ if strings.HasSuffix(certPath, "+rsa") {
2246+ continue
2247+ }
2248+
2249+ // Check if key exists
2250+ keyPath := certPath + "+rsa"
2251+ if _, err := os.Stat(keyPath); os.IsNotExist(err) {
2252+ continue
2253+ }
2254+
2255+ // Try to load
2256+ cert, err := tls.LoadX509KeyPair(certPath, keyPath)
2257+ if err != nil {
2258+ logger.Debug("Failed to load certificate from glob match",
2259+ "domain", domain,
2260+ "certPath", certPath,
2261+ "error", err)
2262+ continue
2263+ }
2264+
2265+ logger.Info("Manually loaded certificate from glob search",
2266+ "domain", domain,
2267+ "certPath", certPath)
2268+ return &cert, nil
2269+ }
2270+
2271+ return nil, fmt.Errorf("no valid certificate found for domain %s", domain)
2272+}
2273+
2274+// min returns the smaller of x or y.
2275+func min(x, y int) int {
2276+ if x < y {
2277+ return x
2278+ }
2279+ return y
2280+}
M internal/proxy/core.go +52 -39
@@ -1,8 +1,7 @@
11 package proxy
22
33 import (
4- "os"
5- "strings"
4+ "net"
65 "sync"
76 "time"
87
@@ -108,58 +107,72 @@ func NewProxy(app interfaces.AppInterface) (*Proxy, error) {
108107 httpServer := echo.New()
109108 httpServer.HideBanner = true
110109
111- // Detect and store our container ID (if we're running in a container)
112- var ourContainerID string
113- if docker.IsRunningInContainer() {
114- // Get our hostname which should be the container ID in Docker/Podman
115- hostname := os.Getenv("HOSTNAME")
116- if hostname != "" {
117- containers, err := docker.ListRunningContainers()
118- if err == nil {
119- for _, container := range containers {
120- // Check if this container is us by comparing hostname with container ID
121- if strings.HasPrefix(hostname, container.ID) {
122- // Store our container ID for future reference
123- ourContainerID = container.ID
124- containerName := strings.TrimLeft(container.Names[0], "/")
125- logger.Info("Gordon identity established",
126- "container_id", ourContainerID,
127- "container_name", containerName)
128- break
129- }
130- }
131- }
132- }
110+ // Initialize container storage and cooldown settings
111+ recentContainers := make(map[string]time.Time)
112+ containerCooldownPeriod := time.Duration(10) * time.Second
113+ // Use a fixed cooldown period of 10 seconds (currently no config option for this)
114+
115+ // Set up DB retry configuration
116+ dbMaxRetries := 3
117+ dbRetryBaseDelay := time.Duration(500) * time.Millisecond
118+
119+ // Initialize the reverseProxyClient with reasonable timeouts
120+ reverseProxyClient := &http.Client{
121+ Timeout: 30 * time.Second,
122+ Transport: &http.Transport{
123+ TLSClientConfig: &tls.Config{
124+ InsecureSkipVerify: true,
125+ },
126+ // Add more detailed timeout settings
127+ TLSHandshakeTimeout: 10 * time.Second,
128+ ResponseHeaderTimeout: 10 * time.Second,
129+ ExpectContinueTimeout: 5 * time.Second,
130+ // Add a custom dialer with explicit timeouts
131+ DialContext: (&net.Dialer{
132+ Timeout: 10 * time.Second,
133+ KeepAlive: 30 * time.Second,
134+ DualStack: true,
135+ }).DialContext,
136+ MaxIdleConns: 100,
137+ MaxIdleConnsPerHost: 10,
138+ IdleConnTimeout: 90 * time.Second,
139+ ForceAttemptHTTP2: true,
140+ },
133141 }
134142
135- // Initialize routes map
136- routes := make(map[string]*ProxyRouteInfo)
137-
138- // Initialize SQL queries
143+ // Initialize the proxy queries
139144 proxyQueries := queries.NewProxyQueries()
140145
141- // Create the proxy
146+ // Create the proxy instance
142147 p := &Proxy{
143148 config: config,
144149 app: app,
145150 httpsServer: httpsServer,
146151 httpServer: httpServer,
147- routes: routes,
148- serverStarted: false,
149- gordonContainerID: ourContainerID,
152+ routes: make(map[string]*ProxyRouteInfo),
153+ reverseProxyClient: reverseProxyClient,
154+ dbMaxRetries: dbMaxRetries,
155+ dbRetryBaseDelay: dbRetryBaseDelay,
156+ recentContainers: recentContainers,
157+ containerCooldownPeriod: containerCooldownPeriod,
150158 shutdown: make(chan struct{}),
151- recentContainers: make(map[string]time.Time),
152- containerCooldownPeriod: 10 * time.Second, // Default 10 second cooldown period
153159 queries: proxyQueries,
154160 }
155161
156- // Now add the middleware with the proxy reference
157- p.setupMiddleware()
158-
159- // Set up the certificate manager
162+ // Setup the certificate manager for HTTPS
160163 p.setupCertManager()
161164
162- logger.Debug("Reverse proxy initialized")
165+ // Verify the certificate manager was properly initialized
166+ if p.certManager == nil {
167+ logger.Error("Certificate manager initialization failed",
168+ "solution", "Check certificate directory permissions and configuration")
169+ } else {
170+ logger.Debug("Certificate manager successfully initialized")
171+ }
172+
173+ // Detect our container ID for later use
174+ p.detectGordonContainer()
175+
163176 return p, nil
164177 }
165178
@@ -1,8 +1,7 @@
1 package proxy1 package proxy
2 2
3 import (3 import (
4- "os"4+ "net"
5- "strings"
6 "sync"5 "sync"
7 "time"6 "time"
8 7
@@ -108,58 +107,72 @@ func NewProxy(app interfaces.AppInterface) (*Proxy, error) {
108 httpServer := echo.New()107 httpServer := echo.New()
109 httpServer.HideBanner = true108 httpServer.HideBanner = true
110 109
111- // Detect and store our container ID (if we're running in a container)110+ // Initialize container storage and cooldown settings
112- var ourContainerID string111+ recentContainers := make(map[string]time.Time)
113- if docker.IsRunningInContainer() {112+ containerCooldownPeriod := time.Duration(10) * time.Second
114- // Get our hostname which should be the container ID in Docker/Podman113+ // Use a fixed cooldown period of 10 seconds (currently no config option for this)
115- hostname := os.Getenv("HOSTNAME")114+
116- if hostname != "" {115+ // Set up DB retry configuration
117- containers, err := docker.ListRunningContainers()116+ dbMaxRetries := 3
118- if err == nil {117+ dbRetryBaseDelay := time.Duration(500) * time.Millisecond
119- for _, container := range containers {118+
120- // Check if this container is us by comparing hostname with container ID119+ // Initialize the reverseProxyClient with reasonable timeouts
121- if strings.HasPrefix(hostname, container.ID) {120+ reverseProxyClient := &http.Client{
122- // Store our container ID for future reference121+ Timeout: 30 * time.Second,
123- ourContainerID = container.ID122+ Transport: &http.Transport{
124- containerName := strings.TrimLeft(container.Names[0], "/")123+ TLSClientConfig: &tls.Config{
125- logger.Info("Gordon identity established",124+ InsecureSkipVerify: true,
126- "container_id", ourContainerID,125+ },
127- "container_name", containerName)126+ // Add more detailed timeout settings
128- break127+ TLSHandshakeTimeout: 10 * time.Second,
129- }128+ ResponseHeaderTimeout: 10 * time.Second,
130- }129+ ExpectContinueTimeout: 5 * time.Second,
131- }130+ // Add a custom dialer with explicit timeouts
132- }131+ DialContext: (&net.Dialer{
132+ Timeout: 10 * time.Second,
133+ KeepAlive: 30 * time.Second,
134+ DualStack: true,
135+ }).DialContext,
136+ MaxIdleConns: 100,
137+ MaxIdleConnsPerHost: 10,
138+ IdleConnTimeout: 90 * time.Second,
139+ ForceAttemptHTTP2: true,
140+ },
133 }141 }
134 142
135- // Initialize routes map143+ // Initialize the proxy queries
136- routes := make(map[string]*ProxyRouteInfo)
137-
138- // Initialize SQL queries
139 proxyQueries := queries.NewProxyQueries()144 proxyQueries := queries.NewProxyQueries()
140 145
141- // Create the proxy146+ // Create the proxy instance
142 p := &Proxy{147 p := &Proxy{
143 config: config,148 config: config,
144 app: app,149 app: app,
145 httpsServer: httpsServer,150 httpsServer: httpsServer,
146 httpServer: httpServer,151 httpServer: httpServer,
147- routes: routes,152+ routes: make(map[string]*ProxyRouteInfo),
148- serverStarted: false,153+ reverseProxyClient: reverseProxyClient,
149- gordonContainerID: ourContainerID,154+ dbMaxRetries: dbMaxRetries,
155+ dbRetryBaseDelay: dbRetryBaseDelay,
156+ recentContainers: recentContainers,
157+ containerCooldownPeriod: containerCooldownPeriod,
150 shutdown: make(chan struct{}),158 shutdown: make(chan struct{}),
151- recentContainers: make(map[string]time.Time),
152- containerCooldownPeriod: 10 * time.Second, // Default 10 second cooldown period
153 queries: proxyQueries,159 queries: proxyQueries,
154 }160 }
155 161
156- // Now add the middleware with the proxy reference162+ // Setup the certificate manager for HTTPS
157- p.setupMiddleware()
158-
159- // Set up the certificate manager
160 p.setupCertManager()163 p.setupCertManager()
161 164
162- logger.Debug("Reverse proxy initialized")165+ // Verify the certificate manager was properly initialized
166+ if p.certManager == nil {
167+ logger.Error("Certificate manager initialization failed",
168+ "solution", "Check certificate directory permissions and configuration")
169+ } else {
170+ logger.Debug("Certificate manager successfully initialized")
171+ }
172+
173+ // Detect our container ID for later use
174+ p.detectGordonContainer()
175+
163 return p, nil176 return p, nil
164 }177 }
165 178
M internal/proxy/middleware.go +6 -6
@@ -31,7 +31,7 @@ func (p *Proxy) setupMiddleware() {
3131 p.httpServer.Use(p.createRequestIDMiddleware())
3232
3333 // Check if rate limiting is disabled via configuration
34- if !p.config.DisableRateLimit {
34+ if p.config.EnableRateLimit {
3535 // Create a new Starskey rate limiter store for HTTPS server
3636 httpsRateLimiter, err := kv.NewStarskeyRateLimiterStore(
3737 "./data/ratelimiter/https", // Path to store rate limit data
@@ -98,12 +98,12 @@ func (p *Proxy) setupMiddleware() {
9898 }
9999
100100 // Add custom logger middleware with request ID if enabled in config
101- if p.config.EnableLogs {
101+ if p.config.EnableHttpLogs {
102102 p.httpsServer.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{
103103 Format: `{"time":"${time_rfc3339_nano}","id":"${id}","remote_ip":"${remote_ip}","host":"${host}","method":"${method}","uri":"${uri}","user_agent":"${user_agent}","status":${status},"error":"${error}","latency":${latency},"latency_human":"${latency_human}","bytes_in":${bytes_in},"bytes_out":${bytes_out}}` + "\n",
104104 CustomTimeFormat: "2006-01-02T15:04:05.00000Z07:00",
105105 Skipper: func(c echo.Context) bool {
106- return !p.config.EnableLogs
106+ return !p.config.EnableHttpLogs
107107 },
108108 }))
109109 logger.Debug("HTTP request logging enabled for HTTPS server")
@@ -112,7 +112,7 @@ func (p *Proxy) setupMiddleware() {
112112 }
113113
114114 // Check if rate limiting is disabled via configuration
115- if !p.config.DisableRateLimit {
115+ if p.config.EnableRateLimit {
116116 // Create a new Starskey rate limiter store for HTTP server
117117 httpRateLimiter, err := kv.NewStarskeyRateLimiterStore(
118118 "./data/ratelimiter/http", // Path to store rate limit data
@@ -179,12 +179,12 @@ func (p *Proxy) setupMiddleware() {
179179 }
180180
181181 // Add custom logger middleware with request ID for HTTP server if enabled in config
182- if p.config.EnableLogs {
182+ if p.config.EnableHttpLogs {
183183 p.httpServer.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{
184184 Format: `{"time":"${time_rfc3339_nano}","id":"${id}","remote_ip":"${remote_ip}","host":"${host}","method":"${method}","uri":"${uri}","user_agent":"${user_agent}","status":${status},"error":"${error}","latency":${latency},"latency_human":"${latency_human}","bytes_in":${bytes_in},"bytes_out":${bytes_out}}` + "\n",
185185 CustomTimeFormat: "2006-01-02T15:04:05.00000Z07:00",
186186 Skipper: func(c echo.Context) bool {
187- return !p.config.EnableLogs
187+ return !p.config.EnableHttpLogs
188188 },
189189 }))
190190 logger.Debug("HTTP request logging enabled for HTTP server")
@@ -31,7 +31,7 @@ func (p *Proxy) setupMiddleware() {
31 p.httpServer.Use(p.createRequestIDMiddleware())31 p.httpServer.Use(p.createRequestIDMiddleware())
32 32
33 // Check if rate limiting is disabled via configuration33 // Check if rate limiting is disabled via configuration
34- if !p.config.DisableRateLimit {34+ if p.config.EnableRateLimit {
35 // Create a new Starskey rate limiter store for HTTPS server35 // Create a new Starskey rate limiter store for HTTPS server
36 httpsRateLimiter, err := kv.NewStarskeyRateLimiterStore(36 httpsRateLimiter, err := kv.NewStarskeyRateLimiterStore(
37 "./data/ratelimiter/https", // Path to store rate limit data37 "./data/ratelimiter/https", // Path to store rate limit data
@@ -98,12 +98,12 @@ func (p *Proxy) setupMiddleware() {
98 }98 }
99 99
100 // Add custom logger middleware with request ID if enabled in config100 // Add custom logger middleware with request ID if enabled in config
101- if p.config.EnableLogs {101+ if p.config.EnableHttpLogs {
102 p.httpsServer.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{102 p.httpsServer.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{
103 Format: `{"time":"${time_rfc3339_nano}","id":"${id}","remote_ip":"${remote_ip}","host":"${host}","method":"${method}","uri":"${uri}","user_agent":"${user_agent}","status":${status},"error":"${error}","latency":${latency},"latency_human":"${latency_human}","bytes_in":${bytes_in},"bytes_out":${bytes_out}}` + "\n",103 Format: `{"time":"${time_rfc3339_nano}","id":"${id}","remote_ip":"${remote_ip}","host":"${host}","method":"${method}","uri":"${uri}","user_agent":"${user_agent}","status":${status},"error":"${error}","latency":${latency},"latency_human":"${latency_human}","bytes_in":${bytes_in},"bytes_out":${bytes_out}}` + "\n",
104 CustomTimeFormat: "2006-01-02T15:04:05.00000Z07:00",104 CustomTimeFormat: "2006-01-02T15:04:05.00000Z07:00",
105 Skipper: func(c echo.Context) bool {105 Skipper: func(c echo.Context) bool {
106- return !p.config.EnableLogs106+ return !p.config.EnableHttpLogs
107 },107 },
108 }))108 }))
109 logger.Debug("HTTP request logging enabled for HTTPS server")109 logger.Debug("HTTP request logging enabled for HTTPS server")
@@ -112,7 +112,7 @@ func (p *Proxy) setupMiddleware() {
112 }112 }
113 113
114 // Check if rate limiting is disabled via configuration114 // Check if rate limiting is disabled via configuration
115- if !p.config.DisableRateLimit {115+ if p.config.EnableRateLimit {
116 // Create a new Starskey rate limiter store for HTTP server116 // Create a new Starskey rate limiter store for HTTP server
117 httpRateLimiter, err := kv.NewStarskeyRateLimiterStore(117 httpRateLimiter, err := kv.NewStarskeyRateLimiterStore(
118 "./data/ratelimiter/http", // Path to store rate limit data118 "./data/ratelimiter/http", // Path to store rate limit data
@@ -179,12 +179,12 @@ func (p *Proxy) setupMiddleware() {
179 }179 }
180 180
181 // Add custom logger middleware with request ID for HTTP server if enabled in config181 // Add custom logger middleware with request ID for HTTP server if enabled in config
182- if p.config.EnableLogs {182+ if p.config.EnableHttpLogs {
183 p.httpServer.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{183 p.httpServer.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{
184 Format: `{"time":"${time_rfc3339_nano}","id":"${id}","remote_ip":"${remote_ip}","host":"${host}","method":"${method}","uri":"${uri}","user_agent":"${user_agent}","status":${status},"error":"${error}","latency":${latency},"latency_human":"${latency_human}","bytes_in":${bytes_in},"bytes_out":${bytes_out}}` + "\n",184 Format: `{"time":"${time_rfc3339_nano}","id":"${id}","remote_ip":"${remote_ip}","host":"${host}","method":"${method}","uri":"${uri}","user_agent":"${user_agent}","status":${status},"error":"${error}","latency":${latency},"latency_human":"${latency_human}","bytes_in":${bytes_in},"bytes_out":${bytes_out}}` + "\n",
185 CustomTimeFormat: "2006-01-02T15:04:05.00000Z07:00",185 CustomTimeFormat: "2006-01-02T15:04:05.00000Z07:00",
186 Skipper: func(c echo.Context) bool {186 Skipper: func(c echo.Context) bool {
187- return !p.config.EnableLogs187+ return !p.config.EnableHttpLogs
188 },188 },
189 }))189 }))
190 logger.Debug("HTTP request logging enabled for HTTP server")190 logger.Debug("HTTP request logging enabled for HTTP server")
M internal/proxy/routes.go +148 -71
@@ -1,6 +1,7 @@
11 package proxy
22
33 import (
4+ "crypto/tls"
45 "database/sql"
56 "fmt"
67 "net"
@@ -13,6 +14,7 @@ import (
1314
1415 "github.com/bnema/gordon/pkg/docker"
1516 "github.com/bnema/gordon/pkg/logger"
17+ "github.com/google/uuid"
1618 "github.com/labstack/echo/v4"
1719 )
1820
@@ -157,78 +159,14 @@ func (p *Proxy) configureRoutes() {
157159 logger.Debug("Admin domain route exists", "domain", adminDomain)
158160 }
159161
160- // Handler function - will be assigned to both HTTP and HTTPS servers
161- handler := echo.HandlerFunc(func(c echo.Context) error {
162- host := c.Request().Host
163-
164- // Strip port if present in the host
165- if hostParts := strings.Split(host, ":"); len(hostParts) > 1 {
166- host = hostParts[0]
167- }
168-
169- // Normalize host (trim trailing dot, convert to lowercase)
170- host = strings.TrimSuffix(strings.ToLower(host), ".")
171-
172- // Check if this is the admin domain
173- adminDomain := p.app.GetConfig().Http.FullDomain()
174- if strings.EqualFold(host, adminDomain) {
175- // Always prioritize admin domain
176- p.mu.RLock()
177- adminRoute, adminOk := p.routes[adminDomain]
178- p.mu.RUnlock()
179-
180- if adminOk {
181- return p.proxyRequest(c, adminRoute)
182- }
183- }
184-
185- // Important: Get the route freshly from the map for every request
186- // This ensures we always use the most up-to-date IP
187- p.mu.RLock()
188- route, ok := p.routes[host]
189- p.mu.RUnlock()
190-
191- if !ok {
192- // Check if the host is an IP address - silently handle without logging warnings
193- if net.ParseIP(host) != nil {
194- // For IP-based requests, just return a 404 without logging warnings
195- return c.String(http.StatusNotFound, "Domain not found")
196- }
197-
198- // Create list of available domains for debugging
199- availableDomains := make([]string, 0, len(p.routes))
200- p.mu.RLock()
201- for d := range p.routes {
202- availableDomains = append(availableDomains, d)
203- }
204- p.mu.RUnlock()
205-
206- // Log warning for non-IP hosts that aren't configured
207- logger.Warn("Request with unknown host",
208- "requested_host", host,
209- "client_ip", c.RealIP(),
210- "available_domains", strings.Join(availableDomains, ", "))
211- return c.String(http.StatusNotFound, "Domain not found")
212- }
213-
214- // Check if the route is active
215- if !route.Active {
216- logger.Warn("Route is not active",
217- "domain", host,
218- "client_ip", c.RealIP())
219- return c.String(http.StatusServiceUnavailable, "Route is not active")
220- }
221- return p.proxyRequest(c, route)
222- })
223-
224162 // Add the handler to the HTTPS server
225163 if p.httpsServer != nil {
226- p.httpsServer.Any("/*", handler)
164+ p.httpsServer.Any("/*", p.proxyRequest)
227165 }
228166
229167 // Also add the handler to the HTTP server for HTTP-01 challenges
230168 if p.httpServer != nil {
231- p.httpServer.Any("/*", handler)
169+ p.httpServer.Any("/*", p.proxyRequest)
232170 }
233171 }
234172
@@ -308,7 +246,7 @@ func (p *Proxy) AddRoute(domainName, containerID, containerIP, containerPort, pr
308246 if err != nil {
309247 if err == sql.ErrNoRows {
310248 // Domain doesn't exist, create it
311- domainID = proxyGenerateUUID()
249+ domainID = uuid.New().String()
312250 now := time.Now().Format(time.RFC3339)
313251 _, err = txExecWithRetry(tx,
314252 p.queries.InsertDomain,
@@ -338,7 +276,7 @@ func (p *Proxy) AddRoute(domainName, containerID, containerIP, containerPort, pr
338276 if err != nil {
339277 if err == sql.ErrNoRows {
340278 // Domain part doesn't exist, create it
341- domainPartID = proxyGenerateUUID()
279+ domainPartID = uuid.New().String()
342280 _, err = txExecWithRetry(tx,
343281 p.queries.InsertDomain,
344282 domainPartID, domainPart, accountID, now, now,
@@ -408,7 +346,7 @@ func (p *Proxy) AddRoute(domainName, containerID, containerIP, containerPort, pr
408346 }
409347 } else if err == sql.ErrNoRows {
410348 // Route doesn't exist, create it
411- routeID := proxyGenerateUUID()
349+ routeID := uuid.New().String()
412350 now := time.Now().Format(time.RFC3339)
413351 _, err = txExecWithRetry(tx,
414352 p.queries.InsertRoute,
@@ -1835,13 +1773,76 @@ func (p *Proxy) detectGordonContainer() string {
18351773 }
18361774
18371775 // proxyRequest proxies an HTTP request to a container
1838-func (p *Proxy) proxyRequest(c echo.Context, route *ProxyRouteInfo) error {
1776+func (p *Proxy) proxyRequest(c echo.Context) error {
1777+ host := c.Request().Host
1778+
1779+ // Strip port if present in the host
1780+ if hostParts := strings.Split(host, ":"); len(hostParts) > 1 {
1781+ host = hostParts[0]
1782+ }
1783+
1784+ // Normalize host (trim trailing dot, convert to lowercase)
1785+ host = strings.TrimSuffix(strings.ToLower(host), ".")
1786+
1787+ // Debug log the incoming request
1788+ logger.Debug("Proxy request received",
1789+ "host", host,
1790+ "method", c.Request().Method,
1791+ "path", c.Request().URL.Path,
1792+ "client_ip", c.RealIP())
1793+
1794+ // Get the route for this host
1795+ p.mu.RLock()
1796+ route, ok := p.routes[host]
1797+ p.mu.RUnlock()
1798+
1799+ if !ok {
1800+ // Check if the host is an IP address
1801+ if net.ParseIP(host) != nil {
1802+ // For IP-based requests, just return a 404 without logging warnings
1803+ return c.String(http.StatusNotFound, "Domain not found")
1804+ }
1805+
1806+ // Log available domains for debugging
1807+ availableDomains := make([]string, 0, len(p.routes))
1808+ p.mu.RLock()
1809+ for d := range p.routes {
1810+ availableDomains = append(availableDomains, d)
1811+ }
1812+ p.mu.RUnlock()
1813+
1814+ logger.Warn("Request with unknown host",
1815+ "requested_host", host,
1816+ "client_ip", c.RealIP(),
1817+ "available_domains", strings.Join(availableDomains, ", "))
1818+ return c.String(http.StatusNotFound, "Domain not found")
1819+ }
1820+
1821+ // Check if the route is active
1822+ if !route.Active {
1823+ logger.Warn("Route is not active",
1824+ "domain", host,
1825+ "client_ip", c.RealIP())
1826+ return c.String(http.StatusServiceUnavailable, "Route is not active")
1827+ }
1828+
1829+ return p.doProxyRequest(c, route)
1830+}
1831+
1832+func (p *Proxy) doProxyRequest(c echo.Context, route *ProxyRouteInfo) error {
18391833 host := c.Request().Host
18401834 containerID := route.ContainerID
18411835 containerIP := route.ContainerIP
18421836 containerPort := route.ContainerPort
18431837 targetProtocol := route.Protocol
18441838
1839+ logger.Debug("Starting proxy request",
1840+ "host", host,
1841+ "container_id", containerID,
1842+ "container_ip", containerIP,
1843+ "container_port", containerPort,
1844+ "protocol", targetProtocol)
1845+
18451846 // Check if this is the admin domain
18461847 isAdminDomain := host == p.app.GetConfig().Http.FullDomain()
18471848
@@ -1879,7 +1880,7 @@ func (p *Proxy) proxyRequest(c echo.Context, route *ProxyRouteInfo) error {
18791880
18801881 // Create the target URL
18811882 targetURL := &url.URL{
1882- Scheme: targetProtocol,
1883+ Scheme: "http", // Always use HTTP internally, regardless of external protocol
18831884 }
18841885
18851886 // Format host properly for both IPv4 and IPv6
@@ -1889,12 +1890,37 @@ func (p *Proxy) proxyRequest(c echo.Context, route *ProxyRouteInfo) error {
18891890 }
18901891 targetURL.Host = fmt.Sprintf("%s:%s", containerIP, containerPort)
18911892
1893+ logger.Debug("Proxying to backend",
1894+ "target_url", targetURL.String(),
1895+ "original_host", host)
1896+
18921897 // Create a reverse proxy
18931898 proxy := httputil.NewSingleHostReverseProxy(targetURL)
18941899
18951900 // Use our custom transport with longer timeouts
18961901 if p.reverseProxyClient != nil {
18971902 proxy.Transport = p.reverseProxyClient.Transport
1903+ } else {
1904+ // Create a transport with reasonable timeouts if none exists
1905+ proxy.Transport = &http.Transport{
1906+ TLSClientConfig: &tls.Config{
1907+ InsecureSkipVerify: true,
1908+ },
1909+ // Use shorter timeouts for internal network
1910+ TLSHandshakeTimeout: 5 * time.Second,
1911+ ResponseHeaderTimeout: 10 * time.Second,
1912+ ExpectContinueTimeout: 3 * time.Second,
1913+ // Add a custom dialer with explicit timeouts
1914+ DialContext: (&net.Dialer{
1915+ Timeout: 5 * time.Second,
1916+ KeepAlive: 30 * time.Second,
1917+ DualStack: true,
1918+ }).DialContext,
1919+ MaxIdleConns: 100,
1920+ MaxIdleConnsPerHost: 10,
1921+ IdleConnTimeout: 90 * time.Second,
1922+ ForceAttemptHTTP2: true,
1923+ }
18981924 }
18991925
19001926 // Set up request director to modify the request before it's sent
@@ -1917,6 +1943,11 @@ func (p *Proxy) proxyRequest(c echo.Context, route *ProxyRouteInfo) error {
19171943
19181944 // Set X-Forwarded-Host to the original host
19191945 req.Header.Set("X-Forwarded-Host", host)
1946+
1947+ logger.Debug("Modified outgoing proxy request",
1948+ "req_host", req.Host,
1949+ "req_url", req.URL.String(),
1950+ "x_forwarded_proto", req.Header.Get("X-Forwarded-Proto"))
19201951 }
19211952
19221953 // Log errors that occur during proxying
@@ -1958,8 +1989,54 @@ func (p *Proxy) proxyRequest(c echo.Context, route *ProxyRouteInfo) error {
19581989 c.Response().Write([]byte("Bad Gateway: Container is unreachable"))
19591990 }
19601991
1992+ // For admin domain, ensure the connection succeeds
1993+ if isAdminDomain {
1994+ // Do a pre-check to make sure the admin service is reachable
1995+ testURL := fmt.Sprintf("http://%s:%s/admin/ping", containerIP, containerPort)
1996+ client := &http.Client{
1997+ Timeout: 2 * time.Second,
1998+ Transport: &http.Transport{
1999+ DialContext: (&net.Dialer{
2000+ Timeout: 1 * time.Second,
2001+ }).DialContext,
2002+ },
2003+ }
2004+
2005+ logger.Debug("Testing admin connection before proxying",
2006+ "test_url", testURL)
2007+
2008+ _, err := client.Get(testURL)
2009+ if err != nil {
2010+ logger.Warn("Admin service pre-check failed, trying alternative connection methods",
2011+ "error", err)
2012+
2013+ // Try alternative IPs for the admin domain if the container IP isn't working
2014+ altIPs := []string{"localhost", "127.0.0.1", p.app.GetConfig().Http.SubDomain}
2015+
2016+ for _, altIP := range altIPs {
2017+ testURL = fmt.Sprintf("http://%s:%s/admin/ping", altIP, containerPort)
2018+ _, err := client.Get(testURL)
2019+ if err == nil {
2020+ logger.Info("Found working admin connection with alternative IP",
2021+ "ip", altIP)
2022+ containerIP = altIP
2023+ targetURL.Host = fmt.Sprintf("%s:%s", containerIP, containerPort)
2024+ break
2025+ }
2026+ }
2027+ }
2028+ }
2029+
19612030 // Serve the request using the reverse proxy
2031+ logger.Debug("Serving proxy request",
2032+ "target", targetURL.String())
19622033 proxy.ServeHTTP(c.Response(), c.Request())
2034+
2035+ logger.Debug("Proxy request completed",
2036+ "host", host,
2037+ "target", targetURL.String(),
2038+ "status", c.Response().Status)
2039+
19632040 return nil
19642041 }
19652042
@@ -1,6 +1,7 @@
1 package proxy1 package proxy
2 2
3 import (3 import (
4+ "crypto/tls"
4 "database/sql"5 "database/sql"
5 "fmt"6 "fmt"
6 "net"7 "net"
@@ -13,6 +14,7 @@ import (
13 14
14 "github.com/bnema/gordon/pkg/docker"15 "github.com/bnema/gordon/pkg/docker"
15 "github.com/bnema/gordon/pkg/logger"16 "github.com/bnema/gordon/pkg/logger"
17+ "github.com/google/uuid"
16 "github.com/labstack/echo/v4"18 "github.com/labstack/echo/v4"
17 )19 )
18 20
@@ -157,78 +159,14 @@ func (p *Proxy) configureRoutes() {
157 logger.Debug("Admin domain route exists", "domain", adminDomain)159 logger.Debug("Admin domain route exists", "domain", adminDomain)
158 }160 }
159 161
160- // Handler function - will be assigned to both HTTP and HTTPS servers
161- handler := echo.HandlerFunc(func(c echo.Context) error {
162- host := c.Request().Host
163-
164- // Strip port if present in the host
165- if hostParts := strings.Split(host, ":"); len(hostParts) > 1 {
166- host = hostParts[0]
167- }
168-
169- // Normalize host (trim trailing dot, convert to lowercase)
170- host = strings.TrimSuffix(strings.ToLower(host), ".")
171-
172- // Check if this is the admin domain
173- adminDomain := p.app.GetConfig().Http.FullDomain()
174- if strings.EqualFold(host, adminDomain) {
175- // Always prioritize admin domain
176- p.mu.RLock()
177- adminRoute, adminOk := p.routes[adminDomain]
178- p.mu.RUnlock()
179-
180- if adminOk {
181- return p.proxyRequest(c, adminRoute)
182- }
183- }
184-
185- // Important: Get the route freshly from the map for every request
186- // This ensures we always use the most up-to-date IP
187- p.mu.RLock()
188- route, ok := p.routes[host]
189- p.mu.RUnlock()
190-
191- if !ok {
192- // Check if the host is an IP address - silently handle without logging warnings
193- if net.ParseIP(host) != nil {
194- // For IP-based requests, just return a 404 without logging warnings
195- return c.String(http.StatusNotFound, "Domain not found")
196- }
197-
198- // Create list of available domains for debugging
199- availableDomains := make([]string, 0, len(p.routes))
200- p.mu.RLock()
201- for d := range p.routes {
202- availableDomains = append(availableDomains, d)
203- }
204- p.mu.RUnlock()
205-
206- // Log warning for non-IP hosts that aren't configured
207- logger.Warn("Request with unknown host",
208- "requested_host", host,
209- "client_ip", c.RealIP(),
210- "available_domains", strings.Join(availableDomains, ", "))
211- return c.String(http.StatusNotFound, "Domain not found")
212- }
213-
214- // Check if the route is active
215- if !route.Active {
216- logger.Warn("Route is not active",
217- "domain", host,
218- "client_ip", c.RealIP())
219- return c.String(http.StatusServiceUnavailable, "Route is not active")
220- }
221- return p.proxyRequest(c, route)
222- })
223-
224 // Add the handler to the HTTPS server162 // Add the handler to the HTTPS server
225 if p.httpsServer != nil {163 if p.httpsServer != nil {
226- p.httpsServer.Any("/*", handler)164+ p.httpsServer.Any("/*", p.proxyRequest)
227 }165 }
228 166
229 // Also add the handler to the HTTP server for HTTP-01 challenges167 // Also add the handler to the HTTP server for HTTP-01 challenges
230 if p.httpServer != nil {168 if p.httpServer != nil {
231- p.httpServer.Any("/*", handler)169+ p.httpServer.Any("/*", p.proxyRequest)
232 }170 }
233 }171 }
234 172
@@ -308,7 +246,7 @@ func (p *Proxy) AddRoute(domainName, containerID, containerIP, containerPort, pr
308 if err != nil {246 if err != nil {
309 if err == sql.ErrNoRows {247 if err == sql.ErrNoRows {
310 // Domain doesn't exist, create it248 // Domain doesn't exist, create it
311- domainID = proxyGenerateUUID()249+ domainID = uuid.New().String()
312 now := time.Now().Format(time.RFC3339)250 now := time.Now().Format(time.RFC3339)
313 _, err = txExecWithRetry(tx,251 _, err = txExecWithRetry(tx,
314 p.queries.InsertDomain,252 p.queries.InsertDomain,
@@ -338,7 +276,7 @@ func (p *Proxy) AddRoute(domainName, containerID, containerIP, containerPort, pr
338 if err != nil {276 if err != nil {
339 if err == sql.ErrNoRows {277 if err == sql.ErrNoRows {
340 // Domain part doesn't exist, create it278 // Domain part doesn't exist, create it
341- domainPartID = proxyGenerateUUID()279+ domainPartID = uuid.New().String()
342 _, err = txExecWithRetry(tx,280 _, err = txExecWithRetry(tx,
343 p.queries.InsertDomain,281 p.queries.InsertDomain,
344 domainPartID, domainPart, accountID, now, now,282 domainPartID, domainPart, accountID, now, now,
@@ -408,7 +346,7 @@ func (p *Proxy) AddRoute(domainName, containerID, containerIP, containerPort, pr
408 }346 }
409 } else if err == sql.ErrNoRows {347 } else if err == sql.ErrNoRows {
410 // Route doesn't exist, create it348 // Route doesn't exist, create it
411- routeID := proxyGenerateUUID()349+ routeID := uuid.New().String()
412 now := time.Now().Format(time.RFC3339)350 now := time.Now().Format(time.RFC3339)
413 _, err = txExecWithRetry(tx,351 _, err = txExecWithRetry(tx,
414 p.queries.InsertRoute,352 p.queries.InsertRoute,
@@ -1835,13 +1773,76 @@ func (p *Proxy) detectGordonContainer() string {
1835 }1773 }
1836 1774
1837 // proxyRequest proxies an HTTP request to a container1775 // proxyRequest proxies an HTTP request to a container
1838-func (p *Proxy) proxyRequest(c echo.Context, route *ProxyRouteInfo) error {1776+func (p *Proxy) proxyRequest(c echo.Context) error {
1777+ host := c.Request().Host
1778+
1779+ // Strip port if present in the host
1780+ if hostParts := strings.Split(host, ":"); len(hostParts) > 1 {
1781+ host = hostParts[0]
1782+ }
1783+
1784+ // Normalize host (trim trailing dot, convert to lowercase)
1785+ host = strings.TrimSuffix(strings.ToLower(host), ".")
1786+
1787+ // Debug log the incoming request
1788+ logger.Debug("Proxy request received",
1789+ "host", host,
1790+ "method", c.Request().Method,
1791+ "path", c.Request().URL.Path,
1792+ "client_ip", c.RealIP())
1793+
1794+ // Get the route for this host
1795+ p.mu.RLock()
1796+ route, ok := p.routes[host]
1797+ p.mu.RUnlock()
1798+
1799+ if !ok {
1800+ // Check if the host is an IP address
1801+ if net.ParseIP(host) != nil {
1802+ // For IP-based requests, just return a 404 without logging warnings
1803+ return c.String(http.StatusNotFound, "Domain not found")
1804+ }
1805+
1806+ // Log available domains for debugging
1807+ availableDomains := make([]string, 0, len(p.routes))
1808+ p.mu.RLock()
1809+ for d := range p.routes {
1810+ availableDomains = append(availableDomains, d)
1811+ }
1812+ p.mu.RUnlock()
1813+
1814+ logger.Warn("Request with unknown host",
1815+ "requested_host", host,
1816+ "client_ip", c.RealIP(),
1817+ "available_domains", strings.Join(availableDomains, ", "))
1818+ return c.String(http.StatusNotFound, "Domain not found")
1819+ }
1820+
1821+ // Check if the route is active
1822+ if !route.Active {
1823+ logger.Warn("Route is not active",
1824+ "domain", host,
1825+ "client_ip", c.RealIP())
1826+ return c.String(http.StatusServiceUnavailable, "Route is not active")
1827+ }
1828+
1829+ return p.doProxyRequest(c, route)
1830+}
1831+
1832+func (p *Proxy) doProxyRequest(c echo.Context, route *ProxyRouteInfo) error {
1839 host := c.Request().Host1833 host := c.Request().Host
1840 containerID := route.ContainerID1834 containerID := route.ContainerID
1841 containerIP := route.ContainerIP1835 containerIP := route.ContainerIP
1842 containerPort := route.ContainerPort1836 containerPort := route.ContainerPort
1843 targetProtocol := route.Protocol1837 targetProtocol := route.Protocol
1844 1838
1839+ logger.Debug("Starting proxy request",
1840+ "host", host,
1841+ "container_id", containerID,
1842+ "container_ip", containerIP,
1843+ "container_port", containerPort,
1844+ "protocol", targetProtocol)
1845+
1845 // Check if this is the admin domain1846 // Check if this is the admin domain
1846 isAdminDomain := host == p.app.GetConfig().Http.FullDomain()1847 isAdminDomain := host == p.app.GetConfig().Http.FullDomain()
1847 1848
@@ -1879,7 +1880,7 @@ func (p *Proxy) proxyRequest(c echo.Context, route *ProxyRouteInfo) error {
1879 1880
1880 // Create the target URL1881 // Create the target URL
1881 targetURL := &url.URL{1882 targetURL := &url.URL{
1882- Scheme: targetProtocol,1883+ Scheme: "http", // Always use HTTP internally, regardless of external protocol
1883 }1884 }
1884 1885
1885 // Format host properly for both IPv4 and IPv61886 // Format host properly for both IPv4 and IPv6
@@ -1889,12 +1890,37 @@ func (p *Proxy) proxyRequest(c echo.Context, route *ProxyRouteInfo) error {
1889 }1890 }
1890 targetURL.Host = fmt.Sprintf("%s:%s", containerIP, containerPort)1891 targetURL.Host = fmt.Sprintf("%s:%s", containerIP, containerPort)
1891 1892
1893+ logger.Debug("Proxying to backend",
1894+ "target_url", targetURL.String(),
1895+ "original_host", host)
1896+
1892 // Create a reverse proxy1897 // Create a reverse proxy
1893 proxy := httputil.NewSingleHostReverseProxy(targetURL)1898 proxy := httputil.NewSingleHostReverseProxy(targetURL)
1894 1899
1895 // Use our custom transport with longer timeouts1900 // Use our custom transport with longer timeouts
1896 if p.reverseProxyClient != nil {1901 if p.reverseProxyClient != nil {
1897 proxy.Transport = p.reverseProxyClient.Transport1902 proxy.Transport = p.reverseProxyClient.Transport
1903+ } else {
1904+ // Create a transport with reasonable timeouts if none exists
1905+ proxy.Transport = &http.Transport{
1906+ TLSClientConfig: &tls.Config{
1907+ InsecureSkipVerify: true,
1908+ },
1909+ // Use shorter timeouts for internal network
1910+ TLSHandshakeTimeout: 5 * time.Second,
1911+ ResponseHeaderTimeout: 10 * time.Second,
1912+ ExpectContinueTimeout: 3 * time.Second,
1913+ // Add a custom dialer with explicit timeouts
1914+ DialContext: (&net.Dialer{
1915+ Timeout: 5 * time.Second,
1916+ KeepAlive: 30 * time.Second,
1917+ DualStack: true,
1918+ }).DialContext,
1919+ MaxIdleConns: 100,
1920+ MaxIdleConnsPerHost: 10,
1921+ IdleConnTimeout: 90 * time.Second,
1922+ ForceAttemptHTTP2: true,
1923+ }
1898 }1924 }
1899 1925
1900 // Set up request director to modify the request before it's sent1926 // Set up request director to modify the request before it's sent
@@ -1917,6 +1943,11 @@ func (p *Proxy) proxyRequest(c echo.Context, route *ProxyRouteInfo) error {
1917 1943
1918 // Set X-Forwarded-Host to the original host1944 // Set X-Forwarded-Host to the original host
1919 req.Header.Set("X-Forwarded-Host", host)1945 req.Header.Set("X-Forwarded-Host", host)
1946+
1947+ logger.Debug("Modified outgoing proxy request",
1948+ "req_host", req.Host,
1949+ "req_url", req.URL.String(),
1950+ "x_forwarded_proto", req.Header.Get("X-Forwarded-Proto"))
1920 }1951 }
1921 1952
1922 // Log errors that occur during proxying1953 // Log errors that occur during proxying
@@ -1958,8 +1989,54 @@ func (p *Proxy) proxyRequest(c echo.Context, route *ProxyRouteInfo) error {
1958 c.Response().Write([]byte("Bad Gateway: Container is unreachable"))1989 c.Response().Write([]byte("Bad Gateway: Container is unreachable"))
1959 }1990 }
1960 1991
1992+ // For admin domain, ensure the connection succeeds
1993+ if isAdminDomain {
1994+ // Do a pre-check to make sure the admin service is reachable
1995+ testURL := fmt.Sprintf("http://%s:%s/admin/ping", containerIP, containerPort)
1996+ client := &http.Client{
1997+ Timeout: 2 * time.Second,
1998+ Transport: &http.Transport{
1999+ DialContext: (&net.Dialer{
2000+ Timeout: 1 * time.Second,
2001+ }).DialContext,
2002+ },
2003+ }
2004+
2005+ logger.Debug("Testing admin connection before proxying",
2006+ "test_url", testURL)
2007+
2008+ _, err := client.Get(testURL)
2009+ if err != nil {
2010+ logger.Warn("Admin service pre-check failed, trying alternative connection methods",
2011+ "error", err)
2012+
2013+ // Try alternative IPs for the admin domain if the container IP isn't working
2014+ altIPs := []string{"localhost", "127.0.0.1", p.app.GetConfig().Http.SubDomain}
2015+
2016+ for _, altIP := range altIPs {
2017+ testURL = fmt.Sprintf("http://%s:%s/admin/ping", altIP, containerPort)
2018+ _, err := client.Get(testURL)
2019+ if err == nil {
2020+ logger.Info("Found working admin connection with alternative IP",
2021+ "ip", altIP)
2022+ containerIP = altIP
2023+ targetURL.Host = fmt.Sprintf("%s:%s", containerIP, containerPort)
2024+ break
2025+ }
2026+ }
2027+ }
2028+ }
2029+
1961 // Serve the request using the reverse proxy2030 // Serve the request using the reverse proxy
2031+ logger.Debug("Serving proxy request",
2032+ "target", targetURL.String())
1962 proxy.ServeHTTP(c.Response(), c.Request())2033 proxy.ServeHTTP(c.Response(), c.Request())
2034+
2035+ logger.Debug("Proxy request completed",
2036+ "host", host,
2037+ "target", targetURL.String(),
2038+ "status", c.Response().Status)
2039+
1963 return nil2040 return nil
1964 }2041 }
1965 2042
M internal/proxy/server.go +182 -13
@@ -4,7 +4,9 @@ import (
44 "context"
55 "crypto/tls"
66 "fmt"
7+ "net"
78 "net/http"
9+ "strings"
810 "time"
911
1012 "github.com/bnema/gordon/pkg/docker"
@@ -167,15 +169,126 @@ func (p *Proxy) Start() error {
167169 "server_name", hello.ServerName)
168170 }
169171
172+ // Verify certificate manager is properly initialized
173+ if p.certManager == nil {
174+ logger.Error("TLS certificate manager is nil during handshake - certificate initialization failed",
175+ "server_name", hello.ServerName,
176+ "solution", "Check certificate configuration and logs for setup errors")
177+
178+ // Use fallback certificate if we have one
179+ if p.fallbackCert != nil {
180+ logger.Warn("Using fallback self-signed certificate due to missing certificate manager",
181+ "server_name", hello.ServerName)
182+ return p.fallbackCert, nil
183+ }
184+
185+ return nil, fmt.Errorf("certificate manager not initialized")
186+ }
187+
170188 // Try to get the certificate from the autocert manager
189+ logger.Debug("Attempting to get certificate from manager",
190+ "server_name", hello.ServerName,
191+ "has_fallback", p.fallbackCert != nil)
192+
193+ // First try checking if we have a valid certificate in cache
194+ cacheKey := "cert-" + hello.ServerName
195+ certCache := p.certManager.Cache
196+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
197+ defer cancel()
198+
199+ if certCache != nil {
200+ certData, err := certCache.Get(ctx, cacheKey)
201+ if err == nil && len(certData) > 0 {
202+ logger.Debug("Found certificate in cache", "domain", hello.ServerName)
203+ // Use directly if found in cache - skip the potentially slower autocert.Manager GetCertificate
204+ keyKey := "key-" + hello.ServerName
205+ keyData, keyErr := certCache.Get(ctx, keyKey)
206+ if keyErr == nil && len(keyData) > 0 {
207+ cert, err := tls.X509KeyPair(certData, keyData)
208+ if err == nil {
209+ logger.Debug("Successfully loaded certificate from cache",
210+ "domain", hello.ServerName)
211+ return &cert, nil
212+ }
213+ logger.Error("Failed to create X509KeyPair from cache",
214+ "domain", hello.ServerName,
215+ "error", err)
216+ } else {
217+ logger.Error("Certificate found in cache but key is missing or invalid",
218+ "domain", hello.ServerName,
219+ "key_error", keyErr)
220+ }
221+ } else if err != nil {
222+ logger.Debug("Certificate not found in cache during TLS handshake",
223+ "domain", hello.ServerName,
224+ "error", err)
225+
226+ // Check if we have it on disk but not in cache (unlikely after preloading)
227+ hasOnDisk, certType := p.checkCertificateInFilesystem(hello.ServerName)
228+ if hasOnDisk {
229+ logger.Warn("Certificate found on disk but not in cache - this should not happen after preloading",
230+ "domain", hello.ServerName,
231+ "type", certType,
232+ "solution", "Check preloadCertificates implementation")
233+ }
234+ }
235+ }
236+
237+ // If not in cache, use the standard GetCertificate method
238+ // Let's Encrypt may be contacted if no certificate exists
171239 cert, err := p.certManager.GetCertificate(hello)
172240
173- // If we can't get a certificate and we have a fallback, use it for the admin domain
174- if (err != nil || cert == nil) && hello.ServerName == adminDomain && p.fallbackCert != nil {
175- logger.Debug("Using fallback certificate for admin domain",
176- "domain", adminDomain,
241+ if err != nil {
242+ logger.Error("Error getting certificate from manager",
243+ "server_name", hello.ServerName,
177244 "error", err)
178- return p.fallbackCert, nil
245+
246+ // Check for rate limit errors
247+ if strings.Contains(err.Error(), "too many certificates") ||
248+ strings.Contains(err.Error(), "rateLimited") {
249+ logger.Error("Let's Encrypt rate limit hit during TLS handshake",
250+ "domain", hello.ServerName,
251+ "solution", "Wait for rate limit to expire or manually add certificate")
252+ }
253+
254+ // Try manually loading certificate files directly as a fallback before using fallback cert
255+ if manualCert, manualErr := p.manuallyLoadCertificate(hello.ServerName); manualErr == nil {
256+ logger.Info("Successfully loaded certificate manually as fallback",
257+ "domain", hello.ServerName)
258+ return manualCert, nil
259+ } else {
260+ logger.Debug("Manual certificate loading also failed",
261+ "domain", hello.ServerName,
262+ "error", manualErr)
263+ }
264+
265+ // If we can't get a certificate and we have a fallback, use it for the admin domain
266+ if hello.ServerName == p.app.GetConfig().Http.FullDomain() && p.fallbackCert != nil {
267+ logger.Debug("Using fallback certificate for admin domain",
268+ "domain", hello.ServerName,
269+ "error", err)
270+ return p.fallbackCert, nil
271+ }
272+ } else if cert == nil {
273+ logger.Error("Certificate manager returned nil certificate without error",
274+ "server_name", hello.ServerName)
275+
276+ // Try manually loading certificate files directly as a fallback
277+ if manualCert, manualErr := p.manuallyLoadCertificate(hello.ServerName); manualErr == nil {
278+ logger.Info("Successfully loaded certificate manually when autocert returned nil",
279+ "domain", hello.ServerName)
280+ return manualCert, nil
281+ }
282+
283+ // If we can't get a certificate and we have a fallback, use it for the admin domain
284+ if hello.ServerName == p.app.GetConfig().Http.FullDomain() && p.fallbackCert != nil {
285+ logger.Debug("Using fallback certificate for admin domain",
286+ "domain", hello.ServerName)
287+ return p.fallbackCert, nil
288+ }
289+ } else {
290+ logger.Debug("Successfully obtained certificate from manager",
291+ "server_name", hello.ServerName)
179292 }
180293
181294 return cert, err
@@ -186,12 +299,16 @@ func (p *Proxy) Start() error {
186299 // Add support for TLS-ALPN-01 challenges
187300 NextProtos: []string{acme.ALPNProto},
188301 },
189- // Add timeouts to prevent hanging connections
190- ReadTimeout: 30 * time.Second,
191- WriteTimeout: 30 * time.Second,
192- IdleTimeout: 120 * time.Second,
302+ // Add increased timeouts to prevent hanging connections
303+ ReadTimeout: 60 * time.Second,
304+ WriteTimeout: 60 * time.Second,
305+ IdleTimeout: 120 * time.Second,
306+ ReadHeaderTimeout: 30 * time.Second, // Add header timeout
193307 }
194308
309+ // Store the HTTPS server for later use in Stop()
310+ p.httpsServer.Server = httpsServer
311+
195312 // Create a custom HTTP server
196313 httpServer := &http.Server{
197314 Addr: fmt.Sprintf(":%s", p.config.HttpPort),
@@ -203,12 +320,24 @@ func (p *Proxy) Start() error {
203320
204321 p.serverStarted = true
205322
206- // Start HTTPS server
323+ // Start HTTPS server with more error details
207324 go func() {
208325 logger.Info("Starting HTTPS server", "address", httpsServer.Addr)
209326 // Using empty strings for cert and key files since we're using GetCertificate
210327 if err := httpsServer.ListenAndServeTLS("", ""); err != nil && err != http.ErrServerClosed {
211- logger.Error("HTTPS server failed", "error", err)
328+ logger.Error("HTTPS server failed", "error", err, "address", httpsServer.Addr)
329+
330+ // Print more diagnostic information
331+ if strings.Contains(err.Error(), "bind: address already in use") {
332+ logger.Error("Port 443 is already in use by another process. Check if another web server is running.",
333+ "solution", "Stop other services using port 443 or configure Gordon to use a different HTTPS port")
334+ } else if strings.Contains(err.Error(), "permission denied") {
335+ logger.Error("Permission denied when binding to port 443. This often happens when running without root privileges.",
336+ "solution", "Use setcap to allow binding to privileged ports or run in a container with proper port mapping")
337+ } else if strings.Contains(err.Error(), "certificate") {
338+ logger.Error("Certificate-related error when starting HTTPS server",
339+ "solution", "Check certificate files and permissions in the certificate directory")
340+ }
212341 }
213342 }()
214343
@@ -385,24 +514,64 @@ func (p *Proxy) checkExternalPortAccess() {
385514
386515 // Test HTTPS port 443
387516 httpsClient := &http.Client{
388- Timeout: 5 * time.Second,
517+ Timeout: 10 * time.Second, // Increased timeout for more reliable testing
389518 Transport: &http.Transport{
390519 TLSClientConfig: &tls.Config{
391520 InsecureSkipVerify: true,
392521 },
522+ // Add more detailed timeout settings
523+ TLSHandshakeTimeout: 5 * time.Second,
524+ ResponseHeaderTimeout: 5 * time.Second,
525+ ExpectContinueTimeout: 1 * time.Second,
526+ // Add a custom dialer with explicit timeouts
527+ DialContext: (&net.Dialer{
528+ Timeout: 5 * time.Second,
529+ KeepAlive: 30 * time.Second,
530+ DualStack: true,
531+ }).DialContext,
393532 },
394533 CheckRedirect: func(req *http.Request, via []*http.Request) error {
395534 return http.ErrUseLastResponse
396535 },
397536 }
398537
538+ // Verify TLS settings are consistent
539+ if p.certManager == nil {
540+ logger.Error("Certificate manager is nil - TLS will not work correctly",
541+ "solution", "Check setupCertManager implementation")
542+ }
543+
544+ // First try a basic TCP connection to port 443 to test if it's open
545+ logger.Info("Testing TCP connectivity to HTTPS port 443...")
546+ conn, tcpErr := net.DialTimeout("tcp", adminDomain+":443", 5*time.Second)
547+ if tcpErr != nil {
548+ logger.Warn("TCP connection to port 443 failed - HTTPS will not work",
549+ "domain", adminDomain,
550+ "error", tcpErr.Error())
551+ } else {
552+ conn.Close()
553+ logger.Info("TCP connection to port 443 successful")
554+ }
555+
556+ // Then try the full HTTPS request
399557 httpsURL := fmt.Sprintf("https://%s/", adminDomain)
558+ logger.Info("Testing HTTPS connection with SNI...", "url", httpsURL)
400559 _, httpsErr := httpsClient.Get(httpsURL)
401560 if httpsErr != nil {
402561 logger.Warn("External HTTPS port 443 might not be accessible",
403562 "error", httpsErr.Error())
563+
564+ // Provide more detailed troubleshooting info
565+ if strings.Contains(httpsErr.Error(), "certificate") {
566+ logger.Warn("TLS certificate issue detected",
567+ "hint", "Certificate verification failed but connection succeeded")
568+ } else if strings.Contains(httpsErr.Error(), "connection refused") {
569+ logger.Warn("Connection refused - firewall may be blocking port 443")
570+ } else if strings.Contains(httpsErr.Error(), "i/o timeout") || strings.Contains(httpsErr.Error(), "deadline exceeded") {
571+ logger.Warn("Connection timeout - port 443 may be filtered or not properly forwarded")
572+ }
404573 } else {
405- logger.Info("External HTTPS port 443 is accessible")
574+ logger.Info("External HTTPS port 443 is accessible - TLS handshake successful")
406575 }
407576 }
408577
@@ -4,7 +4,9 @@ import (
4 "context"4 "context"
5 "crypto/tls"5 "crypto/tls"
6 "fmt"6 "fmt"
7+ "net"
7 "net/http"8 "net/http"
9+ "strings"
8 "time"10 "time"
9 11
10 "github.com/bnema/gordon/pkg/docker"12 "github.com/bnema/gordon/pkg/docker"
@@ -167,15 +169,126 @@ func (p *Proxy) Start() error {
167 "server_name", hello.ServerName)169 "server_name", hello.ServerName)
168 }170 }
169 171
172+ // Verify certificate manager is properly initialized
173+ if p.certManager == nil {
174+ logger.Error("TLS certificate manager is nil during handshake - certificate initialization failed",
175+ "server_name", hello.ServerName,
176+ "solution", "Check certificate configuration and logs for setup errors")
177+
178+ // Use fallback certificate if we have one
179+ if p.fallbackCert != nil {
180+ logger.Warn("Using fallback self-signed certificate due to missing certificate manager",
181+ "server_name", hello.ServerName)
182+ return p.fallbackCert, nil
183+ }
184+
185+ return nil, fmt.Errorf("certificate manager not initialized")
186+ }
187+
170 // Try to get the certificate from the autocert manager188 // Try to get the certificate from the autocert manager
189+ logger.Debug("Attempting to get certificate from manager",
190+ "server_name", hello.ServerName,
191+ "has_fallback", p.fallbackCert != nil)
192+
193+ // First try checking if we have a valid certificate in cache
194+ cacheKey := "cert-" + hello.ServerName
195+ certCache := p.certManager.Cache
196+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
197+ defer cancel()
198+
199+ if certCache != nil {
200+ certData, err := certCache.Get(ctx, cacheKey)
201+ if err == nil && len(certData) > 0 {
202+ logger.Debug("Found certificate in cache", "domain", hello.ServerName)
203+ // Use directly if found in cache - skip the potentially slower autocert.Manager GetCertificate
204+ keyKey := "key-" + hello.ServerName
205+ keyData, keyErr := certCache.Get(ctx, keyKey)
206+ if keyErr == nil && len(keyData) > 0 {
207+ cert, err := tls.X509KeyPair(certData, keyData)
208+ if err == nil {
209+ logger.Debug("Successfully loaded certificate from cache",
210+ "domain", hello.ServerName)
211+ return &cert, nil
212+ }
213+ logger.Error("Failed to create X509KeyPair from cache",
214+ "domain", hello.ServerName,
215+ "error", err)
216+ } else {
217+ logger.Error("Certificate found in cache but key is missing or invalid",
218+ "domain", hello.ServerName,
219+ "key_error", keyErr)
220+ }
221+ } else if err != nil {
222+ logger.Debug("Certificate not found in cache during TLS handshake",
223+ "domain", hello.ServerName,
224+ "error", err)
225+
226+ // Check if we have it on disk but not in cache (unlikely after preloading)
227+ hasOnDisk, certType := p.checkCertificateInFilesystem(hello.ServerName)
228+ if hasOnDisk {
229+ logger.Warn("Certificate found on disk but not in cache - this should not happen after preloading",
230+ "domain", hello.ServerName,
231+ "type", certType,
232+ "solution", "Check preloadCertificates implementation")
233+ }
234+ }
235+ }
236+
237+ // If not in cache, use the standard GetCertificate method
238+ // Let's Encrypt may be contacted if no certificate exists
171 cert, err := p.certManager.GetCertificate(hello)239 cert, err := p.certManager.GetCertificate(hello)
172 240
173- // If we can't get a certificate and we have a fallback, use it for the admin domain241+ if err != nil {
174- if (err != nil || cert == nil) && hello.ServerName == adminDomain && p.fallbackCert != nil {242+ logger.Error("Error getting certificate from manager",
175- logger.Debug("Using fallback certificate for admin domain",243+ "server_name", hello.ServerName,
176- "domain", adminDomain,
177 "error", err)244 "error", err)
178- return p.fallbackCert, nil245+
246+ // Check for rate limit errors
247+ if strings.Contains(err.Error(), "too many certificates") ||
248+ strings.Contains(err.Error(), "rateLimited") {
249+ logger.Error("Let's Encrypt rate limit hit during TLS handshake",
250+ "domain", hello.ServerName,
251+ "solution", "Wait for rate limit to expire or manually add certificate")
252+ }
253+
254+ // Try manually loading certificate files directly as a fallback before using fallback cert
255+ if manualCert, manualErr := p.manuallyLoadCertificate(hello.ServerName); manualErr == nil {
256+ logger.Info("Successfully loaded certificate manually as fallback",
257+ "domain", hello.ServerName)
258+ return manualCert, nil
259+ } else {
260+ logger.Debug("Manual certificate loading also failed",
261+ "domain", hello.ServerName,
262+ "error", manualErr)
263+ }
264+
265+ // If we can't get a certificate and we have a fallback, use it for the admin domain
266+ if hello.ServerName == p.app.GetConfig().Http.FullDomain() && p.fallbackCert != nil {
267+ logger.Debug("Using fallback certificate for admin domain",
268+ "domain", hello.ServerName,
269+ "error", err)
270+ return p.fallbackCert, nil
271+ }
272+ } else if cert == nil {
273+ logger.Error("Certificate manager returned nil certificate without error",
274+ "server_name", hello.ServerName)
275+
276+ // Try manually loading certificate files directly as a fallback
277+ if manualCert, manualErr := p.manuallyLoadCertificate(hello.ServerName); manualErr == nil {
278+ logger.Info("Successfully loaded certificate manually when autocert returned nil",
279+ "domain", hello.ServerName)
280+ return manualCert, nil
281+ }
282+
283+ // If we can't get a certificate and we have a fallback, use it for the admin domain
284+ if hello.ServerName == p.app.GetConfig().Http.FullDomain() && p.fallbackCert != nil {
285+ logger.Debug("Using fallback certificate for admin domain",
286+ "domain", hello.ServerName)
287+ return p.fallbackCert, nil
288+ }
289+ } else {
290+ logger.Debug("Successfully obtained certificate from manager",
291+ "server_name", hello.ServerName)
179 }292 }
180 293
181 return cert, err294 return cert, err
@@ -186,12 +299,16 @@ func (p *Proxy) Start() error {
186 // Add support for TLS-ALPN-01 challenges299 // Add support for TLS-ALPN-01 challenges
187 NextProtos: []string{acme.ALPNProto},300 NextProtos: []string{acme.ALPNProto},
188 },301 },
189- // Add timeouts to prevent hanging connections302+ // Add increased timeouts to prevent hanging connections
190- ReadTimeout: 30 * time.Second,303+ ReadTimeout: 60 * time.Second,
191- WriteTimeout: 30 * time.Second,304+ WriteTimeout: 60 * time.Second,
192- IdleTimeout: 120 * time.Second,305+ IdleTimeout: 120 * time.Second,
306+ ReadHeaderTimeout: 30 * time.Second, // Add header timeout
193 }307 }
194 308
309+ // Store the HTTPS server for later use in Stop()
310+ p.httpsServer.Server = httpsServer
311+
195 // Create a custom HTTP server312 // Create a custom HTTP server
196 httpServer := &http.Server{313 httpServer := &http.Server{
197 Addr: fmt.Sprintf(":%s", p.config.HttpPort),314 Addr: fmt.Sprintf(":%s", p.config.HttpPort),
@@ -203,12 +320,24 @@ func (p *Proxy) Start() error {
203 320
204 p.serverStarted = true321 p.serverStarted = true
205 322
206- // Start HTTPS server323+ // Start HTTPS server with more error details
207 go func() {324 go func() {
208 logger.Info("Starting HTTPS server", "address", httpsServer.Addr)325 logger.Info("Starting HTTPS server", "address", httpsServer.Addr)
209 // Using empty strings for cert and key files since we're using GetCertificate326 // Using empty strings for cert and key files since we're using GetCertificate
210 if err := httpsServer.ListenAndServeTLS("", ""); err != nil && err != http.ErrServerClosed {327 if err := httpsServer.ListenAndServeTLS("", ""); err != nil && err != http.ErrServerClosed {
211- logger.Error("HTTPS server failed", "error", err)328+ logger.Error("HTTPS server failed", "error", err, "address", httpsServer.Addr)
329+
330+ // Print more diagnostic information
331+ if strings.Contains(err.Error(), "bind: address already in use") {
332+ logger.Error("Port 443 is already in use by another process. Check if another web server is running.",
333+ "solution", "Stop other services using port 443 or configure Gordon to use a different HTTPS port")
334+ } else if strings.Contains(err.Error(), "permission denied") {
335+ logger.Error("Permission denied when binding to port 443. This often happens when running without root privileges.",
336+ "solution", "Use setcap to allow binding to privileged ports or run in a container with proper port mapping")
337+ } else if strings.Contains(err.Error(), "certificate") {
338+ logger.Error("Certificate-related error when starting HTTPS server",
339+ "solution", "Check certificate files and permissions in the certificate directory")
340+ }
212 }341 }
213 }()342 }()
214 343
@@ -385,24 +514,64 @@ func (p *Proxy) checkExternalPortAccess() {
385 514
386 // Test HTTPS port 443515 // Test HTTPS port 443
387 httpsClient := &http.Client{516 httpsClient := &http.Client{
388- Timeout: 5 * time.Second,517+ Timeout: 10 * time.Second, // Increased timeout for more reliable testing
389 Transport: &http.Transport{518 Transport: &http.Transport{
390 TLSClientConfig: &tls.Config{519 TLSClientConfig: &tls.Config{
391 InsecureSkipVerify: true,520 InsecureSkipVerify: true,
392 },521 },
522+ // Add more detailed timeout settings
523+ TLSHandshakeTimeout: 5 * time.Second,
524+ ResponseHeaderTimeout: 5 * time.Second,
525+ ExpectContinueTimeout: 1 * time.Second,
526+ // Add a custom dialer with explicit timeouts
527+ DialContext: (&net.Dialer{
528+ Timeout: 5 * time.Second,
529+ KeepAlive: 30 * time.Second,
530+ DualStack: true,
531+ }).DialContext,
393 },532 },
394 CheckRedirect: func(req *http.Request, via []*http.Request) error {533 CheckRedirect: func(req *http.Request, via []*http.Request) error {
395 return http.ErrUseLastResponse534 return http.ErrUseLastResponse
396 },535 },
397 }536 }
398 537
538+ // Verify TLS settings are consistent
539+ if p.certManager == nil {
540+ logger.Error("Certificate manager is nil - TLS will not work correctly",
541+ "solution", "Check setupCertManager implementation")
542+ }
543+
544+ // First try a basic TCP connection to port 443 to test if it's open
545+ logger.Info("Testing TCP connectivity to HTTPS port 443...")
546+ conn, tcpErr := net.DialTimeout("tcp", adminDomain+":443", 5*time.Second)
547+ if tcpErr != nil {
548+ logger.Warn("TCP connection to port 443 failed - HTTPS will not work",
549+ "domain", adminDomain,
550+ "error", tcpErr.Error())
551+ } else {
552+ conn.Close()
553+ logger.Info("TCP connection to port 443 successful")
554+ }
555+
556+ // Then try the full HTTPS request
399 httpsURL := fmt.Sprintf("https://%s/", adminDomain)557 httpsURL := fmt.Sprintf("https://%s/", adminDomain)
558+ logger.Info("Testing HTTPS connection with SNI...", "url", httpsURL)
400 _, httpsErr := httpsClient.Get(httpsURL)559 _, httpsErr := httpsClient.Get(httpsURL)
401 if httpsErr != nil {560 if httpsErr != nil {
402 logger.Warn("External HTTPS port 443 might not be accessible",561 logger.Warn("External HTTPS port 443 might not be accessible",
403 "error", httpsErr.Error())562 "error", httpsErr.Error())
563+
564+ // Provide more detailed troubleshooting info
565+ if strings.Contains(httpsErr.Error(), "certificate") {
566+ logger.Warn("TLS certificate issue detected",
567+ "hint", "Certificate verification failed but connection succeeded")
568+ } else if strings.Contains(httpsErr.Error(), "connection refused") {
569+ logger.Warn("Connection refused - firewall may be blocking port 443")
570+ } else if strings.Contains(httpsErr.Error(), "i/o timeout") || strings.Contains(httpsErr.Error(), "deadline exceeded") {
571+ logger.Warn("Connection timeout - port 443 may be filtered or not properly forwarded")
572+ }
404 } else {573 } else {
405- logger.Info("External HTTPS port 443 is accessible")574+ logger.Info("External HTTPS port 443 is accessible - TLS handshake successful")
406 }575 }
407 }576 }
408 577
M internal/webui/public/assets/css/tailwind.css +0 -221
@@ -1,170 +1,4 @@
11 /*! tailwindcss v4.0.14 | MIT License | https://tailwindcss.com */
2-<<<<<<< HEAD
3-.visible {
4- visibility: visible;
5-}
6-.absolute {
7- position: absolute;
8-}
9-.fixed {
10- position: fixed;
11-}
12-.relative {
13- position: relative;
14-}
15-.static {
16- position: static;
17-}
18-.z-50 {
19- z-index: 50;
20-}
21-.container {
22- width: 100%;
23-}
24-.block {
25- display: block;
26-}
27-.contents {
28- display: contents;
29-}
30-.flex {
31- display: flex;
32-}
33-.hidden {
34- display: none;
35-}
36-.inline {
37- display: inline;
38-}
39-.inline-block {
40- display: inline-block;
41-}
42-.table {
43- display: table;
44-}
45-.h-full {
46- height: 100%;
47-}
48-.w-1\/3 {
49- width: calc(1/3 * 100%);
50-}
51-.w-1\/4 {
52- width: calc(1/4 * 100%);
53-}
54-.w-2\/3 {
55- width: calc(2/3 * 100%);
56-}
57-.w-5\/12 {
58- width: calc(5/12 * 100%);
59-}
60-.w-full {
61- width: 100%;
62-}
63-.flex-1 {
64- flex: 1;
65-}
66-.flex-grow {
67- flex-grow: 1;
68-}
69-.table-auto {
70- table-layout: auto;
71-}
72-.cursor-pointer {
73- cursor: pointer;
74-}
75-.resize {
76- resize: both;
77-}
78-.list-disc {
79- list-style-type: disc;
80-}
81-.flex-col {
82- flex-direction: column;
83-}
84-.flex-row {
85- flex-direction: row;
86-}
87-.content-center {
88- align-content: center;
89-}
90-.items-center {
91- align-items: center;
92-}
93-.justify-between {
94- justify-content: space-between;
95-}
96-.justify-center {
97- justify-content: center;
98-}
99-.justify-end {
100- justify-content: flex-end;
101-}
102-.justify-evenly {
103- justify-content: space-evenly;
104-}
105-.justify-start {
106- justify-content: flex-start;
107-}
108-.rounded-full {
109- border-radius: calc(infinity * 1px);
110-}
111-.border {
112- border-style: var(--tw-border-style);
113- border-width: 1px;
114-}
115-.border-2 {
116- border-style: var(--tw-border-style);
117- border-width: 2px;
118-}
119-.border-t-2 {
120- border-top-style: var(--tw-border-style);
121- border-top-width: 2px;
122-}
123-.bg-transparent {
124- background-color: transparent;
125-}
126-.text-center {
127- text-align: center;
128-}
129-.align-middle {
130- vertical-align: middle;
131-}
132-.opacity-0 {
133- opacity: 0%;
134-}
135-.filter {
136- filter: var(--tw-blur,) var(--tw-brightness,) var(--tw-contrast,) var(--tw-grayscale,) var(--tw-hue-rotate,) var(--tw-invert,) var(--tw-saturate,) var(--tw-sepia,) var(--tw-drop-shadow,);
137-}
138-.transition {
139- transition-property: color, background-color, border-color, outline-color, text-decoration-color, fill, stroke, --tw-gradient-from, --tw-gradient-via, --tw-gradient-to, opacity, box-shadow, transform, translate, scale, rotate, filter, -webkit-backdrop-filter, backdrop-filter;
140- transition-timing-function: var(--tw-ease, ease);
141- transition-duration: var(--tw-duration, 0s);
142-}
143-.transition-opacity {
144- transition-property: opacity;
145- transition-timing-function: var(--tw-ease, ease);
146- transition-duration: var(--tw-duration, 0s);
147-}
148-.duration-150 {
149- --tw-duration: 150ms;
150- transition-duration: 150ms;
151-}
152-.duration-300 {
153- --tw-duration: 300ms;
154- transition-duration: 300ms;
155-}
156-.outline-none {
157- --tw-outline-style: none;
158- outline-style: none;
159-}
160-.\[image\:tag\] {
161- image: tag;
162-}
163-.focus\:outline-none {
164- &:focus {
165- --tw-outline-style: none;
166- outline-style: none;
167-=======
1682 @layer theme, base, components, utilities;
1693 @layer theme {
1704 :root, :host {
@@ -6927,9 +6761,6 @@
69276761 .text-white {
69286762 color: var(--color-white);
69296763 }
6930- .lowercase {
6931- text-transform: lowercase;
6932- }
69336764 .uppercase {
69346765 text-transform: uppercase;
69356766 }
@@ -8036,7 +7867,6 @@
80367867 }
80377868 .text-subtitle {
80387869 color: var(--color-indigo-500);
8039->>>>>>> feat/templ-integration
80407870 }
80417871 }
80427872 @font-face {
@@ -8125,56 +7955,6 @@
81257955 .JetBrainsMonoBoldNF, .iconfb {
81267956 font-family: "JetBrainsMono-Nerd-Font-Bold", sans-serif;
81277957 }
8128-<<<<<<< HEAD
8129-@property --tw-border-style {
8130- syntax: "*";
8131- inherits: false;
8132- initial-value: solid;
8133-}
8134-@property --tw-blur {
8135- syntax: "*";
8136- inherits: false;
8137-}
8138-@property --tw-brightness {
8139- syntax: "*";
8140- inherits: false;
8141-}
8142-@property --tw-contrast {
8143- syntax: "*";
8144- inherits: false;
8145-}
8146-@property --tw-grayscale {
8147- syntax: "*";
8148- inherits: false;
8149-}
8150-@property --tw-hue-rotate {
8151- syntax: "*";
8152- inherits: false;
8153-}
8154-@property --tw-invert {
8155- syntax: "*";
8156- inherits: false;
8157-}
8158-@property --tw-opacity {
8159- syntax: "*";
8160- inherits: false;
8161-}
8162-@property --tw-saturate {
8163- syntax: "*";
8164- inherits: false;
8165-}
8166-@property --tw-sepia {
8167- syntax: "*";
8168- inherits: false;
8169-}
8170-@property --tw-drop-shadow {
8171- syntax: "*";
8172- inherits: false;
8173-}
8174-@property --tw-duration {
8175- syntax: "*";
8176- inherits: false;
8177-=======
81787958 @layer base {
81797959 :where(:root),:root:has(input.theme-controller[value=light]:checked),[data-theme=light] {
81807960 color-scheme: light;
@@ -8662,5 +8442,4 @@
86628442 to {
86638443 transform: rotate(360deg);
86648444 }
8665->>>>>>> feat/templ-integration
86668445 }
@@ -1,170 +1,4 @@
1 /*! tailwindcss v4.0.14 | MIT License | https://tailwindcss.com */1 /*! tailwindcss v4.0.14 | MIT License | https://tailwindcss.com */
2-<<<<<<< HEAD
3-.visible {
4- visibility: visible;
5-}
6-.absolute {
7- position: absolute;
8-}
9-.fixed {
10- position: fixed;
11-}
12-.relative {
13- position: relative;
14-}
15-.static {
16- position: static;
17-}
18-.z-50 {
19- z-index: 50;
20-}
21-.container {
22- width: 100%;
23-}
24-.block {
25- display: block;
26-}
27-.contents {
28- display: contents;
29-}
30-.flex {
31- display: flex;
32-}
33-.hidden {
34- display: none;
35-}
36-.inline {
37- display: inline;
38-}
39-.inline-block {
40- display: inline-block;
41-}
42-.table {
43- display: table;
44-}
45-.h-full {
46- height: 100%;
47-}
48-.w-1\/3 {
49- width: calc(1/3 * 100%);
50-}
51-.w-1\/4 {
52- width: calc(1/4 * 100%);
53-}
54-.w-2\/3 {
55- width: calc(2/3 * 100%);
56-}
57-.w-5\/12 {
58- width: calc(5/12 * 100%);
59-}
60-.w-full {
61- width: 100%;
62-}
63-.flex-1 {
64- flex: 1;
65-}
66-.flex-grow {
67- flex-grow: 1;
68-}
69-.table-auto {
70- table-layout: auto;
71-}
72-.cursor-pointer {
73- cursor: pointer;
74-}
75-.resize {
76- resize: both;
77-}
78-.list-disc {
79- list-style-type: disc;
80-}
81-.flex-col {
82- flex-direction: column;
83-}
84-.flex-row {
85- flex-direction: row;
86-}
87-.content-center {
88- align-content: center;
89-}
90-.items-center {
91- align-items: center;
92-}
93-.justify-between {
94- justify-content: space-between;
95-}
96-.justify-center {
97- justify-content: center;
98-}
99-.justify-end {
100- justify-content: flex-end;
101-}
102-.justify-evenly {
103- justify-content: space-evenly;
104-}
105-.justify-start {
106- justify-content: flex-start;
107-}
108-.rounded-full {
109- border-radius: calc(infinity * 1px);
110-}
111-.border {
112- border-style: var(--tw-border-style);
113- border-width: 1px;
114-}
115-.border-2 {
116- border-style: var(--tw-border-style);
117- border-width: 2px;
118-}
119-.border-t-2 {
120- border-top-style: var(--tw-border-style);
121- border-top-width: 2px;
122-}
123-.bg-transparent {
124- background-color: transparent;
125-}
126-.text-center {
127- text-align: center;
128-}
129-.align-middle {
130- vertical-align: middle;
131-}
132-.opacity-0 {
133- opacity: 0%;
134-}
135-.filter {
136- filter: var(--tw-blur,) var(--tw-brightness,) var(--tw-contrast,) var(--tw-grayscale,) var(--tw-hue-rotate,) var(--tw-invert,) var(--tw-saturate,) var(--tw-sepia,) var(--tw-drop-shadow,);
137-}
138-.transition {
139- transition-property: color, background-color, border-color, outline-color, text-decoration-color, fill, stroke, --tw-gradient-from, --tw-gradient-via, --tw-gradient-to, opacity, box-shadow, transform, translate, scale, rotate, filter, -webkit-backdrop-filter, backdrop-filter;
140- transition-timing-function: var(--tw-ease, ease);
141- transition-duration: var(--tw-duration, 0s);
142-}
143-.transition-opacity {
144- transition-property: opacity;
145- transition-timing-function: var(--tw-ease, ease);
146- transition-duration: var(--tw-duration, 0s);
147-}
148-.duration-150 {
149- --tw-duration: 150ms;
150- transition-duration: 150ms;
151-}
152-.duration-300 {
153- --tw-duration: 300ms;
154- transition-duration: 300ms;
155-}
156-.outline-none {
157- --tw-outline-style: none;
158- outline-style: none;
159-}
160-.\[image\:tag\] {
161- image: tag;
162-}
163-.focus\:outline-none {
164- &:focus {
165- --tw-outline-style: none;
166- outline-style: none;
167-=======
168 @layer theme, base, components, utilities;2 @layer theme, base, components, utilities;
169 @layer theme {3 @layer theme {
170 :root, :host {4 :root, :host {
@@ -6927,9 +6761,6 @@
6927 .text-white {6761 .text-white {
6928 color: var(--color-white);6762 color: var(--color-white);
6929 }6763 }
6930- .lowercase {
6931- text-transform: lowercase;
6932- }
6933 .uppercase {6764 .uppercase {
6934 text-transform: uppercase;6765 text-transform: uppercase;
6935 }6766 }
@@ -8036,7 +7867,6 @@
8036 }7867 }
8037 .text-subtitle {7868 .text-subtitle {
8038 color: var(--color-indigo-500);7869 color: var(--color-indigo-500);
8039->>>>>>> feat/templ-integration
8040 }7870 }
8041 }7871 }
8042 @font-face {7872 @font-face {
@@ -8125,56 +7955,6 @@
8125 .JetBrainsMonoBoldNF, .iconfb {7955 .JetBrainsMonoBoldNF, .iconfb {
8126 font-family: "JetBrainsMono-Nerd-Font-Bold", sans-serif;7956 font-family: "JetBrainsMono-Nerd-Font-Bold", sans-serif;
8127 }7957 }
8128-<<<<<<< HEAD
8129-@property --tw-border-style {
8130- syntax: "*";
8131- inherits: false;
8132- initial-value: solid;
8133-}
8134-@property --tw-blur {
8135- syntax: "*";
8136- inherits: false;
8137-}
8138-@property --tw-brightness {
8139- syntax: "*";
8140- inherits: false;
8141-}
8142-@property --tw-contrast {
8143- syntax: "*";
8144- inherits: false;
8145-}
8146-@property --tw-grayscale {
8147- syntax: "*";
8148- inherits: false;
8149-}
8150-@property --tw-hue-rotate {
8151- syntax: "*";
8152- inherits: false;
8153-}
8154-@property --tw-invert {
8155- syntax: "*";
8156- inherits: false;
8157-}
8158-@property --tw-opacity {
8159- syntax: "*";
8160- inherits: false;
8161-}
8162-@property --tw-saturate {
8163- syntax: "*";
8164- inherits: false;
8165-}
8166-@property --tw-sepia {
8167- syntax: "*";
8168- inherits: false;
8169-}
8170-@property --tw-drop-shadow {
8171- syntax: "*";
8172- inherits: false;
8173-}
8174-@property --tw-duration {
8175- syntax: "*";
8176- inherits: false;
8177-=======
8178 @layer base {7958 @layer base {
8179 :where(:root),:root:has(input.theme-controller[value=light]:checked),[data-theme=light] {7959 :where(:root),:root:has(input.theme-controller[value=light]:checked),[data-theme=light] {
8180 color-scheme: light;7960 color-scheme: light;
@@ -8662,5 +8442,4 @@
8662 to {8442 to {
8663 transform: rotate(360deg);8443 transform: rotate(360deg);
8664 }8444 }
8665->>>>>>> feat/templ-integration
8666 }8445 }

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 7

Tags 106

↑↓ moveenter openesc close