feat: Enhance proxy routing with advanced IP and route management
- Add comprehensive IP consistency checks and route management - Implement periodic route IP verification mechanism - Improve container event handling with more robust IP detection - Enhance logging and error handling for route updates - Add graceful shutdown mechanism for proxy resources - Implement more detailed debugging for container routing and IP changes - Add additional safety checks for container IP mismatches
- commit
- 2358be5dd3185b5938051a3d688fac3686e1a24c
- author
- bamen · · 2025-03-11 14:43 UTC
- parent
- 532f1f1
- signature
- unsigned
2 files · +333 -108
M internal/proxy/core.go +32 -0
| @@ -7,6 +7,7 @@ import ( | |||
| 7 | 7 | "time" | |
| 8 | 8 | ||
| 9 | 9 | "crypto/tls" | |
| 10 | + | "net/http" | |
| 10 | 11 | ||
| 11 | 12 | "github.com/bnema/gordon/internal/common" | |
| 12 | 13 | "github.com/bnema/gordon/internal/interfaces" | |
| @@ -51,6 +52,15 @@ type Proxy struct { | |||
| 51 | 52 | ||
| 52 | 53 | // Store Gordon's own container ID at startup for reliable identification | |
| 53 | 54 | gordonContainerID string | |
| 55 | + | ||
| 56 | + | reverseProxyClient *http.Client | |
| 57 | + | processingGlobalCert bool | |
| 58 | + | dbMaxRetries int | |
| 59 | + | dbRetryBaseDelay time.Duration | |
| 60 | + | acmeManager *autocert.Manager | |
| 61 | + | acmePerDomainRefreshMutex sync.Mutex | |
| 62 | + | acmeDirCache autocert.DirCache | |
| 63 | + | shutdown chan struct{} // Channel for signaling shutdown to background goroutines | |
| 54 | 64 | } | |
| 55 | 65 | ||
| 56 | 66 | // NewProxy creates a new instance of the reverse proxy | |
| @@ -118,6 +128,7 @@ func NewProxy(app interfaces.AppInterface) (*Proxy, error) { | |||
| 118 | 128 | lastBlockedLog: time.Time{}, // Zero time | |
| 119 | 129 | recentlyBlocked: make(map[string]time.Time), | |
| 120 | 130 | gordonContainerID: ourContainerID, | |
| 131 | + | shutdown: make(chan struct{}), | |
| 121 | 132 | } | |
| 122 | 133 | ||
| 123 | 134 | // Now add the middleware with the proxy reference | |
| @@ -129,3 +140,24 @@ func NewProxy(app interfaces.AppInterface) (*Proxy, error) { | |||
| 129 | 140 | logger.Debug("Reverse proxy initialized") | |
| 130 | 141 | return p, nil | |
| 131 | 142 | } | |
| 143 | + | ||
| 144 | + | // Close cleans up resources used by the proxy | |
| 145 | + | func (p *Proxy) Close() { | |
| 146 | + | // Signal all background goroutines to stop | |
| 147 | + | close(p.shutdown) | |
| 148 | + | ||
| 149 | + | // Close the HTTP and HTTPS servers if they exist | |
| 150 | + | if p.httpServer != nil { | |
| 151 | + | if err := p.httpServer.Close(); err != nil { | |
| 152 | + | logger.Error("Error closing HTTP server", "error", err) | |
| 153 | + | } | |
| 154 | + | } | |
| 155 | + | ||
| 156 | + | if p.httpsServer != nil { | |
| 157 | + | if err := p.httpsServer.Close(); err != nil { | |
| 158 | + | logger.Error("Error closing HTTPS server", "error", err) | |
| 159 | + | } | |
| 160 | + | } | |
| 161 | + | ||
| 162 | + | logger.Info("Proxy resources cleaned up") | |
| 163 | + | } | |
| @@ -7,6 +7,7 @@ import ( | |||||
| 7 | "time" | 7 | "time" | ||
| 8 | 8 | ||||
| 9 | "crypto/tls" | 9 | "crypto/tls" | ||
| 10 | + | "net/http" | |||
| 10 | 11 | ||||
| 11 | "github.com/bnema/gordon/internal/common" | 12 | "github.com/bnema/gordon/internal/common" | ||
| 12 | "github.com/bnema/gordon/internal/interfaces" | 13 | "github.com/bnema/gordon/internal/interfaces" | ||
| @@ -51,6 +52,15 @@ type Proxy struct { | |||||
| 51 | 52 | ||||
| 52 | // Store Gordon's own container ID at startup for reliable identification | 53 | // Store Gordon's own container ID at startup for reliable identification | ||
| 53 | gordonContainerID string | 54 | gordonContainerID string | ||
| 55 | + | ||||
| 56 | + | reverseProxyClient *http.Client | |||
| 57 | + | processingGlobalCert bool | |||
| 58 | + | dbMaxRetries int | |||
| 59 | + | dbRetryBaseDelay time.Duration | |||
| 60 | + | acmeManager *autocert.Manager | |||
| 61 | + | acmePerDomainRefreshMutex sync.Mutex | |||
| 62 | + | acmeDirCache autocert.DirCache | |||
| 63 | + | shutdown chan struct{} // Channel for signaling shutdown to background goroutines | |||
| 54 | } | 64 | } | ||
| 55 | 65 | ||||
| 56 | // NewProxy creates a new instance of the reverse proxy | 66 | // NewProxy creates a new instance of the reverse proxy | ||
| @@ -118,6 +128,7 @@ func NewProxy(app interfaces.AppInterface) (*Proxy, error) { | |||||
| 118 | lastBlockedLog: time.Time{}, // Zero time | 128 | lastBlockedLog: time.Time{}, // Zero time | ||
| 119 | recentlyBlocked: make(map[string]time.Time), | 129 | recentlyBlocked: make(map[string]time.Time), | ||
| 120 | gordonContainerID: ourContainerID, | 130 | gordonContainerID: ourContainerID, | ||
| 131 | + | shutdown: make(chan struct{}), | |||
| 121 | } | 132 | } | ||
| 122 | 133 | ||||
| 123 | // Now add the middleware with the proxy reference | 134 | // Now add the middleware with the proxy reference | ||
| @@ -129,3 +140,24 @@ func NewProxy(app interfaces.AppInterface) (*Proxy, error) { | |||||
| 129 | logger.Debug("Reverse proxy initialized") | 140 | logger.Debug("Reverse proxy initialized") | ||
| 130 | return p, nil | 141 | return p, nil | ||
| 131 | } | 142 | } | ||
| 143 | + | ||||
| 144 | + | // Close cleans up resources used by the proxy | |||
| 145 | + | func (p *Proxy) Close() { | |||
| 146 | + | // Signal all background goroutines to stop | |||
| 147 | + | close(p.shutdown) | |||
| 148 | + | ||||
| 149 | + | // Close the HTTP and HTTPS servers if they exist | |||
| 150 | + | if p.httpServer != nil { | |||
| 151 | + | if err := p.httpServer.Close(); err != nil { | |||
| 152 | + | logger.Error("Error closing HTTP server", "error", err) | |||
| 153 | + | } | |||
| 154 | + | } | |||
| 155 | + | ||||
| 156 | + | if p.httpsServer != nil { | |||
| 157 | + | if err := p.httpsServer.Close(); err != nil { | |||
| 158 | + | logger.Error("Error closing HTTPS server", "error", err) | |||
| 159 | + | } | |||
| 160 | + | } | |||
| 161 | + | ||||
| 162 | + | logger.Info("Proxy resources cleaned up") | |||
| 163 | + | } | |||
M internal/proxy/routes.go +301 -108
| @@ -712,17 +712,55 @@ func (p *Proxy) ForceUpdateRouteIP(domain string, newIP string) error { | |||
| 712 | 712 | p.mu.RUnlock() | |
| 713 | 713 | ||
| 714 | 714 | if !exists { | |
| 715 | + | logger.Warn("Route not found for domain when updating IP", "domain", domain) | |
| 715 | 716 | return fmt.Errorf("route not found for domain: %s", domain) | |
| 716 | 717 | } | |
| 717 | 718 | ||
| 718 | 719 | // Get the container ID from the route | |
| 719 | 720 | containerID := route.ContainerID | |
| 720 | 721 | ||
| 722 | + | // Add additional debug to track the IP update | |
| 723 | + | logger.Debug("Attempting to force update route IP", | |
| 724 | + | "domain", domain, | |
| 725 | + | "container_id", containerID, | |
| 726 | + | "old_ip", route.ContainerIP, | |
| 727 | + | "new_ip", newIP) | |
| 728 | + | ||
| 729 | + | // Double-check that the container exists and is running | |
| 730 | + | containerInfo, err := docker.GetContainerInfo(containerID) | |
| 731 | + | if err != nil { | |
| 732 | + | logger.Warn("Container not found when updating route IP, might have been recreated", | |
| 733 | + | "domain", domain, | |
| 734 | + | "container_id", containerID, | |
| 735 | + | "error", err) | |
| 736 | + | // Continue with the update anyway, as the container ID might still be valid | |
| 737 | + | } else { | |
| 738 | + | logger.Debug("Container exists when updating route IP", | |
| 739 | + | "domain", domain, | |
| 740 | + | "container_id", containerID, | |
| 741 | + | "container_state", containerInfo.State.Status) | |
| 742 | + | ||
| 743 | + | // Verify the new IP matches what's in the container | |
| 744 | + | networkName := p.app.GetConfig().ContainerEngine.Network | |
| 745 | + | if containerInfo.NetworkSettings != nil && containerInfo.NetworkSettings.Networks != nil { | |
| 746 | + | if networkSettings, exists := containerInfo.NetworkSettings.Networks[networkName]; exists && | |
| 747 | + | networkSettings.IPAddress != "" && networkSettings.IPAddress != newIP { | |
| 748 | + | logger.Warn("New IP doesn't match container network IP, using container's actual IP", | |
| 749 | + | "domain", domain, | |
| 750 | + | "provided_ip", newIP, | |
| 751 | + | "container_actual_ip", networkSettings.IPAddress) | |
| 752 | + | newIP = networkSettings.IPAddress | |
| 753 | + | } | |
| 754 | + | } | |
| 755 | + | } | |
| 756 | + | ||
| 721 | 757 | // First, update the in-memory route immediately | |
| 722 | 758 | p.mu.Lock() | |
| 723 | 759 | if r, ok := p.routes[domain]; ok { | |
| 724 | 760 | oldIP := r.ContainerIP | |
| 725 | 761 | r.ContainerIP = newIP | |
| 762 | + | // Make sure the route is also marked as active | |
| 763 | + | r.Active = true | |
| 726 | 764 | logger.Info("Force updated route IP in-memory", | |
| 727 | 765 | "domain", domain, | |
| 728 | 766 | "old_ip", oldIP, | |
| @@ -735,33 +773,38 @@ func (p *Proxy) ForceUpdateRouteIP(domain string, newIP string) error { | |||
| 735 | 773 | var domainID string | |
| 736 | 774 | rows, err := p.dbQueryWithRetry("SELECT id FROM domain WHERE name = ?", domain) | |
| 737 | 775 | if err != nil { | |
| 776 | + | logger.Error("Failed to query domain for IP update", "domain", domain, "error", err) | |
| 738 | 777 | return fmt.Errorf("failed to query domain: %w", err) | |
| 739 | 778 | } | |
| 740 | 779 | ||
| 741 | 780 | if !rows.Next() { | |
| 742 | 781 | rows.Close() | |
| 782 | + | logger.Error("Domain not found in database when updating IP", "domain", domain) | |
| 743 | 783 | return fmt.Errorf("domain not found: %s", domain) | |
| 744 | 784 | } | |
| 745 | 785 | ||
| 746 | 786 | if err := rows.Scan(&domainID); err != nil { | |
| 747 | 787 | rows.Close() | |
| 788 | + | logger.Error("Failed to scan domain ID for IP update", "domain", domain, "error", err) | |
| 748 | 789 | return fmt.Errorf("failed to scan domain ID: %w", err) | |
| 749 | 790 | } | |
| 750 | 791 | rows.Close() | |
| 751 | 792 | ||
| 752 | - | // Update the proxy_route record with retry | |
| 793 | + | // Update the proxy_route record with retry and ensure it's marked as active | |
| 753 | 794 | now := time.Now().Format(time.RFC3339) | |
| 754 | 795 | result, err := p.dbExecWithRetry(` | |
| 755 | - | UPDATE proxy_route SET container_ip = ?, updated_at = ? | |
| 796 | + | UPDATE proxy_route SET container_ip = ?, active = 1, updated_at = ? | |
| 756 | 797 | WHERE domain_id = ? AND container_id = ?`, | |
| 757 | 798 | newIP, now, domainID, containerID) | |
| 758 | 799 | ||
| 759 | 800 | if err != nil { | |
| 801 | + | logger.Error("Failed to update IP in database", "domain", domain, "error", err) | |
| 760 | 802 | return fmt.Errorf("failed to update database: %w", err) | |
| 761 | 803 | } | |
| 762 | 804 | ||
| 763 | 805 | rowsAffected, _ := result.RowsAffected() | |
| 764 | 806 | if rowsAffected == 0 { | |
| 807 | + | logger.Warn("No rows affected when updating route IP in database", "domain", domain) | |
| 765 | 808 | return fmt.Errorf("no rows affected when updating route in database") | |
| 766 | 809 | } | |
| 767 | 810 | ||
| @@ -1056,6 +1099,72 @@ func (p *Proxy) StartContainerEventListener() error { | |||
| 1056 | 1099 | ||
| 1057 | 1100 | // Create a callback function to handle container events | |
| 1058 | 1101 | containerEventCallback := func(containerID, containerName, containerIP string) { | |
| 1102 | + | logger.Debug("Container event (start/restart) received", | |
| 1103 | + | "container_id", containerID, | |
| 1104 | + | "container_name", containerName, | |
| 1105 | + | "container_ip", containerIP) | |
| 1106 | + | ||
| 1107 | + | // First, check if there are any existing routes for this container ID | |
| 1108 | + | var existingRoutes = make(map[string]*ProxyRouteInfo) | |
| 1109 | + | p.mu.RLock() | |
| 1110 | + | for domain, route := range p.routes { | |
| 1111 | + | if route.ContainerID == containerID { | |
| 1112 | + | existingRoutes[domain] = route | |
| 1113 | + | } | |
| 1114 | + | } | |
| 1115 | + | p.mu.RUnlock() | |
| 1116 | + | ||
| 1117 | + | // If we found existing routes, check if the IP has changed | |
| 1118 | + | if len(existingRoutes) > 0 { | |
| 1119 | + | logger.Debug("Found existing routes for container", | |
| 1120 | + | "container_id", containerID, | |
| 1121 | + | "route_count", len(existingRoutes)) | |
| 1122 | + | ||
| 1123 | + | // Verify the container IP with an additional check | |
| 1124 | + | verifiedIP, err := docker.GetContainerIPFromNetwork(containerID, networkName) | |
| 1125 | + | if err == nil && verifiedIP != "" && verifiedIP != containerIP { | |
| 1126 | + | logger.Warn("Container IP mismatch from event vs. direct network check", | |
| 1127 | + | "container_id", containerID, | |
| 1128 | + | "event_ip", containerIP, | |
| 1129 | + | "network_ip", verifiedIP) | |
| 1130 | + | ||
| 1131 | + | // Use the directly checked IP as it's more reliable | |
| 1132 | + | containerIP = verifiedIP | |
| 1133 | + | } | |
| 1134 | + | ||
| 1135 | + | // Check and update each route if IP has changed | |
| 1136 | + | for domain, route := range existingRoutes { | |
| 1137 | + | if route.ContainerIP != containerIP { | |
| 1138 | + | logger.Info("Container restarted with new IP - updating route", | |
| 1139 | + | "container_id", containerID, | |
| 1140 | + | "container_name", containerName, | |
| 1141 | + | "domain", domain, | |
| 1142 | + | "old_ip", route.ContainerIP, | |
| 1143 | + | "new_ip", containerIP) | |
| 1144 | + | ||
| 1145 | + | // Update the route IP | |
| 1146 | + | err := p.ForceUpdateRouteIP(domain, containerIP) | |
| 1147 | + | if err != nil { | |
| 1148 | + | logger.Error("Failed to update IP for restarted container", | |
| 1149 | + | "domain", domain, | |
| 1150 | + | "error", err) | |
| 1151 | + | } | |
| 1152 | + | } else { | |
| 1153 | + | // Even if IP is the same, make sure the route is active | |
| 1154 | + | if !route.Active { | |
| 1155 | + | logger.Info("Reactivating route for restarted container", | |
| 1156 | + | "container_id", containerID, | |
| 1157 | + | "container_name", containerName, | |
| 1158 | + | "domain", domain) | |
| 1159 | + | ||
| 1160 | + | // Just reuse ForceUpdateRouteIP with the same IP to activate the route | |
| 1161 | + | p.ForceUpdateRouteIP(domain, containerIP) | |
| 1162 | + | } | |
| 1163 | + | } | |
| 1164 | + | } | |
| 1165 | + | } | |
| 1166 | + | ||
| 1167 | + | // Call the standard recreation handler for additional checks | |
| 1059 | 1168 | p.HandleContainerRecreation(containerID, containerName, containerIP) | |
| 1060 | 1169 | } | |
| 1061 | 1170 | ||
| @@ -1083,10 +1192,85 @@ func (p *Proxy) StartContainerEventListener() error { | |||
| 1083 | 1192 | } | |
| 1084 | 1193 | } | |
| 1085 | 1194 | ||
| 1195 | + | // Schedule periodic check for routes with stale IPs | |
| 1196 | + | go func() { | |
| 1197 | + | ticker := time.NewTicker(5 * time.Minute) | |
| 1198 | + | defer ticker.Stop() | |
| 1199 | + | ||
| 1200 | + | for { | |
| 1201 | + | select { | |
| 1202 | + | case <-ticker.C: | |
| 1203 | + | logger.Debug("Running periodic check for route IP consistency") | |
| 1204 | + | p.checkRoutesForIPConsistency() | |
| 1205 | + | case <-time.After(24 * time.Hour): // This is just a fallback in case the app is shutting down | |
| 1206 | + | return | |
| 1207 | + | } | |
| 1208 | + | } | |
| 1209 | + | }() | |
| 1210 | + | ||
| 1086 | 1211 | // Start the container event listener | |
| 1087 | 1212 | return docker.ListenForContainerEvents(networkName, containerEventCallback, containerStopCallback) | |
| 1088 | 1213 | } | |
| 1089 | 1214 | ||
| 1215 | + | // checkRoutesForIPConsistency verifies that all active routes have the correct container IP | |
| 1216 | + | func (p *Proxy) checkRoutesForIPConsistency() { | |
| 1217 | + | networkName := p.app.GetConfig().ContainerEngine.Network | |
| 1218 | + | ||
| 1219 | + | // Get a copy of the current routes | |
| 1220 | + | p.mu.RLock() | |
| 1221 | + | routesToCheck := make(map[string]*ProxyRouteInfo) | |
| 1222 | + | for domain, route := range p.routes { | |
| 1223 | + | if route.Active { | |
| 1224 | + | routesToCheck[domain] = &ProxyRouteInfo{ | |
| 1225 | + | Domain: route.Domain, | |
| 1226 | + | ContainerID: route.ContainerID, | |
| 1227 | + | ContainerIP: route.ContainerIP, | |
| 1228 | + | ContainerPort: route.ContainerPort, | |
| 1229 | + | Protocol: route.Protocol, | |
| 1230 | + | Path: route.Path, | |
| 1231 | + | Active: route.Active, | |
| 1232 | + | } | |
| 1233 | + | } | |
| 1234 | + | } | |
| 1235 | + | p.mu.RUnlock() | |
| 1236 | + | ||
| 1237 | + | // Check each active route | |
| 1238 | + | for domain, route := range routesToCheck { | |
| 1239 | + | // Skip checking admin domain | |
| 1240 | + | if domain == p.app.GetConfig().Http.FullDomain() { | |
| 1241 | + | continue | |
| 1242 | + | } | |
| 1243 | + | ||
| 1244 | + | // Verify the current container IP | |
| 1245 | + | currentIP, err := docker.GetContainerIPFromNetwork(route.ContainerID, networkName) | |
| 1246 | + | if err != nil { | |
| 1247 | + | // Container might not exist anymore | |
| 1248 | + | logger.Warn("Failed to get current IP for container during consistency check", | |
| 1249 | + | "domain", domain, | |
| 1250 | + | "container_id", route.ContainerID, | |
| 1251 | + | "error", err) | |
| 1252 | + | continue | |
| 1253 | + | } | |
| 1254 | + | ||
| 1255 | + | // If IP has changed, update the route | |
| 1256 | + | if currentIP != "" && currentIP != route.ContainerIP { | |
| 1257 | + | logger.Info("Detected IP mismatch during consistency check", | |
| 1258 | + | "domain", domain, | |
| 1259 | + | "container_id", route.ContainerID, | |
| 1260 | + | "old_ip", route.ContainerIP, | |
| 1261 | + | "current_ip", currentIP) | |
| 1262 | + | ||
| 1263 | + | // Update the route | |
| 1264 | + | err := p.ForceUpdateRouteIP(domain, currentIP) | |
| 1265 | + | if err != nil { | |
| 1266 | + | logger.Error("Failed to update IP during consistency check", | |
| 1267 | + | "domain", domain, | |
| 1268 | + | "error", err) | |
| 1269 | + | } | |
| 1270 | + | } | |
| 1271 | + | } | |
| 1272 | + | } | |
| 1273 | + | ||
| 1090 | 1274 | // markRoutesInactive marks the specified routes as inactive in the database and memory | |
| 1091 | 1275 | func (p *Proxy) markRoutesInactive(domains []string) { | |
| 1092 | 1276 | if len(domains) == 0 { | |
| @@ -1510,143 +1694,152 @@ func (p *Proxy) detectGordonContainer() string { | |||
| 1510 | 1694 | return "gordon" | |
| 1511 | 1695 | } | |
| 1512 | 1696 | ||
| 1513 | - | // proxyRequest handles the actual proxying of the request to the target container | |
| 1697 | + | // proxyRequest proxies an HTTP request to a container | |
| 1514 | 1698 | func (p *Proxy) proxyRequest(c echo.Context, route *ProxyRouteInfo) error { | |
| 1515 | 1699 | host := c.Request().Host | |
| 1700 | + | containerID := route.ContainerID | |
| 1701 | + | containerIP := route.ContainerIP | |
| 1702 | + | containerPort := route.ContainerPort | |
| 1703 | + | targetProtocol := route.Protocol | |
| 1516 | 1704 | ||
| 1517 | - | // For container routes, verify the IP is reachable before proxying | |
| 1518 | - | if host != p.app.GetConfig().Http.FullDomain() { | |
| 1519 | - | // Test TCP connection to container before proxying | |
| 1520 | - | dialer := net.Dialer{Timeout: 500 * time.Millisecond} | |
| 1521 | - | // Format address properly for both IPv4 and IPv6 | |
| 1522 | - | target := route.ContainerIP | |
| 1523 | - | if strings.Contains(target, ":") { | |
| 1524 | - | // IPv6 address needs to be wrapped in brackets | |
| 1525 | - | target = "[" + target + "]" | |
| 1526 | - | } | |
| 1527 | - | conn, err := dialer.Dial("tcp", net.JoinHostPort(target, route.ContainerPort)) | |
| 1528 | - | if err != nil { | |
| 1529 | - | // Connection failed, check if container exists but has a new IP | |
| 1530 | - | containerID := route.ContainerID | |
| 1531 | - | if containerID != "" { | |
| 1532 | - | // Try to get container info and check if it has a different IP | |
| 1533 | - | containerInfo, err := docker.GetContainerInfo(containerID) | |
| 1534 | - | if err == nil && containerInfo.NetworkSettings != nil { | |
| 1535 | - | // Check if container has a different IP | |
| 1536 | - | networkName := p.app.GetConfig().ContainerEngine.Network | |
| 1537 | - | if networkSettings, exists := containerInfo.NetworkSettings.Networks[networkName]; exists && | |
| 1538 | - | networkSettings.IPAddress != "" && networkSettings.IPAddress != route.ContainerIP { | |
| 1539 | - | // Found container with different IP, update the route and use new IP for this request | |
| 1540 | - | logger.Warn("Container IP mismatch detected, using live IP instead of cached value", | |
| 1541 | - | "domain", host, | |
| 1542 | - | "cached_ip", route.ContainerIP, | |
| 1543 | - | "live_ip", networkSettings.IPAddress) | |
| 1544 | - | ||
| 1545 | - | // Update the in-memory route for this request | |
| 1546 | - | route.ContainerIP = networkSettings.IPAddress | |
| 1547 | - | ||
| 1548 | - | // Update the database in background to persist the change | |
| 1549 | - | go func(domain, containerID, containerIP, containerPort string) { | |
| 1550 | - | // Get the domain ID | |
| 1551 | - | var domainID string | |
| 1552 | - | err := p.app.GetDB().QueryRow("SELECT id FROM domain WHERE name = ?", domain).Scan(&domainID) | |
| 1553 | - | if err != nil { | |
| 1554 | - | logger.Error("Failed to find domain ID for IP update", "domain", domain, "error", err) | |
| 1555 | - | return | |
| 1556 | - | } | |
| 1557 | - | ||
| 1558 | - | // Update the proxy_route record | |
| 1559 | - | now := time.Now().Format(time.RFC3339) | |
| 1560 | - | _, err = p.app.GetDB().Exec(` | |
| 1561 | - | UPDATE proxy_route SET container_ip = ?, updated_at = ? | |
| 1562 | - | WHERE domain_id = ? AND container_id = ?`, | |
| 1563 | - | containerIP, now, domainID, containerID) | |
| 1564 | - | ||
| 1565 | - | if err != nil { | |
| 1566 | - | logger.Error("Failed to update container IP in database", | |
| 1567 | - | "domain", domain, | |
| 1568 | - | "containerID", containerID, | |
| 1569 | - | "error", err) | |
| 1570 | - | } else { | |
| 1571 | - | logger.Info("Updated container IP in database after live detection", | |
| 1572 | - | "domain", domain, | |
| 1573 | - | "containerID", containerID, | |
| 1574 | - | "old_ip", route.ContainerIP, | |
| 1575 | - | "new_ip", containerIP) | |
| 1576 | - | } | |
| 1577 | - | }(host, containerID, networkSettings.IPAddress, route.ContainerPort) | |
| 1578 | - | } else { | |
| 1579 | - | logger.Debug("Container exists but connection still failed", | |
| 1580 | - | "domain", host, | |
| 1581 | - | "container_id", containerID, | |
| 1582 | - | "container_ip", route.ContainerIP, | |
| 1583 | - | "container_port", route.ContainerPort, | |
| 1584 | - | "error", err) | |
| 1585 | - | } | |
| 1705 | + | // Always verify the container IP is current before proxying | |
| 1706 | + | networkName := p.app.GetConfig().ContainerEngine.Network | |
| 1707 | + | ||
| 1708 | + | // Get the latest container info to check if IP has changed | |
| 1709 | + | containerInfo, err := docker.GetContainerInfo(containerID) | |
| 1710 | + | if err == nil && containerInfo.NetworkSettings != nil { | |
| 1711 | + | // Check if container has a different IP | |
| 1712 | + | if networkSettings, exists := containerInfo.NetworkSettings.Networks[networkName]; exists && | |
| 1713 | + | networkSettings.IPAddress != "" && networkSettings.IPAddress != containerIP { | |
| 1714 | + | // Found container with different IP, update the route and use new IP for this request | |
| 1715 | + | logger.Warn("Container IP mismatch detected, using live IP instead of cached value", | |
| 1716 | + | "domain", host, | |
| 1717 | + | "cached_ip", containerIP, | |
| 1718 | + | "live_ip", networkSettings.IPAddress) | |
| 1719 | + | ||
| 1720 | + | // Use the new IP for this request | |
| 1721 | + | containerIP = networkSettings.IPAddress | |
| 1722 | + | ||
| 1723 | + | // Update routes in the background | |
| 1724 | + | go func(domain, containerID, containerIP, containerPort string) { | |
| 1725 | + | err := p.ForceUpdateRouteIP(domain, containerIP) | |
| 1726 | + | if err != nil { | |
| 1727 | + | logger.Error("Failed to update IP after mismatch detection", | |
| 1728 | + | "domain", domain, | |
| 1729 | + | "error", err) | |
| 1586 | 1730 | } | |
| 1587 | - | } | |
| 1588 | - | } else { | |
| 1589 | - | // Connection successful, close it | |
| 1590 | - | conn.Close() | |
| 1731 | + | }(host, containerID, containerIP, containerPort) | |
| 1591 | 1732 | } | |
| 1592 | 1733 | } | |
| 1593 | 1734 | ||
| 1594 | - | // Create the target URL | |
| 1595 | - | targetURL := &url.URL{ | |
| 1596 | - | Scheme: "http", // Always use HTTP for internal connections | |
| 1735 | + | // Construct the target URL | |
| 1736 | + | targetURL := fmt.Sprintf("%s://%s:%s", targetProtocol, containerIP, containerPort) | |
| 1737 | + | ||
| 1738 | + | // Add the path from the original request | |
| 1739 | + | if c.Request().URL.Path != "" { | |
| 1740 | + | targetURL += c.Request().URL.Path | |
| 1597 | 1741 | } | |
| 1598 | 1742 | ||
| 1599 | - | // Log protocol conversion if needed | |
| 1600 | - | if route.Protocol == "https" { | |
| 1601 | - | logger.Debug("Converting HTTPS external route to HTTP for internal connection", | |
| 1602 | - | "domain", host, | |
| 1603 | - | "containerID", route.ContainerID, | |
| 1604 | - | "externalProtocol", "https", | |
| 1605 | - | "internalProtocol", "http") | |
| 1743 | + | // Add query string if present | |
| 1744 | + | if c.Request().URL.RawQuery != "" { | |
| 1745 | + | targetURL += "?" + c.Request().URL.RawQuery | |
| 1606 | 1746 | } | |
| 1607 | 1747 | ||
| 1608 | - | // Format host properly for both IPv4 and IPv6 | |
| 1609 | - | containerIP := route.ContainerIP | |
| 1610 | - | if strings.Contains(containerIP, ":") { | |
| 1611 | - | // IPv6 address needs to be wrapped in brackets | |
| 1612 | - | containerIP = "[" + containerIP + "]" | |
| 1748 | + | // Parse the target URL | |
| 1749 | + | target, err := url.Parse(targetURL) | |
| 1750 | + | if err != nil { | |
| 1751 | + | logger.Error("Failed to parse target URL", | |
| 1752 | + | "url", targetURL, | |
| 1753 | + | "error", err) | |
| 1754 | + | return echo.NewHTTPError(http.StatusInternalServerError, "Failed to parse target URL") | |
| 1755 | + | } | |
| 1756 | + | ||
| 1757 | + | // Special logging for admin domain | |
| 1758 | + | if host == p.app.GetConfig().Http.FullDomain() { | |
| 1759 | + | logger.Debug("Proxying request to admin domain", | |
| 1760 | + | "domain", host, | |
| 1761 | + | "target", targetURL, | |
| 1762 | + | "path", c.Request().URL.Path) | |
| 1763 | + | } else { | |
| 1764 | + | logger.Debug("Proxying request host", | |
| 1765 | + | "host", host, | |
| 1766 | + | "target", targetURL, | |
| 1767 | + | "path", c.Request().URL.Path, | |
| 1768 | + | "clientIP", c.RealIP(), | |
| 1769 | + | "remoteAddr", c.Request().RemoteAddr, | |
| 1770 | + | "originalProtocol", c.Scheme()) | |
| 1613 | 1771 | } | |
| 1614 | - | targetURL.Host = fmt.Sprintf("%s:%s", containerIP, route.ContainerPort) | |
| 1615 | 1772 | ||
| 1616 | 1773 | // Create a reverse proxy | |
| 1617 | - | proxy := httputil.NewSingleHostReverseProxy(targetURL) | |
| 1774 | + | proxy := httputil.NewSingleHostReverseProxy(target) | |
| 1775 | + | ||
| 1776 | + | // Use our custom transport with longer timeouts | |
| 1777 | + | if p.reverseProxyClient != nil { | |
| 1778 | + | proxy.Transport = p.reverseProxyClient.Transport | |
| 1779 | + | } | |
| 1618 | 1780 | ||
| 1619 | - | // Update headers to allow for SSL redirection | |
| 1781 | + | // Set up request director to modify the request before it's sent | |
| 1620 | 1782 | originalDirector := proxy.Director | |
| 1621 | 1783 | proxy.Director = func(req *http.Request) { | |
| 1784 | + | // Call the original director first | |
| 1622 | 1785 | originalDirector(req) | |
| 1623 | - | // Use the original protocol for forwarded proto header | |
| 1624 | - | forwardedProto := "http" | |
| 1625 | - | if route.Protocol == "https" { | |
| 1626 | - | forwardedProto = "https" | |
| 1786 | + | ||
| 1787 | + | // Update the Host header to the target host | |
| 1788 | + | req.Host = target.Host | |
| 1789 | + | ||
| 1790 | + | // Set X-Forwarded-For | |
| 1791 | + | if clientIP := c.RealIP(); clientIP != "" { | |
| 1792 | + | req.Header.Set("X-Forwarded-For", clientIP) | |
| 1793 | + | req.Header.Set("X-Real-IP", clientIP) | |
| 1627 | 1794 | } | |
| 1628 | - | req.Header.Set("X-Forwarded-Proto", forwardedProto) | |
| 1629 | - | req.Header.Set("X-Forwarded-Host", host) | |
| 1630 | - | req.Header.Set("X-Forwarded-For", c.RealIP()) | |
| 1631 | - | req.Header.Set("X-Real-IP", c.RealIP()) | |
| 1632 | 1795 | ||
| 1633 | - | // Log all headers for troubleshooting | |
| 1796 | + | // Set X-Forwarded-Proto to the original protocol | |
| 1797 | + | req.Header.Set("X-Forwarded-Proto", c.Scheme()) | |
| 1798 | + | ||
| 1799 | + | // Log request headers for debugging | |
| 1634 | 1800 | logger.Debug("Request headers for proxy", | |
| 1635 | 1801 | "domain", host, | |
| 1636 | 1802 | "X-Forwarded-For", req.Header.Get("X-Forwarded-For"), | |
| 1637 | 1803 | "X-Real-IP", req.Header.Get("X-Real-IP")) | |
| 1638 | 1804 | } | |
| 1639 | 1805 | ||
| 1640 | - | // Add error handling | |
| 1641 | - | proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) { | |
| 1806 | + | // Log errors that occur during proxying | |
| 1807 | + | proxy.ErrorHandler = func(rw http.ResponseWriter, req *http.Request, err error) { | |
| 1642 | 1808 | logger.Error("Proxy error", | |
| 1643 | 1809 | "host", host, | |
| 1644 | - | "path", r.URL.Path, | |
| 1810 | + | "target", targetURL, | |
| 1645 | 1811 | "error", err) | |
| 1646 | - | c.String(http.StatusBadGateway, "Proxy Error: "+err.Error()) | |
| 1812 | + | ||
| 1813 | + | // Check if this is a connection refused error | |
| 1814 | + | if strings.Contains(err.Error(), "connection refused") || | |
| 1815 | + | strings.Contains(err.Error(), "no route to host") || | |
| 1816 | + | strings.Contains(err.Error(), "i/o timeout") { | |
| 1817 | + | // Container might be down or unreachable - mark the route as inactive | |
| 1818 | + | logger.Warn("Container appears to be down, marking route as inactive", | |
| 1819 | + | "domain", host, | |
| 1820 | + | "container_id", containerID) | |
| 1821 | + | ||
| 1822 | + | // Find domains for this container and mark them inactive | |
| 1823 | + | var domains []string | |
| 1824 | + | p.mu.RLock() | |
| 1825 | + | for d, r := range p.routes { | |
| 1826 | + | if r.ContainerID == containerID { | |
| 1827 | + | domains = append(domains, d) | |
| 1828 | + | } | |
| 1829 | + | } | |
| 1830 | + | p.mu.RUnlock() | |
| 1831 | + | ||
| 1832 | + | if len(domains) > 0 { | |
| 1833 | + | go p.markRoutesInactive(domains) | |
| 1834 | + | } | |
| 1835 | + | } | |
| 1836 | + | ||
| 1837 | + | // Return a 502 Bad Gateway error | |
| 1838 | + | c.Response().WriteHeader(http.StatusBadGateway) | |
| 1839 | + | c.Response().Write([]byte("Bad Gateway: Container is unreachable")) | |
| 1647 | 1840 | } | |
| 1648 | 1841 | ||
| 1649 | - | // Serve the request | |
| 1842 | + | // Serve the request using the reverse proxy | |
| 1650 | 1843 | proxy.ServeHTTP(c.Response(), c.Request()) | |
| 1651 | 1844 | return nil | |
| 1652 | 1845 | } | |
| @@ -712,17 +712,55 @@ func (p *Proxy) ForceUpdateRouteIP(domain string, newIP string) error { | |||||
| 712 | p.mu.RUnlock() | 712 | p.mu.RUnlock() | ||
| 713 | 713 | ||||
| 714 | if !exists { | 714 | if !exists { | ||
| 715 | + | logger.Warn("Route not found for domain when updating IP", "domain", domain) | |||
| 715 | return fmt.Errorf("route not found for domain: %s", domain) | 716 | return fmt.Errorf("route not found for domain: %s", domain) | ||
| 716 | } | 717 | } | ||
| 717 | 718 | ||||
| 718 | // Get the container ID from the route | 719 | // Get the container ID from the route | ||
| 719 | containerID := route.ContainerID | 720 | containerID := route.ContainerID | ||
| 720 | 721 | ||||
| 722 | + | // Add additional debug to track the IP update | |||
| 723 | + | logger.Debug("Attempting to force update route IP", | |||
| 724 | + | "domain", domain, | |||
| 725 | + | "container_id", containerID, | |||
| 726 | + | "old_ip", route.ContainerIP, | |||
| 727 | + | "new_ip", newIP) | |||
| 728 | + | ||||
| 729 | + | // Double-check that the container exists and is running | |||
| 730 | + | containerInfo, err := docker.GetContainerInfo(containerID) | |||
| 731 | + | if err != nil { | |||
| 732 | + | logger.Warn("Container not found when updating route IP, might have been recreated", | |||
| 733 | + | "domain", domain, | |||
| 734 | + | "container_id", containerID, | |||
| 735 | + | "error", err) | |||
| 736 | + | // Continue with the update anyway, as the container ID might still be valid | |||
| 737 | + | } else { | |||
| 738 | + | logger.Debug("Container exists when updating route IP", | |||
| 739 | + | "domain", domain, | |||
| 740 | + | "container_id", containerID, | |||
| 741 | + | "container_state", containerInfo.State.Status) | |||
| 742 | + | ||||
| 743 | + | // Verify the new IP matches what's in the container | |||
| 744 | + | networkName := p.app.GetConfig().ContainerEngine.Network | |||
| 745 | + | if containerInfo.NetworkSettings != nil && containerInfo.NetworkSettings.Networks != nil { | |||
| 746 | + | if networkSettings, exists := containerInfo.NetworkSettings.Networks[networkName]; exists && | |||
| 747 | + | networkSettings.IPAddress != "" && networkSettings.IPAddress != newIP { | |||
| 748 | + | logger.Warn("New IP doesn't match container network IP, using container's actual IP", | |||
| 749 | + | "domain", domain, | |||
| 750 | + | "provided_ip", newIP, | |||
| 751 | + | "container_actual_ip", networkSettings.IPAddress) | |||
| 752 | + | newIP = networkSettings.IPAddress | |||
| 753 | + | } | |||
| 754 | + | } | |||
| 755 | + | } | |||
| 756 | + | ||||
| 721 | // First, update the in-memory route immediately | 757 | // First, update the in-memory route immediately | ||
| 722 | p.mu.Lock() | 758 | p.mu.Lock() | ||
| 723 | if r, ok := p.routes[domain]; ok { | 759 | if r, ok := p.routes[domain]; ok { | ||
| 724 | oldIP := r.ContainerIP | 760 | oldIP := r.ContainerIP | ||
| 725 | r.ContainerIP = newIP | 761 | r.ContainerIP = newIP | ||
| 762 | + | // Make sure the route is also marked as active | |||
| 763 | + | r.Active = true | |||
| 726 | logger.Info("Force updated route IP in-memory", | 764 | logger.Info("Force updated route IP in-memory", | ||
| 727 | "domain", domain, | 765 | "domain", domain, | ||
| 728 | "old_ip", oldIP, | 766 | "old_ip", oldIP, | ||
| @@ -735,33 +773,38 @@ func (p *Proxy) ForceUpdateRouteIP(domain string, newIP string) error { | |||||
| 735 | var domainID string | 773 | var domainID string | ||
| 736 | rows, err := p.dbQueryWithRetry("SELECT id FROM domain WHERE name = ?", domain) | 774 | rows, err := p.dbQueryWithRetry("SELECT id FROM domain WHERE name = ?", domain) | ||
| 737 | if err != nil { | 775 | if err != nil { | ||
| 776 | + | logger.Error("Failed to query domain for IP update", "domain", domain, "error", err) | |||
| 738 | return fmt.Errorf("failed to query domain: %w", err) | 777 | return fmt.Errorf("failed to query domain: %w", err) | ||
| 739 | } | 778 | } | ||
| 740 | 779 | ||||
| 741 | if !rows.Next() { | 780 | if !rows.Next() { | ||
| 742 | rows.Close() | 781 | rows.Close() | ||
| 782 | + | logger.Error("Domain not found in database when updating IP", "domain", domain) | |||
| 743 | return fmt.Errorf("domain not found: %s", domain) | 783 | return fmt.Errorf("domain not found: %s", domain) | ||
| 744 | } | 784 | } | ||
| 745 | 785 | ||||
| 746 | if err := rows.Scan(&domainID); err != nil { | 786 | if err := rows.Scan(&domainID); err != nil { | ||
| 747 | rows.Close() | 787 | rows.Close() | ||
| 788 | + | logger.Error("Failed to scan domain ID for IP update", "domain", domain, "error", err) | |||
| 748 | return fmt.Errorf("failed to scan domain ID: %w", err) | 789 | return fmt.Errorf("failed to scan domain ID: %w", err) | ||
| 749 | } | 790 | } | ||
| 750 | rows.Close() | 791 | rows.Close() | ||
| 751 | 792 | ||||
| 752 | - | // Update the proxy_route record with retry | 793 | + | // Update the proxy_route record with retry and ensure it's marked as active |
| 753 | now := time.Now().Format(time.RFC3339) | 794 | now := time.Now().Format(time.RFC3339) | ||
| 754 | result, err := p.dbExecWithRetry(` | 795 | result, err := p.dbExecWithRetry(` | ||
| 755 | - | UPDATE proxy_route SET container_ip = ?, updated_at = ? | 796 | + | UPDATE proxy_route SET container_ip = ?, active = 1, updated_at = ? |
| 756 | WHERE domain_id = ? AND container_id = ?`, | 797 | WHERE domain_id = ? AND container_id = ?`, | ||
| 757 | newIP, now, domainID, containerID) | 798 | newIP, now, domainID, containerID) | ||
| 758 | 799 | ||||
| 759 | if err != nil { | 800 | if err != nil { | ||
| 801 | + | logger.Error("Failed to update IP in database", "domain", domain, "error", err) | |||
| 760 | return fmt.Errorf("failed to update database: %w", err) | 802 | return fmt.Errorf("failed to update database: %w", err) | ||
| 761 | } | 803 | } | ||
| 762 | 804 | ||||
| 763 | rowsAffected, _ := result.RowsAffected() | 805 | rowsAffected, _ := result.RowsAffected() | ||
| 764 | if rowsAffected == 0 { | 806 | if rowsAffected == 0 { | ||
| 807 | + | logger.Warn("No rows affected when updating route IP in database", "domain", domain) | |||
| 765 | return fmt.Errorf("no rows affected when updating route in database") | 808 | return fmt.Errorf("no rows affected when updating route in database") | ||
| 766 | } | 809 | } | ||
| 767 | 810 | ||||
| @@ -1056,6 +1099,72 @@ func (p *Proxy) StartContainerEventListener() error { | |||||
| 1056 | 1099 | ||||
| 1057 | // Create a callback function to handle container events | 1100 | // Create a callback function to handle container events | ||
| 1058 | containerEventCallback := func(containerID, containerName, containerIP string) { | 1101 | containerEventCallback := func(containerID, containerName, containerIP string) { | ||
| 1102 | + | logger.Debug("Container event (start/restart) received", | |||
| 1103 | + | "container_id", containerID, | |||
| 1104 | + | "container_name", containerName, | |||
| 1105 | + | "container_ip", containerIP) | |||
| 1106 | + | ||||
| 1107 | + | // First, check if there are any existing routes for this container ID | |||
| 1108 | + | var existingRoutes = make(map[string]*ProxyRouteInfo) | |||
| 1109 | + | p.mu.RLock() | |||
| 1110 | + | for domain, route := range p.routes { | |||
| 1111 | + | if route.ContainerID == containerID { | |||
| 1112 | + | existingRoutes[domain] = route | |||
| 1113 | + | } | |||
| 1114 | + | } | |||
| 1115 | + | p.mu.RUnlock() | |||
| 1116 | + | ||||
| 1117 | + | // If we found existing routes, check if the IP has changed | |||
| 1118 | + | if len(existingRoutes) > 0 { | |||
| 1119 | + | logger.Debug("Found existing routes for container", | |||
| 1120 | + | "container_id", containerID, | |||
| 1121 | + | "route_count", len(existingRoutes)) | |||
| 1122 | + | ||||
| 1123 | + | // Verify the container IP with an additional check | |||
| 1124 | + | verifiedIP, err := docker.GetContainerIPFromNetwork(containerID, networkName) | |||
| 1125 | + | if err == nil && verifiedIP != "" && verifiedIP != containerIP { | |||
| 1126 | + | logger.Warn("Container IP mismatch from event vs. direct network check", | |||
| 1127 | + | "container_id", containerID, | |||
| 1128 | + | "event_ip", containerIP, | |||
| 1129 | + | "network_ip", verifiedIP) | |||
| 1130 | + | ||||
| 1131 | + | // Use the directly checked IP as it's more reliable | |||
| 1132 | + | containerIP = verifiedIP | |||
| 1133 | + | } | |||
| 1134 | + | ||||
| 1135 | + | // Check and update each route if IP has changed | |||
| 1136 | + | for domain, route := range existingRoutes { | |||
| 1137 | + | if route.ContainerIP != containerIP { | |||
| 1138 | + | logger.Info("Container restarted with new IP - updating route", | |||
| 1139 | + | "container_id", containerID, | |||
| 1140 | + | "container_name", containerName, | |||
| 1141 | + | "domain", domain, | |||
| 1142 | + | "old_ip", route.ContainerIP, | |||
| 1143 | + | "new_ip", containerIP) | |||
| 1144 | + | ||||
| 1145 | + | // Update the route IP | |||
| 1146 | + | err := p.ForceUpdateRouteIP(domain, containerIP) | |||
| 1147 | + | if err != nil { | |||
| 1148 | + | logger.Error("Failed to update IP for restarted container", | |||
| 1149 | + | "domain", domain, | |||
| 1150 | + | "error", err) | |||
| 1151 | + | } | |||
| 1152 | + | } else { | |||
| 1153 | + | // Even if IP is the same, make sure the route is active | |||
| 1154 | + | if !route.Active { | |||
| 1155 | + | logger.Info("Reactivating route for restarted container", | |||
| 1156 | + | "container_id", containerID, | |||
| 1157 | + | "container_name", containerName, | |||
| 1158 | + | "domain", domain) | |||
| 1159 | + | ||||
| 1160 | + | // Just reuse ForceUpdateRouteIP with the same IP to activate the route | |||
| 1161 | + | p.ForceUpdateRouteIP(domain, containerIP) | |||
| 1162 | + | } | |||
| 1163 | + | } | |||
| 1164 | + | } | |||
| 1165 | + | } | |||
| 1166 | + | ||||
| 1167 | + | // Call the standard recreation handler for additional checks | |||
| 1059 | p.HandleContainerRecreation(containerID, containerName, containerIP) | 1168 | p.HandleContainerRecreation(containerID, containerName, containerIP) | ||
| 1060 | } | 1169 | } | ||
| 1061 | 1170 | ||||
| @@ -1083,10 +1192,85 @@ func (p *Proxy) StartContainerEventListener() error { | |||||
| 1083 | } | 1192 | } | ||
| 1084 | } | 1193 | } | ||
| 1085 | 1194 | ||||
| 1195 | + | // Schedule periodic check for routes with stale IPs | |||
| 1196 | + | go func() { | |||
| 1197 | + | ticker := time.NewTicker(5 * time.Minute) | |||
| 1198 | + | defer ticker.Stop() | |||
| 1199 | + | ||||
| 1200 | + | for { | |||
| 1201 | + | select { | |||
| 1202 | + | case <-ticker.C: | |||
| 1203 | + | logger.Debug("Running periodic check for route IP consistency") | |||
| 1204 | + | p.checkRoutesForIPConsistency() | |||
| 1205 | + | case <-time.After(24 * time.Hour): // This is just a fallback in case the app is shutting down | |||
| 1206 | + | return | |||
| 1207 | + | } | |||
| 1208 | + | } | |||
| 1209 | + | }() | |||
| 1210 | + | ||||
| 1086 | // Start the container event listener | 1211 | // Start the container event listener | ||
| 1087 | return docker.ListenForContainerEvents(networkName, containerEventCallback, containerStopCallback) | 1212 | return docker.ListenForContainerEvents(networkName, containerEventCallback, containerStopCallback) | ||
| 1088 | } | 1213 | } | ||
| 1089 | 1214 | ||||
| 1215 | + | // checkRoutesForIPConsistency verifies that all active routes have the correct container IP | |||
| 1216 | + | func (p *Proxy) checkRoutesForIPConsistency() { | |||
| 1217 | + | networkName := p.app.GetConfig().ContainerEngine.Network | |||
| 1218 | + | ||||
| 1219 | + | // Get a copy of the current routes | |||
| 1220 | + | p.mu.RLock() | |||
| 1221 | + | routesToCheck := make(map[string]*ProxyRouteInfo) | |||
| 1222 | + | for domain, route := range p.routes { | |||
| 1223 | + | if route.Active { | |||
| 1224 | + | routesToCheck[domain] = &ProxyRouteInfo{ | |||
| 1225 | + | Domain: route.Domain, | |||
| 1226 | + | ContainerID: route.ContainerID, | |||
| 1227 | + | ContainerIP: route.ContainerIP, | |||
| 1228 | + | ContainerPort: route.ContainerPort, | |||
| 1229 | + | Protocol: route.Protocol, | |||
| 1230 | + | Path: route.Path, | |||
| 1231 | + | Active: route.Active, | |||
| 1232 | + | } | |||
| 1233 | + | } | |||
| 1234 | + | } | |||
| 1235 | + | p.mu.RUnlock() | |||
| 1236 | + | ||||
| 1237 | + | // Check each active route | |||
| 1238 | + | for domain, route := range routesToCheck { | |||
| 1239 | + | // Skip checking admin domain | |||
| 1240 | + | if domain == p.app.GetConfig().Http.FullDomain() { | |||
| 1241 | + | continue | |||
| 1242 | + | } | |||
| 1243 | + | ||||
| 1244 | + | // Verify the current container IP | |||
| 1245 | + | currentIP, err := docker.GetContainerIPFromNetwork(route.ContainerID, networkName) | |||
| 1246 | + | if err != nil { | |||
| 1247 | + | // Container might not exist anymore | |||
| 1248 | + | logger.Warn("Failed to get current IP for container during consistency check", | |||
| 1249 | + | "domain", domain, | |||
| 1250 | + | "container_id", route.ContainerID, | |||
| 1251 | + | "error", err) | |||
| 1252 | + | continue | |||
| 1253 | + | } | |||
| 1254 | + | ||||
| 1255 | + | // If IP has changed, update the route | |||
| 1256 | + | if currentIP != "" && currentIP != route.ContainerIP { | |||
| 1257 | + | logger.Info("Detected IP mismatch during consistency check", | |||
| 1258 | + | "domain", domain, | |||
| 1259 | + | "container_id", route.ContainerID, | |||
| 1260 | + | "old_ip", route.ContainerIP, | |||
| 1261 | + | "current_ip", currentIP) | |||
| 1262 | + | ||||
| 1263 | + | // Update the route | |||
| 1264 | + | err := p.ForceUpdateRouteIP(domain, currentIP) | |||
| 1265 | + | if err != nil { | |||
| 1266 | + | logger.Error("Failed to update IP during consistency check", | |||
| 1267 | + | "domain", domain, | |||
| 1268 | + | "error", err) | |||
| 1269 | + | } | |||
| 1270 | + | } | |||
| 1271 | + | } | |||
| 1272 | + | } | |||
| 1273 | + | ||||
| 1090 | // markRoutesInactive marks the specified routes as inactive in the database and memory | 1274 | // markRoutesInactive marks the specified routes as inactive in the database and memory | ||
| 1091 | func (p *Proxy) markRoutesInactive(domains []string) { | 1275 | func (p *Proxy) markRoutesInactive(domains []string) { | ||
| 1092 | if len(domains) == 0 { | 1276 | if len(domains) == 0 { | ||
| @@ -1510,143 +1694,152 @@ func (p *Proxy) detectGordonContainer() string { | |||||
| 1510 | return "gordon" | 1694 | return "gordon" | ||
| 1511 | } | 1695 | } | ||
| 1512 | 1696 | ||||
| 1513 | - | // proxyRequest handles the actual proxying of the request to the target container | 1697 | + | // proxyRequest proxies an HTTP request to a container |
| 1514 | func (p *Proxy) proxyRequest(c echo.Context, route *ProxyRouteInfo) error { | 1698 | func (p *Proxy) proxyRequest(c echo.Context, route *ProxyRouteInfo) error { | ||
| 1515 | host := c.Request().Host | 1699 | host := c.Request().Host | ||
| 1700 | + | containerID := route.ContainerID | |||
| 1701 | + | containerIP := route.ContainerIP | |||
| 1702 | + | containerPort := route.ContainerPort | |||
| 1703 | + | targetProtocol := route.Protocol | |||
| 1516 | 1704 | ||||
| 1517 | - | // For container routes, verify the IP is reachable before proxying | 1705 | + | // Always verify the container IP is current before proxying |
| 1518 | - | if host != p.app.GetConfig().Http.FullDomain() { | 1706 | + | networkName := p.app.GetConfig().ContainerEngine.Network |
| 1519 | - | // Test TCP connection to container before proxying | 1707 | + | |
| 1520 | - | dialer := net.Dialer{Timeout: 500 * time.Millisecond} | 1708 | + | // Get the latest container info to check if IP has changed |
| 1521 | - | // Format address properly for both IPv4 and IPv6 | 1709 | + | containerInfo, err := docker.GetContainerInfo(containerID) |
| 1522 | - | target := route.ContainerIP | 1710 | + | if err == nil && containerInfo.NetworkSettings != nil { |
| 1523 | - | if strings.Contains(target, ":") { | 1711 | + | // Check if container has a different IP |
| 1524 | - | // IPv6 address needs to be wrapped in brackets | 1712 | + | if networkSettings, exists := containerInfo.NetworkSettings.Networks[networkName]; exists && |
| 1525 | - | target = "[" + target + "]" | 1713 | + | networkSettings.IPAddress != "" && networkSettings.IPAddress != containerIP { |
| 1526 | - | } | 1714 | + | // Found container with different IP, update the route and use new IP for this request |
| 1527 | - | conn, err := dialer.Dial("tcp", net.JoinHostPort(target, route.ContainerPort)) | 1715 | + | logger.Warn("Container IP mismatch detected, using live IP instead of cached value", |
| 1528 | - | if err != nil { | 1716 | + | "domain", host, |
| 1529 | - | // Connection failed, check if container exists but has a new IP | 1717 | + | "cached_ip", containerIP, |
| 1530 | - | containerID := route.ContainerID | 1718 | + | "live_ip", networkSettings.IPAddress) |
| 1531 | - | if containerID != "" { | 1719 | + | |
| 1532 | - | // Try to get container info and check if it has a different IP | 1720 | + | // Use the new IP for this request |
| 1533 | - | containerInfo, err := docker.GetContainerInfo(containerID) | 1721 | + | containerIP = networkSettings.IPAddress |
| 1534 | - | if err == nil && containerInfo.NetworkSettings != nil { | 1722 | + | |
| 1535 | - | // Check if container has a different IP | 1723 | + | // Update routes in the background |
| 1536 | - | networkName := p.app.GetConfig().ContainerEngine.Network | 1724 | + | go func(domain, containerID, containerIP, containerPort string) { |
| 1537 | - | if networkSettings, exists := containerInfo.NetworkSettings.Networks[networkName]; exists && | 1725 | + | err := p.ForceUpdateRouteIP(domain, containerIP) |
| 1538 | - | networkSettings.IPAddress != "" && networkSettings.IPAddress != route.ContainerIP { | 1726 | + | if err != nil { |
| 1539 | - | // Found container with different IP, update the route and use new IP for this request | 1727 | + | logger.Error("Failed to update IP after mismatch detection", |
| 1540 | - | logger.Warn("Container IP mismatch detected, using live IP instead of cached value", | 1728 | + | "domain", domain, |
| 1541 | - | "domain", host, | 1729 | + | "error", err) |
| 1542 | - | "cached_ip", route.ContainerIP, | |||
| 1543 | - | "live_ip", networkSettings.IPAddress) | |||
| 1544 | - | ||||
| 1545 | - | // Update the in-memory route for this request | |||
| 1546 | - | route.ContainerIP = networkSettings.IPAddress | |||
| 1547 | - | ||||
| 1548 | - | // Update the database in background to persist the change | |||
| 1549 | - | go func(domain, containerID, containerIP, containerPort string) { | |||
| 1550 | - | // Get the domain ID | |||
| 1551 | - | var domainID string | |||
| 1552 | - | err := p.app.GetDB().QueryRow("SELECT id FROM domain WHERE name = ?", domain).Scan(&domainID) | |||
| 1553 | - | if err != nil { | |||
| 1554 | - | logger.Error("Failed to find domain ID for IP update", "domain", domain, "error", err) | |||
| 1555 | - | return | |||
| 1556 | - | } | |||
| 1557 | - | ||||
| 1558 | - | // Update the proxy_route record | |||
| 1559 | - | now := time.Now().Format(time.RFC3339) | |||
| 1560 | - | _, err = p.app.GetDB().Exec(` | |||
| 1561 | - | UPDATE proxy_route SET container_ip = ?, updated_at = ? | |||
| 1562 | - | WHERE domain_id = ? AND container_id = ?`, | |||
| 1563 | - | containerIP, now, domainID, containerID) | |||
| 1564 | - | ||||
| 1565 | - | if err != nil { | |||
| 1566 | - | logger.Error("Failed to update container IP in database", | |||
| 1567 | - | "domain", domain, | |||
| 1568 | - | "containerID", containerID, | |||
| 1569 | - | "error", err) | |||
| 1570 | - | } else { | |||
| 1571 | - | logger.Info("Updated container IP in database after live detection", | |||
| 1572 | - | "domain", domain, | |||
| 1573 | - | "containerID", containerID, | |||
| 1574 | - | "old_ip", route.ContainerIP, | |||
| 1575 | - | "new_ip", containerIP) | |||
| 1576 | - | } | |||
| 1577 | - | }(host, containerID, networkSettings.IPAddress, route.ContainerPort) | |||
| 1578 | - | } else { | |||
| 1579 | - | logger.Debug("Container exists but connection still failed", | |||
| 1580 | - | "domain", host, | |||
| 1581 | - | "container_id", containerID, | |||
| 1582 | - | "container_ip", route.ContainerIP, | |||
| 1583 | - | "container_port", route.ContainerPort, | |||
| 1584 | - | "error", err) | |||
| 1585 | - | } | |||
| 1586 | } | 1730 | } | ||
| 1587 | - | } | 1731 | + | }(host, containerID, containerIP, containerPort) |
| 1588 | - | } else { | |||
| 1589 | - | // Connection successful, close it | |||
| 1590 | - | conn.Close() | |||
| 1591 | } | 1732 | } | ||
| 1592 | } | 1733 | } | ||
| 1593 | 1734 | ||||
| 1594 | - | // Create the target URL | 1735 | + | // Construct the target URL |
| 1595 | - | targetURL := &url.URL{ | 1736 | + | targetURL := fmt.Sprintf("%s://%s:%s", targetProtocol, containerIP, containerPort) |
| 1596 | - | Scheme: "http", // Always use HTTP for internal connections | 1737 | + | |
| 1738 | + | // Add the path from the original request | |||
| 1739 | + | if c.Request().URL.Path != "" { | |||
| 1740 | + | targetURL += c.Request().URL.Path | |||
| 1597 | } | 1741 | } | ||
| 1598 | 1742 | ||||
| 1599 | - | // Log protocol conversion if needed | 1743 | + | // Add query string if present |
| 1600 | - | if route.Protocol == "https" { | 1744 | + | if c.Request().URL.RawQuery != "" { |
| 1601 | - | logger.Debug("Converting HTTPS external route to HTTP for internal connection", | 1745 | + | targetURL += "?" + c.Request().URL.RawQuery |
| 1602 | - | "domain", host, | |||
| 1603 | - | "containerID", route.ContainerID, | |||
| 1604 | - | "externalProtocol", "https", | |||
| 1605 | - | "internalProtocol", "http") | |||
| 1606 | } | 1746 | } | ||
| 1607 | 1747 | ||||
| 1608 | - | // Format host properly for both IPv4 and IPv6 | 1748 | + | // Parse the target URL |
| 1609 | - | containerIP := route.ContainerIP | 1749 | + | target, err := url.Parse(targetURL) |
| 1610 | - | if strings.Contains(containerIP, ":") { | 1750 | + | if err != nil { |
| 1611 | - | // IPv6 address needs to be wrapped in brackets | 1751 | + | logger.Error("Failed to parse target URL", |
| 1612 | - | containerIP = "[" + containerIP + "]" | 1752 | + | "url", targetURL, |
| 1753 | + | "error", err) | |||
| 1754 | + | return echo.NewHTTPError(http.StatusInternalServerError, "Failed to parse target URL") | |||
| 1755 | + | } | |||
| 1756 | + | ||||
| 1757 | + | // Special logging for admin domain | |||
| 1758 | + | if host == p.app.GetConfig().Http.FullDomain() { | |||
| 1759 | + | logger.Debug("Proxying request to admin domain", | |||
| 1760 | + | "domain", host, | |||
| 1761 | + | "target", targetURL, | |||
| 1762 | + | "path", c.Request().URL.Path) | |||
| 1763 | + | } else { | |||
| 1764 | + | logger.Debug("Proxying request host", | |||
| 1765 | + | "host", host, | |||
| 1766 | + | "target", targetURL, | |||
| 1767 | + | "path", c.Request().URL.Path, | |||
| 1768 | + | "clientIP", c.RealIP(), | |||
| 1769 | + | "remoteAddr", c.Request().RemoteAddr, | |||
| 1770 | + | "originalProtocol", c.Scheme()) | |||
| 1613 | } | 1771 | } | ||
| 1614 | - | targetURL.Host = fmt.Sprintf("%s:%s", containerIP, route.ContainerPort) | |||
| 1615 | 1772 | ||||
| 1616 | // Create a reverse proxy | 1773 | // Create a reverse proxy | ||
| 1617 | - | proxy := httputil.NewSingleHostReverseProxy(targetURL) | 1774 | + | proxy := httputil.NewSingleHostReverseProxy(target) |
| 1775 | + | ||||
| 1776 | + | // Use our custom transport with longer timeouts | |||
| 1777 | + | if p.reverseProxyClient != nil { | |||
| 1778 | + | proxy.Transport = p.reverseProxyClient.Transport | |||
| 1779 | + | } | |||
| 1618 | 1780 | ||||
| 1619 | - | // Update headers to allow for SSL redirection | 1781 | + | // Set up request director to modify the request before it's sent |
| 1620 | originalDirector := proxy.Director | 1782 | originalDirector := proxy.Director | ||
| 1621 | proxy.Director = func(req *http.Request) { | 1783 | proxy.Director = func(req *http.Request) { | ||
| 1784 | + | // Call the original director first | |||
| 1622 | originalDirector(req) | 1785 | originalDirector(req) | ||
| 1623 | - | // Use the original protocol for forwarded proto header | 1786 | + | |
| 1624 | - | forwardedProto := "http" | 1787 | + | // Update the Host header to the target host |
| 1625 | - | if route.Protocol == "https" { | 1788 | + | req.Host = target.Host |
| 1626 | - | forwardedProto = "https" | 1789 | + | |
| 1790 | + | // Set X-Forwarded-For | |||
| 1791 | + | if clientIP := c.RealIP(); clientIP != "" { | |||
| 1792 | + | req.Header.Set("X-Forwarded-For", clientIP) | |||
| 1793 | + | req.Header.Set("X-Real-IP", clientIP) | |||
| 1627 | } | 1794 | } | ||
| 1628 | - | req.Header.Set("X-Forwarded-Proto", forwardedProto) | |||
| 1629 | - | req.Header.Set("X-Forwarded-Host", host) | |||
| 1630 | - | req.Header.Set("X-Forwarded-For", c.RealIP()) | |||
| 1631 | - | req.Header.Set("X-Real-IP", c.RealIP()) | |||
| 1632 | 1795 | ||||
| 1633 | - | // Log all headers for troubleshooting | 1796 | + | // Set X-Forwarded-Proto to the original protocol |
| 1797 | + | req.Header.Set("X-Forwarded-Proto", c.Scheme()) | |||
| 1798 | + | ||||
| 1799 | + | // Log request headers for debugging | |||
| 1634 | logger.Debug("Request headers for proxy", | 1800 | logger.Debug("Request headers for proxy", | ||
| 1635 | "domain", host, | 1801 | "domain", host, | ||
| 1636 | "X-Forwarded-For", req.Header.Get("X-Forwarded-For"), | 1802 | "X-Forwarded-For", req.Header.Get("X-Forwarded-For"), | ||
| 1637 | "X-Real-IP", req.Header.Get("X-Real-IP")) | 1803 | "X-Real-IP", req.Header.Get("X-Real-IP")) | ||
| 1638 | } | 1804 | } | ||
| 1639 | 1805 | ||||
| 1640 | - | // Add error handling | 1806 | + | // Log errors that occur during proxying |
| 1641 | - | proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) { | 1807 | + | proxy.ErrorHandler = func(rw http.ResponseWriter, req *http.Request, err error) { |
| 1642 | logger.Error("Proxy error", | 1808 | logger.Error("Proxy error", | ||
| 1643 | "host", host, | 1809 | "host", host, | ||
| 1644 | - | "path", r.URL.Path, | 1810 | + | "target", targetURL, |
| 1645 | "error", err) | 1811 | "error", err) | ||
| 1646 | - | c.String(http.StatusBadGateway, "Proxy Error: "+err.Error()) | 1812 | + | |
| 1813 | + | // Check if this is a connection refused error | |||
| 1814 | + | if strings.Contains(err.Error(), "connection refused") || | |||
| 1815 | + | strings.Contains(err.Error(), "no route to host") || | |||
| 1816 | + | strings.Contains(err.Error(), "i/o timeout") { | |||
| 1817 | + | // Container might be down or unreachable - mark the route as inactive | |||
| 1818 | + | logger.Warn("Container appears to be down, marking route as inactive", | |||
| 1819 | + | "domain", host, | |||
| 1820 | + | "container_id", containerID) | |||
| 1821 | + | ||||
| 1822 | + | // Find domains for this container and mark them inactive | |||
| 1823 | + | var domains []string | |||
| 1824 | + | p.mu.RLock() | |||
| 1825 | + | for d, r := range p.routes { | |||
| 1826 | + | if r.ContainerID == containerID { | |||
| 1827 | + | domains = append(domains, d) | |||
| 1828 | + | } | |||
| 1829 | + | } | |||
| 1830 | + | p.mu.RUnlock() | |||
| 1831 | + | ||||
| 1832 | + | if len(domains) > 0 { | |||
| 1833 | + | go p.markRoutesInactive(domains) | |||
| 1834 | + | } | |||
| 1835 | + | } | |||
| 1836 | + | ||||
| 1837 | + | // Return a 502 Bad Gateway error | |||
| 1838 | + | c.Response().WriteHeader(http.StatusBadGateway) | |||
| 1839 | + | c.Response().Write([]byte("Bad Gateway: Container is unreachable")) | |||
| 1647 | } | 1840 | } | ||
| 1648 | 1841 | ||||
| 1649 | - | // Serve the request | 1842 | + | // Serve the request using the reverse proxy |
| 1650 | proxy.ServeHTTP(c.Response(), c.Request()) | 1843 | proxy.ServeHTTP(c.Response(), c.Request()) | ||
| 1651 | return nil | 1844 | return nil | ||
| 1652 | } | 1845 | } | ||