fix: rewrite upload image logic client and server side to avoid timeouts

commit: 1186987bb2fe420e324ef0c5032a06303b4581fb
author: bnema · 2y ago · 2024-04-21 01:40 UTC
parent: 0e99910
signature: unsigned

2 files · +47 -30

M internal/cli/cmd/deploy.go +25 -22


@@ -7,6 +7,7 @@ import (
7	7		"net/http"
8	8		"os"
9	9		"strings"
	10	+	"sync"
10	11		"time"
11	12
12	13		"github.com/bnema/gordon/internal/cli"

@@ -85,7 +86,11 @@ func NewDeployCommand(a cli.App) cobra.Command {
85	86		},
86	87		}
87	88
	89	+	var wg sync.WaitGroup
	90	+	wg.Add(1)
	91	+
88	92		go func() {
	93	+	defer wg.Done()
89	94		resp, err := handler.SendHTTPRequest(a, &reqPayload, "POST", "/push")
90	95		if err != nil {
91	96		errCh <- fmt.Errorf("error sending HTTP request: %w", err)

@@ -136,32 +141,30 @@ func NewDeployCommand(a cli.App) cobra.Command {
136	141		errCh <- fmt.Errorf("error running progress bar TUI: %w", err)
137	142		return
138	143		}
139		-	done := false
140		-
141		-	for !done {
142		-	select {
143		-	case err := <-errCh:
144		-	if err != nil {
145		-	fmt.Println(err)
146		-	return
147		-	}
148		-	case <-time.After(1 * time.Second):
149		-	// Check if the progress bar is done
150		-	if m.Done {
151		-	done = true
152		-	}
	144	+	// Wait for the deployment goroutine to complete
	145	+	wg.Wait()
	146	+
	147	+	// Check for errors from the deployment goroutine
	148	+	select {
	149	+	case err := <-errCh:
	150	+	if err != nil {
	151	+	fmt.Println(err)
	152	+	return
153	153		}
	154	+	default:
154	155		}
155	156
156		-	// Close the reader
157		-	err = progressReader.Close()
158		-	if err != nil {
159		-	fmt.Println("Error closing reader:", err)
160		-	return
	157	+	// Check if the progress bar is done
	158	+	if m.Done {
	159	+	// Close the reader
	160	+	err = progressReader.Close()
	161	+	if err != nil {
	162	+	fmt.Println("Error closing reader:", err)
	163	+	return
	164	+	}
	165	+	} else {
	166	+	fmt.Println("Deployment completed, but progress bar is not done.")
161	167		}
162		-
163		-	m.Done = true
164		-
165	168		},
166	169		}
167	170


@@ -7,6 +7,7 @@ import (
7		"net/http"	7		"net/http"
8		"os"	8		"os"
9		"strings"	9		"strings"
			10	+	"sync"
10		"time"	11		"time"
11			12
12		"github.com/bnema/gordon/internal/cli"	13		"github.com/bnema/gordon/internal/cli"

@@ -85,7 +86,11 @@ func NewDeployCommand(a cli.App) cobra.Command {
85		},	86		},
86		}	87		}
87			88
			89	+	var wg sync.WaitGroup
			90	+	wg.Add(1)
			91	+
88		go func() {	92		go func() {
			93	+	defer wg.Done()
89		resp, err := handler.SendHTTPRequest(a, &reqPayload, "POST", "/push")	94		resp, err := handler.SendHTTPRequest(a, &reqPayload, "POST", "/push")
90		if err != nil {	95		if err != nil {
91		errCh <- fmt.Errorf("error sending HTTP request: %w", err)	96		errCh <- fmt.Errorf("error sending HTTP request: %w", err)

@@ -136,32 +141,30 @@ func NewDeployCommand(a cli.App) cobra.Command {
136		errCh <- fmt.Errorf("error running progress bar TUI: %w", err)	141		errCh <- fmt.Errorf("error running progress bar TUI: %w", err)
137		return	142		return
138		}	143		}
139	-	done := false	144	+	// Wait for the deployment goroutine to complete
140	-		145	+	wg.Wait()
141	-	for !done {	146	+
142	-	select {	147	+	// Check for errors from the deployment goroutine
143	-	case err := <-errCh:	148	+	select {
144	-	if err != nil {	149	+	case err := <-errCh:
145	-	fmt.Println(err)	150	+	if err != nil {
146	-	return	151	+	fmt.Println(err)
147	-	}	152	+	return
148	-	case <-time.After(1 * time.Second):
149	-	// Check if the progress bar is done
150	-	if m.Done {
151	-	done = true
152	-	}
153		}	153		}
			154	+	default:
154		}	155		}
155			156
156	-	// Close the reader	157	+	// Check if the progress bar is done
157	-	err = progressReader.Close()	158	+	if m.Done {
158	-	if err != nil {	159	+	// Close the reader
159	-	fmt.Println("Error closing reader:", err)	160	+	err = progressReader.Close()
160	-	return	161	+	if err != nil {
			162	+	fmt.Println("Error closing reader:", err)
			163	+	return
			164	+	}
			165	+	} else {
			166	+	fmt.Println("Deployment completed, but progress bar is not done.")
161		}	167		}
162	-
163	-	m.Done = true
164	-
165		},	168		},
166		}	169		}
167			170

M internal/httpserve/handler/uploadcontainerimage.go +22 -8

@@ -1,10 +1,10 @@
1	1		package handler
2	2
3	3		import (
4		-	"bytes"
5	4		"fmt"
6	5		"io"
7	6		"net/http"
	7	+	"os"
8	8
9	9		"github.com/bnema/gordon/internal/server"
10	10		"github.com/bnema/gordon/internal/templating/render"

@@ -13,7 +13,8 @@ import (
13	13		"github.com/labstack/echo/v4"
14	14		)
15	15
16		-	const MaxUploadSize = 1024 * 1024 * 1024 // 1GB
	16	+	// 10G
	17	+	const MaxUploadSize = 10 * 1024 * 1024 * 1024 // 10GB
17	18
18	19		// UploadImageHandler handles the /upload-image route to show the form
19	20		func UploadImageGETHandler(c echo.Context, a *server.App) error {

@@ -46,19 +47,32 @@ func UploadImagePOSTHandler(c echo.Context, a *server.App) error {
46	47		if err != nil {
47	48		return sendError(c, err)
48	49		}
	50	+	defer file.Close()
49	51
50		-	// Get the file size
51		-	buf := new(bytes.Buffer)
52		-	_, err = io.Copy(buf, file)
	52	+	// Get the filename
	53	+	filename := header.Filename
	54	+
	55	+	// Create a temporary file to store the uploaded chunks
	56	+	tempFile, err := os.CreateTemp("", "upload-*")
53	57		if err != nil {
54	58		return sendError(c, err)
55	59		}
	60	+	defer tempFile.Close()
56	61
57		-	// Get the filename
58		-	filename := header.Filename
	62	+	// Copy the file chunks to the temporary file
	63	+	_, err = io.Copy(tempFile, file)
	64	+	if err != nil {
	65	+	return sendError(c, err)
	66	+	}
	67	+
	68	+	// Reset the temporary file pointer
	69	+	_, err = tempFile.Seek(0, io.SeekStart)
	70	+	if err != nil {
	71	+	return sendError(c, err)
	72	+	}
59	73
60	74		// Save the image to the storage directory
61		-	saveInPath, err := store.SaveImageToStorage(&a.Config, filename, buf)
	75	+	saveInPath, err := store.SaveImageToStorage(&a.Config, filename, tempFile)
62	76		if err != nil {
63	77		return sendError(c, err)
64	78		}

@@ -1,10 +1,10 @@
1		package handler	1		package handler
2			2
3		import (	3		import (
4	-	"bytes"
5		"fmt"	4		"fmt"
6		"io"	5		"io"
7		"net/http"	6		"net/http"
			7	+	"os"
8			8
9		"github.com/bnema/gordon/internal/server"	9		"github.com/bnema/gordon/internal/server"
10		"github.com/bnema/gordon/internal/templating/render"	10		"github.com/bnema/gordon/internal/templating/render"

@@ -13,7 +13,8 @@ import (
13		"github.com/labstack/echo/v4"	13		"github.com/labstack/echo/v4"
14		)	14		)
15			15
16	-	const MaxUploadSize = 1024 * 1024 * 1024 // 1GB	16	+	// 10G
			17	+	const MaxUploadSize = 10 * 1024 * 1024 * 1024 // 10GB
17			18
18		// UploadImageHandler handles the /upload-image route to show the form	19		// UploadImageHandler handles the /upload-image route to show the form
19		func UploadImageGETHandler(c echo.Context, a *server.App) error {	20		func UploadImageGETHandler(c echo.Context, a *server.App) error {

@@ -46,19 +47,32 @@ func UploadImagePOSTHandler(c echo.Context, a *server.App) error {
46		if err != nil {	47		if err != nil {
47		return sendError(c, err)	48		return sendError(c, err)
48		}	49		}
			50	+	defer file.Close()
49			51
50	-	// Get the file size	52	+	// Get the filename
51	-	buf := new(bytes.Buffer)	53	+	filename := header.Filename
52	-	_, err = io.Copy(buf, file)	54	+
			55	+	// Create a temporary file to store the uploaded chunks
			56	+	tempFile, err := os.CreateTemp("", "upload-*")
53		if err != nil {	57		if err != nil {
54		return sendError(c, err)	58		return sendError(c, err)
55		}	59		}
			60	+	defer tempFile.Close()
56			61
57	-	// Get the filename	62	+	// Copy the file chunks to the temporary file
58	-	filename := header.Filename	63	+	_, err = io.Copy(tempFile, file)
			64	+	if err != nil {
			65	+	return sendError(c, err)
			66	+	}
			67	+
			68	+	// Reset the temporary file pointer
			69	+	_, err = tempFile.Seek(0, io.SeekStart)
			70	+	if err != nil {
			71	+	return sendError(c, err)
			72	+	}
59			73
60		// Save the image to the storage directory	74		// Save the image to the storage directory
61	-	saveInPath, err := store.SaveImageToStorage(&a.Config, filename, buf)	75	+	saveInPath, err := store.SaveImageToStorage(&a.Config, filename, tempFile)
62		if err != nil {	76		if err != nil {
63		return sendError(c, err)	77		return sendError(c, err)
64		}	78		}