Add backend features to upload and save a container image into a storage dir and return imageId

commit: b08c43b2cae992191a7baea5fafca05493673673
author: bnema · 3y ago · 2023-09-23 08:15 UTC
parent: 82cbfba
signature: unsigned

7 files · +165 -10

M cmd/srv/main.go +2 -0


@@ -23,6 +23,7 @@ func cleanup(a app.App, memDb sql.DB) {
23	23
24	24		func main() {
25	25		a := app.NewApp()
	26	+	fmt.Println(a)
26	27
27	28		// Initialize database
28	29		memDb, err := app.InitializeDB(a)

@@ -37,6 +38,7 @@ func main() {
37	38		if err != nil {
38	39		log.Printf("Error: %s", err)
39	40		}
	41	+
40	42		// Setup a channel to capture termination signals
41	43		sigs := make(chan os.Signal, 1)
42	44		signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)


@@ -23,6 +23,7 @@ func cleanup(a app.App, memDb sql.DB) {
23		23
24	func main() {	24		func main() {
25	a := app.NewApp()	25		a := app.NewApp()
		26	+	fmt.Println(a)
26		27
27	// Initialize database	28		// Initialize database
28	memDb, err := app.InitializeDB(a)	29		memDb, err := app.InitializeDB(a)

@@ -37,6 +38,7 @@ func main() {
37	if err != nil {	38		if err != nil {
38	log.Printf("Error: %s", err)	39		log.Printf("Error: %s", err)
39	}	40		}
		41	+
40	// Setup a channel to capture termination signals	42		// Setup a channel to capture termination signals
41	sigs := make(chan os.Signal, 1)	43		sigs := make(chan os.Signal, 1)
42	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)	44		signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)

M internal/app/app.go +1 -1


@@ -38,6 +38,7 @@ type AppConfig struct {
38	38
39	39		type GeneralConfig struct {
40	40		RunEnv string `yaml:"runEnv"`
	41	+	StorageDir string `yaml:"storageDir"`
41	42		BuildVersion string
42	43		}
43	44

@@ -98,7 +99,6 @@ func NewApp() *App {
98	99
99	100		// If you have any other initializations like generating OAuth URL, do them here
100	101		OauthCallbackURL = GenerateOauthCallbackURL(config)
101		-
102	102		return a
103	103		}
104	104

 type GeneralConfig struct {
 	RunEnv       string `yaml:"runEnv"`
+	StorageDir   string `yaml:"storageDir"`
 	BuildVersion string
 }
 	// If you have any other initializations like generating OAuth URL, do them here
 	OauthCallbackURL = GenerateOauthCallbackURL(config)
 	return a
 }

A internal/gotemplate/models/html/fragments/uploadimage.gohtml +1 -0

@@ -0,0 +1,1 @@
	1	+	<!-- Form to upload a container image (.tar.gz format) -->

@@ -0,0 +1,1 @@
			1	+	<!-- Form to upload a container image (.tar.gz format) -->

R internal/httpserve/handler/docker.go → internal/httpserve/handler/containers.go +8 -9 renamed, 90% similar


@@ -30,13 +30,13 @@ type HumanReadableContainer struct {
30	30		Name string
31	31		Ports []string
32	32		ShortID string
33		-	CreatedStr string // Human-readable Created time
34		-	SizeStr string // Human-readable Size
35		-	UpSince string // Human-readable time elapsed since the container was started
36		-	StateColor string // Color of the state badge
	33	+	CreatedStr string
	34	+	SizeStr string
	35	+	UpSince string
	36	+	StateColor string
37	37		}
38	38
39		-	// ImageManagerHandler handles the /image-manager route
	39	+	// ImageManagerHandler handles the /image-manager route (HTMX route)
40	40		func ImageManagerHandler(c echo.Context, a *app.App) error {
41	41		images, err := docker.ListContainerImages()
42	42		if err != nil {

@@ -113,7 +113,7 @@ func ContainerManagerHandler(c echo.Context, a *app.App) error {
113	113		if err != nil {
114	114		return sendError(c, err)
115	115		}
116		-	// We need Name / Command / Created / Status / Ports / State
	116	+
117	117		var humanReadableContainers []HumanReadableContainer
118	118
119	119		for _, container := range containers {

@@ -123,7 +123,7 @@ func ContainerManagerHandler(c echo.Context, a *app.App) error {
123	123		createdStr := humanize.TimeAgo(createdTime)
124	124		sizeStr := humanize.BytesToReadableSize(container.SizeRw)
125	125		stateColor := "green"
126		-	// If the container is exited, set the state color to red
	126	+
127	127		if container.State == "exited" {
128	128		stateColor = "red"
129	129		}

@@ -169,14 +169,13 @@ func ContainerManagerDeleteHandler(c echo.Context, a *app.App) error {
169	169		if err != nil {
170	170		return sendError(c, err)
171	171		}
172		-	// Since it is HTMX we return a html div with the message "Removed"
173	172		return c.HTML(http.StatusOK, `Success`)
174	173		}
175	174
176	175		// ContainerManagerStopHandler handles the /container-manager/stop route
177	176		func ContainerManagerStopHandler(c echo.Context, a *app.App) error {
178	177		ID := c.Param("ID")
179		-	// Stop the container gracefully with a timeout of 10 seconds
	178	+	// Stop the container gracefully with a timeout
180	179		stopped, err := docker.StopContainerGracefully(ID, 3*time.Second)
181	180		if err != nil {
182	181		return sendError(c, err)


@@ -30,13 +30,13 @@ type HumanReadableContainer struct {
30		Name string	30		Name string
31		Ports []string	31		Ports []string
32		ShortID string	32		ShortID string
33	-	CreatedStr string // Human-readable Created time	33	+	CreatedStr string
34	-	SizeStr string // Human-readable Size	34	+	SizeStr string
35	-	UpSince string // Human-readable time elapsed since the container was started	35	+	UpSince string
36	-	StateColor string // Color of the state badge	36	+	StateColor string
37		}	37		}
38			38
39	-	// ImageManagerHandler handles the /image-manager route	39	+	// ImageManagerHandler handles the /image-manager route (HTMX route)
40		func ImageManagerHandler(c echo.Context, a *app.App) error {	40		func ImageManagerHandler(c echo.Context, a *app.App) error {
41		images, err := docker.ListContainerImages()	41		images, err := docker.ListContainerImages()
42		if err != nil {	42		if err != nil {

@@ -113,7 +113,7 @@ func ContainerManagerHandler(c echo.Context, a *app.App) error {
113		if err != nil {	113		if err != nil {
114		return sendError(c, err)	114		return sendError(c, err)
115		}	115		}
116	-	// We need Name / Command / Created / Status / Ports / State	116	+
117		var humanReadableContainers []HumanReadableContainer	117		var humanReadableContainers []HumanReadableContainer
118			118
119		for _, container := range containers {	119		for _, container := range containers {

@@ -123,7 +123,7 @@ func ContainerManagerHandler(c echo.Context, a *app.App) error {
123		createdStr := humanize.TimeAgo(createdTime)	123		createdStr := humanize.TimeAgo(createdTime)
124		sizeStr := humanize.BytesToReadableSize(container.SizeRw)	124		sizeStr := humanize.BytesToReadableSize(container.SizeRw)
125		stateColor := "green"	125		stateColor := "green"
126	-	// If the container is exited, set the state color to red	126	+
127		if container.State == "exited" {	127		if container.State == "exited" {
128		stateColor = "red"	128		stateColor = "red"
129		}	129		}

@@ -169,14 +169,13 @@ func ContainerManagerDeleteHandler(c echo.Context, a *app.App) error {
169		if err != nil {	169		if err != nil {
170		return sendError(c, err)	170		return sendError(c, err)
171		}	171		}
172	-	// Since it is HTMX we return a html div with the message "Removed"
173		return c.HTML(http.StatusOK, `Success`)	172		return c.HTML(http.StatusOK, `Success`)
174		}	173		}
175			174
176		// ContainerManagerStopHandler handles the /container-manager/stop route	175		// ContainerManagerStopHandler handles the /container-manager/stop route
177		func ContainerManagerStopHandler(c echo.Context, a *app.App) error {	176		func ContainerManagerStopHandler(c echo.Context, a *app.App) error {
178		ID := c.Param("ID")	177		ID := c.Param("ID")
179	-	// Stop the container gracefully with a timeout of 10 seconds	178	+	// Stop the container gracefully with a timeout
180		stopped, err := docker.StopContainerGracefully(ID, 3*time.Second)	179		stopped, err := docker.StopContainerGracefully(ID, 3*time.Second)
181		if err != nil {	180		if err != nil {
182		return sendError(c, err)	181		return sendError(c, err)

A internal/httpserve/handler/uploadimage.go +59 -0

@@ -0,0 +1,59 @@
	1	+	package handler
	2	+
	3	+	import (
	4	+	"io"
	5	+	"net/http"
	6	+	"os"
	7	+	"strings"
	8	+
	9	+	"github.com/bnema/gordon/internal/app"
	10	+	"github.com/labstack/echo/v4"
	11	+	)
	12	+
	13	+	const MaxUploadSize = 1024 * 1024 * 1024 // 1GB
	14	+
	15	+	// UploadImageHandler handles the /upload-image route
	16	+	func UploadImageHandler(c echo.Context, a *app.App) error {
	17	+	// Set upload size limit
	18	+	c.Request().Body = http.MaxBytesReader(c.Response(), c.Request().Body, MaxUploadSize)
	19	+	if err := c.Request().ParseMultipartForm(MaxUploadSize); err != nil {
	20	+	return c.String(http.StatusRequestEntityTooLarge, "File size exceeded")
	21	+	}
	22	+
	23	+	// Get the file from the form
	24	+	file, _, err := c.Request().FormFile("file")
	25	+	if err != nil {
	26	+	return sendError(c, err)
	27	+	}
	28	+	defer file.Close()
	29	+
	30	+	// Check file type
	31	+	buffer := make([]byte, 512)
	32	+	if _, err := file.Read(buffer); err != nil {
	33	+	return sendError(c, err)
	34	+	}
	35	+	contentType := http.DetectContentType(buffer)
	36	+	if !strings.Contains(contentType, "application/gzip") {
	37	+	return c.String(http.StatusUnsupportedMediaType, "Invalid file type")
	38	+	}
	39	+	if _, err := file.Seek(0, io.SeekStart); err != nil {
	40	+	return sendError(c, err)
	41	+	}
	42	+
	43	+	// Save to temporary file
	44	+	tmpFile, err := os.CreateTemp("/tmp", "upload-*.tar.gz")
	45	+	if err != nil {
	46	+	return sendError(c, err)
	47	+	}
	48	+	defer os.Remove(tmpFile.Name())
	49	+	if _, err := io.Copy(tmpFile, file); err != nil {
	50	+	return sendError(c, err)
	51	+	}
	52	+
	53	+	// Close and flush the temporary file to make sure all content is written
	54	+	if err := tmpFile.Close(); err != nil {
	55	+	return sendError(c, err)
	56	+	}
	57	+
	58	+	return c.HTML(http.StatusOK, "Success")
	59	+	}

@@ -0,0 +1,59 @@
	1	+	package handler
	2	+
	3	+	import (
	4	+	"io"
	5	+	"net/http"
	6	+	"os"
	7	+	"strings"
	8	+
	9	+	"github.com/bnema/gordon/internal/app"
	10	+	"github.com/labstack/echo/v4"
	11	+	)
	12	+
	13	+	const MaxUploadSize = 1024 * 1024 * 1024 // 1GB
	14	+
	15	+	// UploadImageHandler handles the /upload-image route
	16	+	func UploadImageHandler(c echo.Context, a *app.App) error {
	17	+	// Set upload size limit
	18	+	c.Request().Body = http.MaxBytesReader(c.Response(), c.Request().Body, MaxUploadSize)
	19	+	if err := c.Request().ParseMultipartForm(MaxUploadSize); err != nil {
	20	+	return c.String(http.StatusRequestEntityTooLarge, "File size exceeded")
	21	+	}
	22	+
	23	+	// Get the file from the form
	24	+	file, _, err := c.Request().FormFile("file")
	25	+	if err != nil {
	26	+	return sendError(c, err)
	27	+	}
	28	+	defer file.Close()
	29	+
	30	+	// Check file type
	31	+	buffer := make([]byte, 512)
	32	+	if _, err := file.Read(buffer); err != nil {
	33	+	return sendError(c, err)
	34	+	}
	35	+	contentType := http.DetectContentType(buffer)
	36	+	if !strings.Contains(contentType, "application/gzip") {
	37	+	return c.String(http.StatusUnsupportedMediaType, "Invalid file type")
	38	+	}
	39	+	if _, err := file.Seek(0, io.SeekStart); err != nil {
	40	+	return sendError(c, err)
	41	+	}
	42	+
	43	+	// Save to temporary file
	44	+	tmpFile, err := os.CreateTemp("/tmp", "upload-*.tar.gz")
	45	+	if err != nil {
	46	+	return sendError(c, err)
	47	+	}
	48	+	defer os.Remove(tmpFile.Name())
	49	+	if _, err := io.Copy(tmpFile, file); err != nil {
	50	+	return sendError(c, err)
	51	+	}
	52	+
	53	+	// Close and flush the temporary file to make sure all content is written
	54	+	if err := tmpFile.Close(); err != nil {
	55	+	return sendError(c, err)
	56	+	}
	57	+
	58	+	return c.HTML(http.StatusOK, "Success")
	59	+	}

M pkg/utils/docker/images.go +53 -0


@@ -2,6 +2,11 @@ package docker
2	2
3	3		import (
4	4		"context"
	5	+	"encoding/json"
	6	+	"fmt"
	7	+	"io"
	8	+	"os"
	9	+	"strings"
5	10
6	11		"github.com/docker/docker/api/types"
7	12		)

@@ -29,3 +34,51 @@ func DeleteContainerImage(imageID string) error {
29	34
30	35		return nil
31	36		}
	37	+
	38	+	func LoadImage(imagePath string) (string, error) {
	39	+	// Check if the Docker client has been initialized
	40	+	CheckIfInitialized()
	41	+
	42	+	// Open the image file
	43	+	imageFile, err := os.Open(imagePath)
	44	+	if err != nil {
	45	+	return "", err
	46	+	}
	47	+	defer imageFile.Close()
	48	+
	49	+	// Load the image using the Docker client
	50	+	resp, err := dockerCli.ImageLoad(context.Background(), imageFile, false)
	51	+	if err != nil {
	52	+	return "", err
	53	+	}
	54	+
	55	+	defer resp.Body.Close()
	56	+
	57	+	// Read the JSON response to get the image ID
	58	+	decoder := json.NewDecoder(resp.Body)
	59	+
	60	+	var imageID string
	61	+	for {
	62	+	var jm map[string]interface{}
	63	+	if err := decoder.Decode(&jm); err == io.EOF {
	64	+	break
	65	+	} else if err != nil {
	66	+	return "", fmt.Errorf("could not decode response: %v", err)
	67	+	}
	68	+
	69	+	if id, exists := jm["stream"]; exists {
	70	+	if strID, ok := id.(string); ok {
	71	+	// Extract image ID from the stream.
	72	+	// This is a naive example; you may need to adjust this part
	73	+	// to correctly parse your specific Docker version's output.
	74	+	imageID = strings.TrimSpace(strID)
	75	+	}
	76	+	}
	77	+	}
	78	+
	79	+	if imageID == "" {
	80	+	return "", fmt.Errorf("could not find image ID in response")
	81	+	}
	82	+
	83	+	return imageID, nil
	84	+	}


@@ -2,6 +2,11 @@ package docker
2		2
3	import (	3		import (
4	"context"	4		"context"
		5	+	"encoding/json"
		6	+	"fmt"
		7	+	"io"
		8	+	"os"
		9	+	"strings"
5		10
6	"github.com/docker/docker/api/types"	11		"github.com/docker/docker/api/types"
7	)	12		)

@@ -29,3 +34,51 @@ func DeleteContainerImage(imageID string) error {
29		34
30	return nil	35		return nil
31	}	36		}
		37	+
		38	+	func LoadImage(imagePath string) (string, error) {
		39	+	// Check if the Docker client has been initialized
		40	+	CheckIfInitialized()
		41	+
		42	+	// Open the image file
		43	+	imageFile, err := os.Open(imagePath)
		44	+	if err != nil {
		45	+	return "", err
		46	+	}
		47	+	defer imageFile.Close()
		48	+
		49	+	// Load the image using the Docker client
		50	+	resp, err := dockerCli.ImageLoad(context.Background(), imageFile, false)
		51	+	if err != nil {
		52	+	return "", err
		53	+	}
		54	+
		55	+	defer resp.Body.Close()
		56	+
		57	+	// Read the JSON response to get the image ID
		58	+	decoder := json.NewDecoder(resp.Body)
		59	+
		60	+	var imageID string
		61	+	for {
		62	+	var jm map[string]interface{}
		63	+	if err := decoder.Decode(&jm); err == io.EOF {
		64	+	break
		65	+	} else if err != nil {
		66	+	return "", fmt.Errorf("could not decode response: %v", err)
		67	+	}
		68	+
		69	+	if id, exists := jm["stream"]; exists {
		70	+	if strID, ok := id.(string); ok {
		71	+	// Extract image ID from the stream.
		72	+	// This is a naive example; you may need to adjust this part
		73	+	// to correctly parse your specific Docker version's output.
		74	+	imageID = strings.TrimSpace(strID)
		75	+	}
		76	+	}
		77	+	}
		78	+
		79	+	if imageID == "" {
		80	+	return "", fmt.Errorf("could not find image ID in response")
		81	+	}
		82	+
		83	+	return imageID, nil
		84	+	}

A pkg/utils/store/storeimages.go +41 -0

@@ -0,0 +1,41 @@
	1	+	package store
	2	+
	3	+	import (
	4	+	"fmt"
	5	+	"os"
	6	+	"path/filepath"
	7	+
	8	+	"github.com/bnema/gordon/internal/app"
	9	+	"github.com/bnema/gordon/pkg/utils/docker"
	10	+	"github.com/docker/docker/api/types"
	11	+	)
	12	+
	13	+	type StorageConfig struct {
	14	+	StorageDir string
	15	+	ImageID string
	16	+	Images []types.ImageSummary
	17	+	}
	18	+
	19	+	func NewStorageConfig(config app.AppConfig) StorageConfig {
	20	+	return &StorageConfig{
	21	+	StorageDir: config.General.StorageDir,
	22	+	}
	23	+	}
	24	+
	25	+	func (sc *StorageConfig) SaveImageToStorage(imagePath string) (string, error) {
	26	+
	27	+	imageId, err := docker.LoadImage(imagePath)
	28	+	if err != nil {
	29	+	return "", fmt.Errorf("failed to load image: %v", err)
	30	+	}
	31	+
	32	+	// Generate the new path for the image in the storage directory
	33	+	newPath := filepath.Join(sc.StorageDir, imageId+".tar")
	34	+
	35	+	err = os.Rename(imagePath, newPath)
	36	+	if err != nil {
	37	+	return "", fmt.Errorf("failed to move image to storage directory: %v", err)
	38	+	}
	39	+
	40	+	return imageId, nil
	41	+	}

@@ -0,0 +1,41 @@
	1	+	package store
	2	+
	3	+	import (
	4	+	"fmt"
	5	+	"os"
	6	+	"path/filepath"
	7	+
	8	+	"github.com/bnema/gordon/internal/app"
	9	+	"github.com/bnema/gordon/pkg/utils/docker"
	10	+	"github.com/docker/docker/api/types"
	11	+	)
	12	+
	13	+	type StorageConfig struct {
	14	+	StorageDir string
	15	+	ImageID string
	16	+	Images []types.ImageSummary
	17	+	}
	18	+
	19	+	func NewStorageConfig(config app.AppConfig) StorageConfig {
	20	+	return &StorageConfig{
	21	+	StorageDir: config.General.StorageDir,
	22	+	}
	23	+	}
	24	+
	25	+	func (sc *StorageConfig) SaveImageToStorage(imagePath string) (string, error) {
	26	+
	27	+	imageId, err := docker.LoadImage(imagePath)
	28	+	if err != nil {
	29	+	return "", fmt.Errorf("failed to load image: %v", err)
	30	+	}
	31	+
	32	+	// Generate the new path for the image in the storage directory
	33	+	newPath := filepath.Join(sc.StorageDir, imageId+".tar")
	34	+
	35	+	err = os.Rename(imagePath, newPath)
	36	+	if err != nil {
	37	+	return "", fmt.Errorf("failed to move image to storage directory: %v", err)
	38	+	}
	39	+
	40	+	return imageId, nil
	41	+	}