Backend: Refactor image import process and improve error handling

- Update ImportImageToEngine to return image ID
- Modify PostDeploy to use new ImportImageToEngine return value
- Remove redundant GetImageIDByName call in PostDeploy
- Enhance GetImageIDByName with more robust image name matching
- Add logging throughout image import process
- Improve error handling and cleanup in case of failures
- Update FromPayloadStructToCmdParams to accept imageID parameter

commit: 2c1a8526c239c6abc3f306306dff9e03bb14742c
author: bamen · 2y ago · 2024-08-28 12:12 UTC
parent: fe7fdda
signature: unsigned

6 files · +135 -39

M internal/httpserve/handler/clientendpoins.go +10 -10


@@ -90,32 +90,32 @@ func PostDeploy(c echo.Context, a *server.App) error {
90	90		fmt.Printf("Image saved successfully to: %s\n", imagePath)
91	91
92	92		// Import the tar in docker
93		-	err = docker.ImportImageToEngine(imagePath)
	93	+	imageID, err := docker.ImportImageToEngine(imagePath)
94	94		if err != nil {
95	95		store.RemoveFromStorage(imagePath) // Clean up the saved image if import fails
96	96		return sendJsonError(c, fmt.Errorf("failed to import image: %v", err))
97	97		}
98	98
	99	+	if imageID == "" {
	100	+	return sendJsonError(c, fmt.Errorf("imported image ID is empty"))
	101	+	}
	102	+
	103	+	// Debug
	104	+	fmt.Printf("Image ID: %s\n", imageID)
	105	+
99	106		// Remove the image from the storage
100	107		err = store.RemoveFromStorage(imagePath)
101	108		if err != nil {
102	109		return sendJsonError(c, fmt.Errorf("failed to remove temporary image file: %v", err))
103	110		}
104	111
105		-	// append localhost to the image name since it's a manually imported image and not from a registry
106		-	payload.ImageName = "localhost/" + payload.ImageName
107		-
108		-	// Get the image ID
109		-	imageID, err := docker.GetImageIDByName(payload.ImageName)
110		-	if err != nil {
111		-	return sendJsonError(c, fmt.Errorf("failed to get image ID: %v", err))
112		-	}
	112	+	fmt.Printf("Image imported successfully: %s\n", payload.ImageName)
113	113
114	114		// Update the payload with the image ID
115	115		payload.ImageID = imageID
116	116
117	117		// Create the container using cmdparams.FromPayloadStructToCmdParams
118		-	params, err := cmdparams.FromPayloadStructToCmdParams(payload, a)
	118	+	params, err := cmdparams.FromPayloadStructToCmdParams(payload, a, imageID)
119	119		if err != nil {
120	120		return sendJsonError(c, fmt.Errorf("failed to create command parameters: %v", err))
121	121		}

 	fmt.Printf("Image saved successfully to: %s\n", imagePath)
 	// Import the tar in docker
-	err = docker.ImportImageToEngine(imagePath)
+	imageID, err := docker.ImportImageToEngine(imagePath)
 	if err != nil {
 		store.RemoveFromStorage(imagePath) // Clean up the saved image if import fails
 		return sendJsonError(c, fmt.Errorf("failed to import image: %v", err))
 	}
+	if imageID == "" {
+		return sendJsonError(c, fmt.Errorf("imported image ID is empty"))
+	}
+	// Debug
+	fmt.Printf("Image ID: %s\n", imageID)
 	// Remove the image from the storage
 	err = store.RemoveFromStorage(imagePath)
 	if err != nil {
 		return sendJsonError(c, fmt.Errorf("failed to remove temporary image file: %v", err))
 	}
-	// append localhost to the image name since it's a manually imported image and not from a registry
+	fmt.Printf("Image imported successfully: %s\n", payload.ImageName)
-	payload.ImageName = "localhost/" + payload.ImageName
-	// Get the image ID
-	imageID, err := docker.GetImageIDByName(payload.ImageName)
-	if err != nil {
-		return sendJsonError(c, fmt.Errorf("failed to get image ID: %v", err))
-	}
 	// Update the payload with the image ID
 	payload.ImageID = imageID
 	// Create the container using cmdparams.FromPayloadStructToCmdParams
-	params, err := cmdparams.FromPayloadStructToCmdParams(payload, a)
+	params, err := cmdparams.FromPayloadStructToCmdParams(payload, a, imageID)
 	if err != nil {
 		return sendJsonError(c, fmt.Errorf("failed to create command parameters: %v", err))
 	}

M internal/httpserve/handler/containers.go +0 -1


@@ -132,7 +132,6 @@ func ImageManagerComponent(c echo.Context, a *server.App) error {
132	132
133	133		// ImageManagerDelete handles the /image-manager/delete route
134	134		func ImageManagerDelete(c echo.Context, a *server.App) error {
135		-	//
136	135		ShortID := c.Param("ID")
137	136
138	137		imageID, exists := safelyInteractWithIDMap(Fetch, ShortID)


@@ -132,7 +132,6 @@ func ImageManagerComponent(c echo.Context, a *server.App) error {
132			132
133		// ImageManagerDelete handles the /image-manager/delete route	133	// ImageManagerDelete handles the /image-manager/delete route
134		func ImageManagerDelete(c echo.Context, a *server.App) error {	134	func ImageManagerDelete(c echo.Context, a *server.App) error {
135	-	//
136		ShortID := c.Param("ID")	135	ShortID := c.Param("ID")
137			136
138		imageID, exists := safelyInteractWithIDMap(Fetch, ShortID)	137	imageID, exists := safelyInteractWithIDMap(Fetch, ShortID)

M internal/httpserve/handler/uploadcontainerimage.go +1 -1


@@ -85,7 +85,7 @@ func UploadImagePOSTHandler(c echo.Context, a *server.App) error {
85	85		}
86	86
87	87		// Import the image into Docker
88		-	err = docker.ImportImageToEngine(saveInPath)
	88	+	_, err = docker.ImportImageToEngine(saveInPath)
89	89		if err != nil {
90	90		return fmt.Errorf("failed to import image to Docker: %v", err)
91	91		}


@@ -85,7 +85,7 @@ func UploadImagePOSTHandler(c echo.Context, a *server.App) error {
85		}	85		}
86			86
87		// Import the image into Docker	87		// Import the image into Docker
88	-	err = docker.ImportImageToEngine(saveInPath)	88	+	_, err = docker.ImportImageToEngine(saveInPath)
89		if err != nil {	89		if err != nil {
90		return fmt.Errorf("failed to import image to Docker: %v", err)	90		return fmt.Errorf("failed to import image to Docker: %v", err)
91		}	91		}

M internal/templating/cmdparams/fromcli.go +1 -6


@@ -11,12 +11,7 @@ import (
11	11		)
12	12
13	13		// FromYAMLStructToCmdParams converts a YAMLContainerParams struct to a ContainerCommandParams struct
14		-	func FromPayloadStructToCmdParams(ppl common.DeployPayload, a server.App) (docker.ContainerCommandParams, error) {
15		-
16		-	imageID, err := docker.GetImageIDByName(ppl.ImageName)
17		-	if err != nil {
18		-	return docker.ContainerCommandParams{}, err
19		-	}
	14	+	func FromPayloadStructToCmdParams(ppl common.DeployPayload, a server.App, imageID string) (docker.ContainerCommandParams, error) {
20	15
21	16		// volume empty for now
22	17		volumeSlice := ParseVolumeSlice("")

 )
 // FromYAMLStructToCmdParams converts a YAMLContainerParams struct to a ContainerCommandParams struct
-func FromPayloadStructToCmdParams(ppl *common.DeployPayload, a *server.App) (docker.ContainerCommandParams, error) {
+func FromPayloadStructToCmdParams(ppl *common.DeployPayload, a *server.App, imageID string) (docker.ContainerCommandParams, error) {
-	imageID, err := docker.GetImageIDByName(ppl.ImageName)
-	if err != nil {
-		return docker.ContainerCommandParams{}, err
-	}
 	// volume empty for now
 	volumeSlice := ParseVolumeSlice("")

M pkg/docker/client.go +2 -2


@@ -59,11 +59,11 @@ func CheckIfInitialized() error {
59	59		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
60	60		defer cancel()
61	61
62		-	containers, err := dockerCli.ContainerList(ctx, container.ListOptions{})
	62	+	_, err := dockerCli.ContainerList(ctx, container.ListOptions{})
63	63		if err != nil {
64	64		return fmt.Errorf("cannot connect to Docker daemon: %s", err)
65	65		}
66		-	fmt.Printf("Found %d containers\n", len(containers))
	66	+
67	67		return nil
68	68		}
69	69


@@ -59,11 +59,11 @@ func CheckIfInitialized() error {
59		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)	59		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
60		defer cancel()	60		defer cancel()
61			61
62	-	containers, err := dockerCli.ContainerList(ctx, container.ListOptions{})	62	+	_, err := dockerCli.ContainerList(ctx, container.ListOptions{})
63		if err != nil {	63		if err != nil {
64		return fmt.Errorf("cannot connect to Docker daemon: %s", err)	64		return fmt.Errorf("cannot connect to Docker daemon: %s", err)
65		}	65		}
66	-	fmt.Printf("Found %d containers\n", len(containers))	66	+
67		return nil	67		return nil
68		}	68		}
69			69

M pkg/docker/images.go +121 -19

@@ -1,10 +1,15 @@
1	1		package docker
2	2
3	3		import (
	4	+	"bufio"
4	5		"context"
	6	+	"encoding/json"
5	7		"fmt"
6	8		"io"
	9	+	"log"
7	10		"os"
	11	+	"regexp"
	12	+	"strings"
8	13
9	14		"github.com/docker/docker/api/types"
10	15		"github.com/docker/docker/api/types/image"

@@ -45,46 +50,143 @@ func CheckIfImageExists(imageID string) (bool, error) {
45	50		}
46	51
47	52		func GetImageIDByName(imageName string) (string, error) {
48		-
49	53		images, err := dockerCli.ImageList(context.Background(), image.ListOptions{})
50	54		if err != nil {
51	55		return "", fmt.Errorf("failed to list images: %w", err)
52	56		}
53	57
54		-	// Search for the image we just loaded
55		-	var imageID string
56		-	for _, image := range images {
57		-	for _, tag := range image.RepoTags {
58		-	if tag == imageName {
59		-	imageID = image.ID
	58	+	normalizedName, tag := normalizeImageName(imageName)
	59	+	fmt.Printf("Searching for image: %s:%s\n", normalizedName, tag)
	60	+
	61	+	tagPattern := createTagPattern(tag)
	62	+
	63	+	for _, img := range images {
	64	+	for _, repoTag := range img.RepoTags {
	65	+	normalizedRepoTag, repoTagValue := normalizeImageName(repoTag)
	66	+	fmt.Printf("Comparing with: %s:%s\n", normalizedRepoTag, repoTagValue)
	67	+
	68	+	// Check for exact match (including with docker.io/ prefix)
	69	+	if repoTag == imageName \|\| repoTag == "docker.io/"+imageName {
	70	+	return img.ID, nil
	71	+	}
	72	+
	73	+	// Check for localhost match
	74	+	if repoTag == "localhost/"+imageName {
	75	+	return img.ID, nil
	76	+	}
	77	+
	78	+	// Check for match with normalized name and tag
	79	+	if (normalizedRepoTag == normalizedName \|\|
	80	+	normalizedRepoTag == "docker.io/"+normalizedName) &&
	81	+	tagPattern.MatchString(repoTagValue) {
	82	+	return img.ID, nil
60	83		}
61	84		}
62	85		}
63	86
64		-	if imageID == "" {
65		-	return "", fmt.Errorf("image not found")
	87	+	return "", fmt.Errorf("image not found: %s", imageName)
	88	+	}
	89	+
	90	+	func normalizeImageName(name string) (string, string) {
	91	+	// Split off the tag if present
	92	+	parts := strings.SplitN(name, ":", 2)
	93	+	normalizedName := parts[0]
	94	+	tag := "latest"
	95	+	if len(parts) > 1 {
	96	+	tag = parts[1]
	97	+	}
	98	+
	99	+	// Remove any leading "docker.io/" if present
	100	+	normalizedName = strings.TrimPrefix(normalizedName, "docker.io/")
	101	+
	102	+	// Handle repository names with multiple segments
	103	+	segments := strings.Split(normalizedName, "/")
	104	+	if len(segments) > 1 {
	105	+	// If there are multiple segments, join all but the last
	106	+	normalizedName = strings.Join(segments[:len(segments)-1], "/") + "/" + segments[len(segments)-1]
66	107		}
67	108
68		-	return imageID, nil
	109	+	return normalizedName, tag
	110	+	}
	111	+
	112	+	func createTagPattern(tag string) *regexp.Regexp {
	113	+	if tag == "" \|\| tag == "latest" {
	114	+	// Match any tag if the requested tag is empty or "latest"
	115	+	return regexp.MustCompile(`.*`)
	116	+	}
	117	+
	118	+	// Escape special regex characters in the tag
	119	+	escapedTag := regexp.QuoteMeta(tag)
	120	+
	121	+	// Create a pattern that matches the tag exactly or as a prefix
	122	+	// This allows matching "1.0" with "1.0.1", "1.0-alpine", etc.
	123	+	return regexp.MustCompile(`^` + escapedTag + `($\|[\.-])`)
69	124		}
70	125
71		-	// ImportImageToEngine imports an image to the Docker engine
72		-	func ImportImageToEngine(imageFilePath string) error {
73		-	// Open the image file
	126	+	// ImportImageToEngine imports an image to the Docker engine and returns the image ID
	127	+	func ImportImageToEngine(imageFilePath string) (string, error) {
	128	+	log.Printf("Starting to import image from file: %s", imageFilePath)
	129	+
74	130		imageFile, err := os.Open(imageFilePath)
75	131		if err != nil {
76		-	return fmt.Errorf("failed to open image file: %w", err)
	132	+	log.Printf("Failed to open image file: %v", err)
	133	+	return "", fmt.Errorf("failed to open image file: %w", err)
77	134		}
78	135		defer imageFile.Close()
79	136
80		-	// Import the image using the Docker client
81		-	importedImage, err := dockerCli.ImageLoad(context.Background(), imageFile, false)
	137	+	log.Printf("Image file opened successfully")
	138	+
	139	+	resp, err := dockerCli.ImageLoad(context.Background(), imageFile, true)
82	140		if err != nil {
83		-	return fmt.Errorf("failed to import image: %w", err)
	141	+	log.Printf("Failed to load image: %v", err)
	142	+	return "", fmt.Errorf("failed to import image: %w", err)
84	143		}
85		-	defer importedImage.Body.Close()
	144	+	defer resp.Body.Close()
86	145
87		-	return nil
	146	+	log.Printf("Image load initiated, parsing response")
	147	+
	148	+	var imageName string
	149	+	scanner := bufio.NewScanner(resp.Body)
	150	+	for scanner.Scan() {
	151	+	line := scanner.Text()
	152	+	log.Printf("Response line: %s", line)
	153	+
	154	+	var message struct {
	155	+	Stream string `json:"stream"`
	156	+	}
	157	+	if err := json.Unmarshal([]byte(line), &message); err != nil {
	158	+	log.Printf("Failed to unmarshal JSON: %v", err)
	159	+	continue // Skip lines that aren't JSON
	160	+	}
	161	+
	162	+	log.Printf("Parsed message stream: %s", message.Stream)
	163	+
	164	+	if strings.HasPrefix(message.Stream, "Loaded image: ") {
	165	+	imageName = strings.TrimSpace(strings.TrimPrefix(message.Stream, "Loaded image: "))
	166	+	log.Printf("Found image name: %s", imageName)
	167	+	break
	168	+	}
	169	+	}
	170	+
	171	+	if err := scanner.Err(); err != nil {
	172	+	log.Printf("Error reading response: %v", err)
	173	+	return "", fmt.Errorf("error reading response: %w", err)
	174	+	}
	175	+
	176	+	if imageName == "" {
	177	+	log.Printf("Failed to find image name in import response")
	178	+	return "", fmt.Errorf("failed to find image name in import response")
	179	+	}
	180	+
	181	+	// Now we need to get the image ID from the image name
	182	+	imageID, err := GetImageIDByName(imageName)
	183	+	if err != nil {
	184	+	log.Printf("Failed to get image ID for name %s: %v", imageName, err)
	185	+	return "", fmt.Errorf("failed to get image ID: %w", err)
	186	+	}
	187	+
	188	+	log.Printf("Successfully imported image with ID: %s", imageID)
	189	+	return imageID, nil
88	190		}
89	191
90	192		// ExportImageFromEngine exports an image from the Docker engine and returns it as an io.Reader

@@ -1,10 +1,15 @@
1		package docker	1		package docker
2			2
3		import (	3		import (
			4	+	"bufio"
4		"context"	5		"context"
			6	+	"encoding/json"
5		"fmt"	7		"fmt"
6		"io"	8		"io"
			9	+	"log"
7		"os"	10		"os"
			11	+	"regexp"
			12	+	"strings"
8			13
9		"github.com/docker/docker/api/types"	14		"github.com/docker/docker/api/types"
10		"github.com/docker/docker/api/types/image"	15		"github.com/docker/docker/api/types/image"

@@ -45,46 +50,143 @@ func CheckIfImageExists(imageID string) (bool, error) {
45		}	50		}
46			51
47		func GetImageIDByName(imageName string) (string, error) {	52		func GetImageIDByName(imageName string) (string, error) {
48	-
49		images, err := dockerCli.ImageList(context.Background(), image.ListOptions{})	53		images, err := dockerCli.ImageList(context.Background(), image.ListOptions{})
50		if err != nil {	54		if err != nil {
51		return "", fmt.Errorf("failed to list images: %w", err)	55		return "", fmt.Errorf("failed to list images: %w", err)
52		}	56		}
53			57
54	-	// Search for the image we just loaded	58	+	normalizedName, tag := normalizeImageName(imageName)
55	-	var imageID string	59	+	fmt.Printf("Searching for image: %s:%s\n", normalizedName, tag)
56	-	for _, image := range images {	60	+
57	-	for _, tag := range image.RepoTags {	61	+	tagPattern := createTagPattern(tag)
58	-	if tag == imageName {	62	+
59	-	imageID = image.ID	63	+	for _, img := range images {
			64	+	for _, repoTag := range img.RepoTags {
			65	+	normalizedRepoTag, repoTagValue := normalizeImageName(repoTag)
			66	+	fmt.Printf("Comparing with: %s:%s\n", normalizedRepoTag, repoTagValue)
			67	+
			68	+	// Check for exact match (including with docker.io/ prefix)
			69	+	if repoTag == imageName \|\| repoTag == "docker.io/"+imageName {
			70	+	return img.ID, nil
			71	+	}
			72	+
			73	+	// Check for localhost match
			74	+	if repoTag == "localhost/"+imageName {
			75	+	return img.ID, nil
			76	+	}
			77	+
			78	+	// Check for match with normalized name and tag
			79	+	if (normalizedRepoTag == normalizedName \|\|
			80	+	normalizedRepoTag == "docker.io/"+normalizedName) &&
			81	+	tagPattern.MatchString(repoTagValue) {
			82	+	return img.ID, nil
60		}	83		}
61		}	84		}
62		}	85		}
63			86
64	-	if imageID == "" {	87	+	return "", fmt.Errorf("image not found: %s", imageName)
65	-	return "", fmt.Errorf("image not found")	88	+	}
			89	+
			90	+	func normalizeImageName(name string) (string, string) {
			91	+	// Split off the tag if present
			92	+	parts := strings.SplitN(name, ":", 2)
			93	+	normalizedName := parts[0]
			94	+	tag := "latest"
			95	+	if len(parts) > 1 {
			96	+	tag = parts[1]
			97	+	}
			98	+
			99	+	// Remove any leading "docker.io/" if present
			100	+	normalizedName = strings.TrimPrefix(normalizedName, "docker.io/")
			101	+
			102	+	// Handle repository names with multiple segments
			103	+	segments := strings.Split(normalizedName, "/")
			104	+	if len(segments) > 1 {
			105	+	// If there are multiple segments, join all but the last
			106	+	normalizedName = strings.Join(segments[:len(segments)-1], "/") + "/" + segments[len(segments)-1]
66		}	107		}
67			108
68	-	return imageID, nil	109	+	return normalizedName, tag
			110	+	}
			111	+
			112	+	func createTagPattern(tag string) *regexp.Regexp {
			113	+	if tag == "" \|\| tag == "latest" {
			114	+	// Match any tag if the requested tag is empty or "latest"
			115	+	return regexp.MustCompile(`.*`)
			116	+	}
			117	+
			118	+	// Escape special regex characters in the tag
			119	+	escapedTag := regexp.QuoteMeta(tag)
			120	+
			121	+	// Create a pattern that matches the tag exactly or as a prefix
			122	+	// This allows matching "1.0" with "1.0.1", "1.0-alpine", etc.
			123	+	return regexp.MustCompile(`^` + escapedTag + `($\|[\.-])`)
69		}	124		}
70			125
71	-	// ImportImageToEngine imports an image to the Docker engine	126	+	// ImportImageToEngine imports an image to the Docker engine and returns the image ID
72	-	func ImportImageToEngine(imageFilePath string) error {	127	+	func ImportImageToEngine(imageFilePath string) (string, error) {
73	-	// Open the image file	128	+	log.Printf("Starting to import image from file: %s", imageFilePath)
			129	+
74		imageFile, err := os.Open(imageFilePath)	130		imageFile, err := os.Open(imageFilePath)
75		if err != nil {	131		if err != nil {
76	-	return fmt.Errorf("failed to open image file: %w", err)	132	+	log.Printf("Failed to open image file: %v", err)
			133	+	return "", fmt.Errorf("failed to open image file: %w", err)
77		}	134		}
78		defer imageFile.Close()	135		defer imageFile.Close()
79			136
80	-	// Import the image using the Docker client	137	+	log.Printf("Image file opened successfully")
81	-	importedImage, err := dockerCli.ImageLoad(context.Background(), imageFile, false)	138	+
			139	+	resp, err := dockerCli.ImageLoad(context.Background(), imageFile, true)
82		if err != nil {	140		if err != nil {
83	-	return fmt.Errorf("failed to import image: %w", err)	141	+	log.Printf("Failed to load image: %v", err)
			142	+	return "", fmt.Errorf("failed to import image: %w", err)
84		}	143		}
85	-	defer importedImage.Body.Close()	144	+	defer resp.Body.Close()
86			145
87	-	return nil	146	+	log.Printf("Image load initiated, parsing response")
			147	+
			148	+	var imageName string
			149	+	scanner := bufio.NewScanner(resp.Body)
			150	+	for scanner.Scan() {
			151	+	line := scanner.Text()
			152	+	log.Printf("Response line: %s", line)
			153	+
			154	+	var message struct {
			155	+	Stream string `json:"stream"`
			156	+	}
			157	+	if err := json.Unmarshal([]byte(line), &message); err != nil {
			158	+	log.Printf("Failed to unmarshal JSON: %v", err)
			159	+	continue // Skip lines that aren't JSON
			160	+	}
			161	+
			162	+	log.Printf("Parsed message stream: %s", message.Stream)
			163	+
			164	+	if strings.HasPrefix(message.Stream, "Loaded image: ") {
			165	+	imageName = strings.TrimSpace(strings.TrimPrefix(message.Stream, "Loaded image: "))
			166	+	log.Printf("Found image name: %s", imageName)
			167	+	break
			168	+	}
			169	+	}
			170	+
			171	+	if err := scanner.Err(); err != nil {
			172	+	log.Printf("Error reading response: %v", err)
			173	+	return "", fmt.Errorf("error reading response: %w", err)
			174	+	}
			175	+
			176	+	if imageName == "" {
			177	+	log.Printf("Failed to find image name in import response")
			178	+	return "", fmt.Errorf("failed to find image name in import response")
			179	+	}
			180	+
			181	+	// Now we need to get the image ID from the image name
			182	+	imageID, err := GetImageIDByName(imageName)
			183	+	if err != nil {
			184	+	log.Printf("Failed to get image ID for name %s: %v", imageName, err)
			185	+	return "", fmt.Errorf("failed to get image ID: %w", err)
			186	+	}
			187	+
			188	+	log.Printf("Successfully imported image with ID: %s", imageID)
			189	+	return imageID, nil
88		}	190		}
89			191
90		// ExportImageFromEngine exports an image from the Docker engine and returns it as an io.Reader	192		// ExportImageFromEngine exports an image from the Docker engine and returns it as an io.Reader