discover_bedrock_capture_live_test.go RawHistoryFolder

1package provider
2
3import (
4	"context"
5	"encoding/json"
6	"net/http"
7	"os"
8	"strings"
9	"testing"
10	"time"
11)
12
13// Live Bedrock discovery diagnostic / capture helper.
14//
15// This is NOT a validation test. It lets a developer with real AWS
16// credentials capture actual control-plane and Price List responses to
17// verify discovery output and re-check the pricing situation. It never
18// runs in `go test ./...` unless explicitly enabled, so CI and unpaid
19// checkouts never hit the network.
20//
21// Background: a live capture (Sep 2026, us-east-1) confirmed the AWS
22// Price List API returns only stale, input-only rows for legacy Anthropic
23// models (Claude 2.x / 3 Sonnet / 3 Haiku) and no output/cache prices for
24// any current model, so zot does NOT enrich pricing from it — the catalog
25// stays authoritative. This helper is retained to detect if/when AWS
26// fixes that, and to confirm ListFoundationModels / ListInferenceProfiles
27// shapes when they change.
28//
29// Enable and run:
30//
31//	ZOT_BEDROCK_CAPTURE=1 \
32//	AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... [AWS_SESSION_TOKEN=...] \
33//	AWS_REGION=us-east-1 \
34//	go test ./packages/provider -run TestBedrockCaptureLive -v
35//
36// Or with a profile:
37//
38//	ZOT_BEDROCK_CAPTURE=1 AWS_PROFILE=my-sso AWS_REGION=us-east-1 \
39//	go test ./packages/provider -run TestBedrockCaptureLive -v
40//
41// Output is written to files under $ZOT_BEDROCK_CAPTURE_DIR (default: a
42// temp dir printed in the log). Secrets are never captured — only the
43// public model/pricing JSON.
44//
45// Requires the caller's IAM principal to allow:
46//   - bedrock:ListFoundationModels
47//   - bedrock:ListInferenceProfiles
48//   - pricing:GetProducts
49func TestBedrockCaptureLive(t *testing.T) {
50	if strings.TrimSpace(os.Getenv("ZOT_BEDROCK_CAPTURE")) == "" {
51		t.Skip("set ZOT_BEDROCK_CAPTURE=1 (with AWS creds) to capture live Bedrock responses")
52	}
53
54	_, sigv4 := resolveBedrockAuth("")
55	if sigv4 == nil {
56		t.Fatal("no SigV4 credentials found; set AWS_ACCESS_KEY_ID+AWS_SECRET_ACCESS_KEY or AWS_PROFILE")
57	}
58	region := bedrockResolveRegion()
59	t.Logf("capturing Bedrock responses for region=%s", region)
60
61	outDir := strings.TrimSpace(os.Getenv("ZOT_BEDROCK_CAPTURE_DIR"))
62	if outDir == "" {
63		outDir = t.TempDir()
64	}
65	t.Logf("writing captures to %s", outDir)
66
67	client := &http.Client{Timeout: 30 * time.Second}
68	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
69	defer cancel()
70
71	// 1. ListFoundationModels (control plane, GET).
72	fmURL := "https://bedrock." + region + ".amazonaws.com/foundation-models"
73	if body, err := bedrockSignedControlPlane(ctx, client, sigv4, region, http.MethodGet, fmURL, nil); err != nil {
74		t.Errorf("ListFoundationModels failed: %v", err)
75	} else {
76		writeCapture(t, outDir, "foundation-models.json", body)
77		// Also log the summarized model IDs so the capturer can eyeball them.
78		logBedrockModelIDs(t, body)
79	}
80
81	// 2. ListInferenceProfiles (control plane, GET, paginated).
82	ipURL := "https://bedrock." + region + ".amazonaws.com/inference-profiles?maxResults=1000"
83	if body, err := bedrockSignedControlPlane(ctx, client, sigv4, region, http.MethodGet, ipURL, nil); err != nil {
84		t.Errorf("ListInferenceProfiles failed: %v", err)
85	} else {
86		writeCapture(t, outDir, "inference-profiles.json", body)
87	}
88
89	// 3. GetProducts for AmazonBedrock (Price List API, POST).
90	//    Capture the FIRST page raw so we can inspect real attribute
91	//    names, inferenceType values, feature strings, and units. We
92	//    deliberately do not paginate here — one page is enough to fix
93	//    the fixtures, and it keeps the captured file small.
94	captureBedrockPricingPage(ctx, t, client, sigv4, region, outDir)
95
96	t.Logf("done. Inspect the files in %s and share them (they contain only public pricing/model data).", outDir)
97}
98
99// captureBedrockPricingPage issues a single GetProducts page filtered to
100// the region and the Anthropic provider (the family we care about most)
101// and writes the raw response. Anthropic is the case the aws-cli #9567
102// report flagged as possibly missing output-token rows, so capturing it
103// specifically lets us confirm or refute that.
104func captureBedrockPricingPage(ctx context.Context, t *testing.T, client *http.Client, creds *bedrockSigV4Creds, region, outDir string) {
105	const pricingRegion = "us-east-1"
106	url := "https://api.pricing." + pricingRegion + ".amazonaws.com/"
107
108	reqBody := map[string]any{
109		"ServiceCode":   "AmazonBedrock",
110		"FormatVersion": "aws_v1",
111		"MaxResults":    100,
112		"Filters": []map[string]string{
113			{"Type": "TERM_MATCH", "Field": "regionCode", "Value": region},
114			{"Type": "TERM_MATCH", "Field": "provider", "Value": "Anthropic"},
115		},
116	}
117	payload, err := json.Marshal(reqBody)
118	if err != nil {
119		t.Errorf("marshal pricing request: %v", err)
120		return
121	}
122	req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytesReader(payload))
123	if err != nil {
124		t.Errorf("build pricing request: %v", err)
125		return
126	}
127	req.Header.Set("content-type", "application/x-amz-json-1.1")
128	req.Header.Set("x-amz-target", "AWSPriceListService.GetProducts")
129	if err := signSigV4(req, payload, "pricing", pricingRegion, creds, time.Now().UTC()); err != nil {
130		t.Errorf("sign pricing request: %v", err)
131		return
132	}
133	resp, err := client.Do(req)
134	if err != nil {
135		t.Errorf("pricing request failed: %v", err)
136		return
137	}
138	defer resp.Body.Close()
139	var page struct {
140		PriceList []string `json:"PriceList"`
141		NextToken string   `json:"NextToken"`
142	}
143	body := readAll(t, resp)
144	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
145		t.Errorf("pricing http %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
146		return
147	}
148	writeCapture(t, outDir, "pricing-anthropic-raw.json", body)
149
150	// Pretty-print each PriceList entry (they are JSON-encoded strings)
151	// into a single readable file, and log a compact attribute summary.
152	if err := json.Unmarshal(body, &page); err != nil {
153		t.Errorf("parse pricing page: %v", err)
154		return
155	}
156	t.Logf("pricing page returned %d PriceList entries (NextToken present: %v)", len(page.PriceList), page.NextToken != "")
157	var pretty strings.Builder
158	for _, raw := range page.PriceList {
159		var obj any
160		if err := json.Unmarshal([]byte(raw), &obj); err != nil {
161			continue
162		}
163		b, _ := json.MarshalIndent(obj, "", "  ")
164		pretty.Write(b)
165		pretty.WriteByte('\n')
166		logBedrockPriceAttributes(t, raw)
167	}
168	writeCapture(t, outDir, "pricing-anthropic-pretty.json", []byte(pretty.String()))
169}
170
171func logBedrockModelIDs(t *testing.T, body []byte) {
172	var fm struct {
173		ModelSummaries []struct {
174			ModelID          string   `json:"modelId"`
175			OutputModalities []string `json:"outputModalities"`
176		} `json:"modelSummaries"`
177	}
178	if err := json.Unmarshal(body, &fm); err != nil {
179		return
180	}
181	t.Logf("ListFoundationModels: %d models", len(fm.ModelSummaries))
182	for _, s := range fm.ModelSummaries {
183		if strings.Contains(strings.ToLower(s.ModelID), "anthropic") {
184			t.Logf("  modelId=%s outputModalities=%v", s.ModelID, s.OutputModalities)
185		}
186	}
187}
188
189func logBedrockPriceAttributes(t *testing.T, raw string) {
190	var entry struct {
191		Product struct {
192			Attributes map[string]string `json:"attributes"`
193		} `json:"product"`
194	}
195	if err := json.Unmarshal([]byte(raw), &entry); err != nil {
196		return
197	}
198	a := entry.Product.Attributes
199	t.Logf("  price: model=%q inferenceType=%q feature=%q usagetype=%q",
200		a["model"], a["inferenceType"], a["feature"], a["usagetype"])
201}
202
203func writeCapture(t *testing.T, dir, name string, body []byte) {
204	t.Helper()
205	path := dir + string(os.PathSeparator) + name
206	if err := os.WriteFile(path, body, 0o644); err != nil {
207		t.Errorf("write %s: %v", path, err)
208		return
209	}
210	t.Logf("wrote %s (%d bytes)", path, len(body))
211}
212
213func readAll(t *testing.T, resp *http.Response) []byte {
214	t.Helper()
215	buf := make([]byte, 0, 1<<20)
216	tmp := make([]byte, 32<<10)
217	for {
218		n, err := resp.Body.Read(tmp)
219		if n > 0 {
220			buf = append(buf, tmp[:n]...)
221		}
222		if err != nil {
223			break
224		}
225		if len(buf) > 32<<20 {
226			break
227		}
228	}
229	return buf
230}