detached 1a49277

core: read sessions with line reader instead of bufio.Scanner

Loading or exporting a session containing very large JSONL rows
(image blocks, big tool outputs, compacted history) failed with
'bufio.Scanner: token too long' — Scanner caps each token to its
buffer size, even when bumped to 20 MiB. A single oversized row
blocked OpenSession entirely so an existing long session could
not be resumed.

Switch session readers (OpenSession, SessionUsage, describeSession,
sessionHasNoMessages, ExportSession, ImportSession, BranchSession,
firstUserPrompt) to a shared bufio.Reader.ReadBytes-based JSONL
helper that handles arbitrarily long lines. Add a regression test
that opens and exports a session containing a >20 MiB row.
commit
1a49277130e730448b16d0284be4ee361fc9510a
author
patriceckhart · · 2026-05-04 10:14 UTC
parent
b45f107
signature
unsigned
refs
tag: v0.1.49
3 files · +188 -113
Unified
UnifiedSplit
M internal/core/session.go +54 -35
@@ -2,10 +2,12 @@ package core
22
33 import (
44 "bufio"
5+ "bytes"
56 "crypto/sha256"
67 "encoding/hex"
78 "encoding/json"
89 "fmt"
10+ "io"
911 "os"
1012 "path/filepath"
1113 "sort"
@@ -110,6 +112,27 @@ func NewSession(root, cwd, providerName, model, version string) (*Session, error
110112 return s, nil
111113 }
112114
115+func forEachJSONLLine(r io.Reader, fn func([]byte) error) error {
116+ br := bufio.NewReader(r)
117+ for {
118+ line, err := br.ReadBytes('\n')
119+ if len(line) > 0 {
120+ line = bytes.TrimRight(line, "\r\n")
121+ if len(line) > 0 {
122+ if ferr := fn(line); ferr != nil {
123+ return ferr
124+ }
125+ }
126+ }
127+ if err == io.EOF {
128+ return nil
129+ }
130+ if err != nil {
131+ return err
132+ }
133+ }
134+}
135+
113136 // SessionUsage returns the most recent cumulative usage row stored in
114137 // a session file. Sessions append one usage row per completed turn; the
115138 // latest row's cumulative field is the session total. Missing usage rows
@@ -122,21 +145,19 @@ func SessionUsage(path string) (provider.Usage, error) {
122145 defer f.Close()
123146
124147 var usage provider.Usage
125- sc := bufio.NewScanner(f)
126- sc.Buffer(make([]byte, 0, 64*1024), 20*1024*1024)
127- for sc.Scan() {
148+ if err := forEachJSONLLine(f, func(line []byte) error {
128149 var head sessionLineHead
129- if err := json.Unmarshal(sc.Bytes(), &head); err != nil || head.Type != "usage" {
130- continue
150+ if err := json.Unmarshal(line, &head); err != nil || head.Type != "usage" {
151+ return nil
131152 }
132153 var row struct {
133154 Cumulative provider.Usage `json:"cumulative"`
134155 }
135- if err := json.Unmarshal(sc.Bytes(), &row); err == nil {
156+ if err := json.Unmarshal(line, &row); err == nil {
136157 usage = row.Cumulative
137158 }
138- }
139- if err := sc.Err(); err != nil {
159+ return nil
160+ }); err != nil {
140161 return provider.Usage{}, err
141162 }
142163 return usage, nil
@@ -152,28 +173,26 @@ func OpenSession(path string) (*Session, []provider.Message, error) {
152173
153174 var meta SessionMeta
154175 var messages []provider.Message
155- sc := bufio.NewScanner(f)
156- sc.Buffer(make([]byte, 0, 64*1024), 20*1024*1024)
157- for sc.Scan() {
176+ if err := forEachJSONLLine(f, func(line []byte) error {
158177 var head sessionLineHead
159- if err := json.Unmarshal(sc.Bytes(), &head); err != nil {
160- continue
178+ if err := json.Unmarshal(line, &head); err != nil {
179+ return nil
161180 }
162181 switch head.Type {
163182 case "meta":
164183 var row struct {
165184 Meta SessionMeta `json:"meta"`
166185 }
167- if err := json.Unmarshal(sc.Bytes(), &row); err == nil {
186+ if err := json.Unmarshal(line, &row); err == nil {
168187 meta = row.Meta
169188 }
170189 case "message":
171- if msg, err := hydrateMessage(sc.Bytes()); err == nil && len(msg.Content) > 0 {
190+ if msg, err := hydrateMessage(line); err == nil && len(msg.Content) > 0 {
172191 messages = append(messages, msg)
173192 }
174193 }
175- }
176- if err := sc.Err(); err != nil {
194+ return nil
195+ }); err != nil {
177196 return nil, nil, err
178197 }
179198 messages = repairToolUseResultPairs(messages)
@@ -333,19 +352,17 @@ func describeSession(path string) SessionSummary {
333352 return s
334353 }
335354 defer f.Close()
336- sc := bufio.NewScanner(f)
337- sc.Buffer(make([]byte, 0, 64*1024), 20*1024*1024)
338- for sc.Scan() {
355+ _ = forEachJSONLLine(f, func(line []byte) error {
339356 var head sessionLineHead
340- if err := json.Unmarshal(sc.Bytes(), &head); err != nil {
341- continue
357+ if err := json.Unmarshal(line, &head); err != nil {
358+ return nil
342359 }
343360 switch head.Type {
344361 case "meta":
345362 var row struct {
346363 Meta SessionMeta `json:"meta"`
347364 }
348- if err := json.Unmarshal(sc.Bytes(), &row); err == nil {
365+ if err := json.Unmarshal(line, &row); err == nil {
349366 s.Started = row.Meta.Started
350367 s.Model = row.Meta.Model
351368 s.Provider = row.Meta.Provider
@@ -354,24 +371,25 @@ func describeSession(path string) SessionSummary {
354371 case "message":
355372 s.MessageCount++
356373 if s.FirstUserText == "" {
357- s.FirstUserText = firstUserText(sc.Bytes())
374+ s.FirstUserText = firstUserText(line)
358375 }
359376 case "rename":
360377 var row struct {
361378 Title string `json:"title"`
362379 }
363- if err := json.Unmarshal(sc.Bytes(), &row); err == nil && row.Title != "" {
380+ if err := json.Unmarshal(line, &row); err == nil && row.Title != "" {
364381 s.Title = row.Title
365382 }
366383 case "usage":
367384 var row struct {
368385 Cumulative provider.Usage `json:"cumulative"`
369386 }
370- if err := json.Unmarshal(sc.Bytes(), &row); err == nil {
387+ if err := json.Unmarshal(line, &row); err == nil {
371388 s.TotalCost = row.Cumulative.CostUSD
372389 }
373390 }
374- }
391+ return nil
392+ })
375393 return s
376394 }
377395
@@ -429,18 +447,19 @@ func sessionHasNoMessages(path string) bool {
429447 return false
430448 }
431449 defer f.Close()
432- sc := bufio.NewScanner(f)
433- sc.Buffer(make([]byte, 0, 64*1024), 20*1024*1024)
434- for sc.Scan() {
450+ hasMessage := false
451+ _ = forEachJSONLLine(f, func(line []byte) error {
435452 var head sessionLineHead
436- if err := json.Unmarshal(sc.Bytes(), &head); err != nil {
437- continue
453+ if err := json.Unmarshal(line, &head); err != nil {
454+ return nil
438455 }
439456 if head.Type == "message" {
440- return false
457+ hasMessage = true
458+ return io.EOF
441459 }
442- }
443- return true
460+ return nil
461+ })
462+ return !hasMessage
444463 }
445464
446465 // ListSessions returns session file paths for cwd, most-recently-
@@ -2,10 +2,12 @@ package core
2 2
3 import (3 import (
4 "bufio"4 "bufio"
5+ "bytes"
5 "crypto/sha256"6 "crypto/sha256"
6 "encoding/hex"7 "encoding/hex"
7 "encoding/json"8 "encoding/json"
8 "fmt"9 "fmt"
10+ "io"
9 "os"11 "os"
10 "path/filepath"12 "path/filepath"
11 "sort"13 "sort"
@@ -110,6 +112,27 @@ func NewSession(root, cwd, providerName, model, version string) (*Session, error
110 return s, nil112 return s, nil
111 }113 }
112 114
115+func forEachJSONLLine(r io.Reader, fn func([]byte) error) error {
116+ br := bufio.NewReader(r)
117+ for {
118+ line, err := br.ReadBytes('\n')
119+ if len(line) > 0 {
120+ line = bytes.TrimRight(line, "\r\n")
121+ if len(line) > 0 {
122+ if ferr := fn(line); ferr != nil {
123+ return ferr
124+ }
125+ }
126+ }
127+ if err == io.EOF {
128+ return nil
129+ }
130+ if err != nil {
131+ return err
132+ }
133+ }
134+}
135+
113 // SessionUsage returns the most recent cumulative usage row stored in136 // SessionUsage returns the most recent cumulative usage row stored in
114 // a session file. Sessions append one usage row per completed turn; the137 // a session file. Sessions append one usage row per completed turn; the
115 // latest row's cumulative field is the session total. Missing usage rows138 // latest row's cumulative field is the session total. Missing usage rows
@@ -122,21 +145,19 @@ func SessionUsage(path string) (provider.Usage, error) {
122 defer f.Close()145 defer f.Close()
123 146
124 var usage provider.Usage147 var usage provider.Usage
125- sc := bufio.NewScanner(f)148+ if err := forEachJSONLLine(f, func(line []byte) error {
126- sc.Buffer(make([]byte, 0, 64*1024), 20*1024*1024)
127- for sc.Scan() {
128 var head sessionLineHead149 var head sessionLineHead
129- if err := json.Unmarshal(sc.Bytes(), &head); err != nil || head.Type != "usage" {150+ if err := json.Unmarshal(line, &head); err != nil || head.Type != "usage" {
130- continue151+ return nil
131 }152 }
132 var row struct {153 var row struct {
133 Cumulative provider.Usage `json:"cumulative"`154 Cumulative provider.Usage `json:"cumulative"`
134 }155 }
135- if err := json.Unmarshal(sc.Bytes(), &row); err == nil {156+ if err := json.Unmarshal(line, &row); err == nil {
136 usage = row.Cumulative157 usage = row.Cumulative
137 }158 }
138- }159+ return nil
139- if err := sc.Err(); err != nil {160+ }); err != nil {
140 return provider.Usage{}, err161 return provider.Usage{}, err
141 }162 }
142 return usage, nil163 return usage, nil
@@ -152,28 +173,26 @@ func OpenSession(path string) (*Session, []provider.Message, error) {
152 173
153 var meta SessionMeta174 var meta SessionMeta
154 var messages []provider.Message175 var messages []provider.Message
155- sc := bufio.NewScanner(f)176+ if err := forEachJSONLLine(f, func(line []byte) error {
156- sc.Buffer(make([]byte, 0, 64*1024), 20*1024*1024)
157- for sc.Scan() {
158 var head sessionLineHead177 var head sessionLineHead
159- if err := json.Unmarshal(sc.Bytes(), &head); err != nil {178+ if err := json.Unmarshal(line, &head); err != nil {
160- continue179+ return nil
161 }180 }
162 switch head.Type {181 switch head.Type {
163 case "meta":182 case "meta":
164 var row struct {183 var row struct {
165 Meta SessionMeta `json:"meta"`184 Meta SessionMeta `json:"meta"`
166 }185 }
167- if err := json.Unmarshal(sc.Bytes(), &row); err == nil {186+ if err := json.Unmarshal(line, &row); err == nil {
168 meta = row.Meta187 meta = row.Meta
169 }188 }
170 case "message":189 case "message":
171- if msg, err := hydrateMessage(sc.Bytes()); err == nil && len(msg.Content) > 0 {190+ if msg, err := hydrateMessage(line); err == nil && len(msg.Content) > 0 {
172 messages = append(messages, msg)191 messages = append(messages, msg)
173 }192 }
174 }193 }
175- }194+ return nil
176- if err := sc.Err(); err != nil {195+ }); err != nil {
177 return nil, nil, err196 return nil, nil, err
178 }197 }
179 messages = repairToolUseResultPairs(messages)198 messages = repairToolUseResultPairs(messages)
@@ -333,19 +352,17 @@ func describeSession(path string) SessionSummary {
333 return s352 return s
334 }353 }
335 defer f.Close()354 defer f.Close()
336- sc := bufio.NewScanner(f)355+ _ = forEachJSONLLine(f, func(line []byte) error {
337- sc.Buffer(make([]byte, 0, 64*1024), 20*1024*1024)
338- for sc.Scan() {
339 var head sessionLineHead356 var head sessionLineHead
340- if err := json.Unmarshal(sc.Bytes(), &head); err != nil {357+ if err := json.Unmarshal(line, &head); err != nil {
341- continue358+ return nil
342 }359 }
343 switch head.Type {360 switch head.Type {
344 case "meta":361 case "meta":
345 var row struct {362 var row struct {
346 Meta SessionMeta `json:"meta"`363 Meta SessionMeta `json:"meta"`
347 }364 }
348- if err := json.Unmarshal(sc.Bytes(), &row); err == nil {365+ if err := json.Unmarshal(line, &row); err == nil {
349 s.Started = row.Meta.Started366 s.Started = row.Meta.Started
350 s.Model = row.Meta.Model367 s.Model = row.Meta.Model
351 s.Provider = row.Meta.Provider368 s.Provider = row.Meta.Provider
@@ -354,24 +371,25 @@ func describeSession(path string) SessionSummary {
354 case "message":371 case "message":
355 s.MessageCount++372 s.MessageCount++
356 if s.FirstUserText == "" {373 if s.FirstUserText == "" {
357- s.FirstUserText = firstUserText(sc.Bytes())374+ s.FirstUserText = firstUserText(line)
358 }375 }
359 case "rename":376 case "rename":
360 var row struct {377 var row struct {
361 Title string `json:"title"`378 Title string `json:"title"`
362 }379 }
363- if err := json.Unmarshal(sc.Bytes(), &row); err == nil && row.Title != "" {380+ if err := json.Unmarshal(line, &row); err == nil && row.Title != "" {
364 s.Title = row.Title381 s.Title = row.Title
365 }382 }
366 case "usage":383 case "usage":
367 var row struct {384 var row struct {
368 Cumulative provider.Usage `json:"cumulative"`385 Cumulative provider.Usage `json:"cumulative"`
369 }386 }
370- if err := json.Unmarshal(sc.Bytes(), &row); err == nil {387+ if err := json.Unmarshal(line, &row); err == nil {
371 s.TotalCost = row.Cumulative.CostUSD388 s.TotalCost = row.Cumulative.CostUSD
372 }389 }
373 }390 }
374- }391+ return nil
392+ })
375 return s393 return s
376 }394 }
377 395
@@ -429,18 +447,19 @@ func sessionHasNoMessages(path string) bool {
429 return false447 return false
430 }448 }
431 defer f.Close()449 defer f.Close()
432- sc := bufio.NewScanner(f)450+ hasMessage := false
433- sc.Buffer(make([]byte, 0, 64*1024), 20*1024*1024)451+ _ = forEachJSONLLine(f, func(line []byte) error {
434- for sc.Scan() {
435 var head sessionLineHead452 var head sessionLineHead
436- if err := json.Unmarshal(sc.Bytes(), &head); err != nil {453+ if err := json.Unmarshal(line, &head); err != nil {
437- continue454+ return nil
438 }455 }
439 if head.Type == "message" {456 if head.Type == "message" {
440- return false457+ hasMessage = true
458+ return io.EOF
441 }459 }
442- }460+ return nil
443- return true461+ })
462+ return !hasMessage
444 }463 }
445 464
446 // ListSessions returns session file paths for cwd, most-recently-465 // ListSessions returns session file paths for cwd, most-recently-
M internal/core/session_portable.go +84 -78
@@ -2,6 +2,7 @@ package core
22
33 import (
44 "bufio"
5+ "bytes"
56 "encoding/json"
67 "errors"
78 "fmt"
@@ -66,7 +67,11 @@ func ExportSession(srcPath, dstPath string) (string, error) {
6667 firstPrompt := ""
6768 if !strings.HasSuffix(strings.ToLower(dstPath), PortableExt) {
6869 if fi, _ := os.Stat(dstPath); fi == nil || fi.IsDir() {
69- firstPrompt = firstUserPrompt(sc)
70+ p, err := firstUserPrompt(src)
71+ if err != nil {
72+ return "", fmt.Errorf("export: read first prompt: %w", err)
73+ }
74+ firstPrompt = p
7075 }
7176 }
7277
@@ -110,28 +115,32 @@ func ExportSession(srcPath, dstPath string) (string, error) {
110115 return "", err
111116 }
112117
113- // Stream every non-meta row verbatim.
114- sc2 := bufio.NewScanner(src)
115- sc2.Buffer(make([]byte, 0, 64*1024), 20*1024*1024)
116- for sc2.Scan() {
117- line := sc2.Bytes()
118- var h sessionLineHead
119- if err := json.Unmarshal(line, &h); err != nil {
120- continue
121- }
122- if h.Type == "meta" {
123- continue // already wrote a rewritten copy above
118+ // Stream every non-meta row verbatim. Use ReadBytes instead of
119+ // bufio.Scanner: large sessions can contain very long JSONL rows
120+ // (image blocks, big tool outputs, compacted history) that exceed
121+ // Scanner's token limit and fail with "token too long".
122+ r := bufio.NewReader(src)
123+ for {
124+ line, err := r.ReadBytes('\n')
125+ if len(line) > 0 {
126+ line = bytes.TrimRight(line, "\r\n")
127+ var h sessionLineHead
128+ if err := json.Unmarshal(line, &h); err == nil && h.Type != "meta" {
129+ if _, werr := bw.Write(line); werr != nil {
130+ return "", werr
131+ }
132+ if werr := bw.WriteByte('\n'); werr != nil {
133+ return "", werr
134+ }
135+ }
124136 }
125- if _, err := bw.Write(line); err != nil {
126- return "", err
137+ if err == io.EOF {
138+ break
127139 }
128- if err := bw.WriteByte('\n'); err != nil {
129- return "", err
140+ if err != nil {
141+ return "", fmt.Errorf("export: read source: %w", err)
130142 }
131143 }
132- if err := sc2.Err(); err != nil {
133- return "", fmt.Errorf("export: read source: %w", err)
134- }
135144 if err := bw.Flush(); err != nil {
136145 return "", err
137146 }
@@ -207,29 +216,22 @@ func ImportSession(srcPath, root, cwd, version string) (string, error) {
207216 return "", err
208217 }
209218
210- // Rewind the source and stream every non-meta row.
219+ // Rewind the source and stream every non-meta row. Avoid
220+ // bufio.Scanner so exported sessions with huge JSONL rows import
221+ // cleanly.
211222 if _, err := src.Seek(0, io.SeekStart); err != nil {
212223 return "", fmt.Errorf("import: rewind: %w", err)
213224 }
214- sc2 := bufio.NewScanner(src)
215- sc2.Buffer(make([]byte, 0, 64*1024), 20*1024*1024)
216- for sc2.Scan() {
217- line := sc2.Bytes()
225+ if err := forEachJSONLLine(src, func(line []byte) error {
218226 var h sessionLineHead
219- if err := json.Unmarshal(line, &h); err != nil {
220- continue
221- }
222- if h.Type == "meta" {
223- continue
227+ if err := json.Unmarshal(line, &h); err != nil || h.Type == "meta" {
228+ return nil
224229 }
225230 if _, err := bw.Write(line); err != nil {
226- return "", err
231+ return err
227232 }
228- if err := bw.WriteByte('\n'); err != nil {
229- return "", err
230- }
231- }
232- if err := sc2.Err(); err != nil {
233+ return bw.WriteByte('\n')
234+ }); err != nil {
233235 return "", fmt.Errorf("import: read source: %w", err)
234236 }
235237 if err := bw.Flush(); err != nil {
@@ -321,34 +323,39 @@ func BranchSession(parentPath, root, cwd, version string, upToMessageIdx int) (s
321323
322324 // Copy message rows up to the cut point, plus all usage rows
323325 // that land before the cut (they describe the cost of those
324- // messages).
326+ // messages). Rewind and use the large-row-safe JSONL reader.
327+ if _, err := src.Seek(0, io.SeekStart); err != nil {
328+ return "", fmt.Errorf("branch: rewind parent: %w", err)
329+ }
325330 msgCount := 0
326- for sc.Scan() && msgCount < upToMessageIdx {
327- line := sc.Bytes()
331+ if err := forEachJSONLLine(src, func(line []byte) error {
332+ if msgCount >= upToMessageIdx {
333+ return io.EOF
334+ }
328335 var h sessionLineHead
329336 if err := json.Unmarshal(line, &h); err != nil {
330- continue
337+ return nil
331338 }
332339 switch h.Type {
333340 case "message":
334341 if _, err := bw.Write(line); err != nil {
335- return "", err
342+ return err
336343 }
337344 if err := bw.WriteByte('\n'); err != nil {
338- return "", err
345+ return err
339346 }
340347 msgCount++
341348 case "usage":
342349 if _, err := bw.Write(line); err != nil {
343- return "", err
350+ return err
344351 }
345352 if err := bw.WriteByte('\n'); err != nil {
346- return "", err
353+ return err
347354 }
348355 // don't increment msgCount for usage rows
349356 }
350- }
351- if err := sc.Err(); err != nil {
357+ return nil
358+ }); err != nil && err != io.EOF {
352359 return "", fmt.Errorf("branch: read parent: %w", err)
353360 }
354361 if err := bw.Flush(); err != nil {
@@ -456,43 +463,42 @@ func FindSessionByID(root, cwd, id string) string {
456463 return ""
457464 }
458465
459-// firstUserPrompt scans forward from the current scanner position
460-// looking for the first user-role message and returns its text
461-// (trimmed, short). Used to build a humane export filename.
462-func firstUserPrompt(sc *bufio.Scanner) string {
463- for sc.Scan() {
464- var line sessionLine
465- if err := json.Unmarshal(sc.Bytes(), &line); err != nil {
466- continue
467- }
468- if line.Type != "message" || line.Message == nil || line.Message.Role != "user" {
469- continue
470- }
471- for _, c := range line.Message.Content {
472- // Use type name to avoid an import of provider here beyond
473- // the already-imported alias; TextBlock is the only content
474- // shape that yields a useful preview, so we just look for
475- // something with a reasonable string form.
476- s := fmt.Sprintf("%v", c)
477- _ = s // formatted value is too noisy; go straight to typed path
466+// firstUserPrompt scans forward from the current source position
467+// looking for the first user-role message and returns its text.
468+// Used to build a humane export filename. Uses Reader instead of
469+// Scanner so a very large JSONL row before the first user prompt
470+// cannot trip Scanner's token limit.
471+func firstUserPrompt(src io.Reader) (string, error) {
472+ r := bufio.NewReader(src)
473+ for {
474+ lineBytes, err := r.ReadBytes('\n')
475+ if len(lineBytes) > 0 {
476+ lineBytes = bytes.TrimRight(lineBytes, "\r\n")
477+ var line sessionLine
478+ if err := json.Unmarshal(lineBytes, &line); err == nil {
479+ if line.Type == "message" && line.Message != nil && line.Message.Role == "user" {
480+ b, _ := json.Marshal(line.Message)
481+ var m struct {
482+ Content []struct {
483+ Text string `json:"text"`
484+ } `json:"content"`
485+ }
486+ _ = json.Unmarshal(b, &m)
487+ for _, c := range m.Content {
488+ if c.Text != "" {
489+ return c.Text, nil
490+ }
491+ }
492+ }
493+ }
478494 }
479- // Simpler: marshal the message and fish out the first "text"
480- // we can find. Avoids reaching into the provider package just
481- // for an interface type check.
482- b, _ := json.Marshal(line.Message)
483- var m struct {
484- Content []struct {
485- Text string `json:"text"`
486- } `json:"content"`
495+ if err == io.EOF {
496+ return "", nil
487497 }
488- _ = json.Unmarshal(b, &m)
489- for _, c := range m.Content {
490- if c.Text != "" {
491- return c.Text
492- }
498+ if err != nil {
499+ return "", err
493500 }
494501 }
495- return ""
496502 }
497503
498504 // filenameFor builds a descriptive .zotsession filename from the
@@ -2,6 +2,7 @@ package core
2 2
3 import (3 import (
4 "bufio"4 "bufio"
5+ "bytes"
5 "encoding/json"6 "encoding/json"
6 "errors"7 "errors"
7 "fmt"8 "fmt"
@@ -66,7 +67,11 @@ func ExportSession(srcPath, dstPath string) (string, error) {
66 firstPrompt := ""67 firstPrompt := ""
67 if !strings.HasSuffix(strings.ToLower(dstPath), PortableExt) {68 if !strings.HasSuffix(strings.ToLower(dstPath), PortableExt) {
68 if fi, _ := os.Stat(dstPath); fi == nil || fi.IsDir() {69 if fi, _ := os.Stat(dstPath); fi == nil || fi.IsDir() {
69- firstPrompt = firstUserPrompt(sc)70+ p, err := firstUserPrompt(src)
71+ if err != nil {
72+ return "", fmt.Errorf("export: read first prompt: %w", err)
73+ }
74+ firstPrompt = p
70 }75 }
71 }76 }
72 77
@@ -110,28 +115,32 @@ func ExportSession(srcPath, dstPath string) (string, error) {
110 return "", err115 return "", err
111 }116 }
112 117
113- // Stream every non-meta row verbatim.118+ // Stream every non-meta row verbatim. Use ReadBytes instead of
114- sc2 := bufio.NewScanner(src)119+ // bufio.Scanner: large sessions can contain very long JSONL rows
115- sc2.Buffer(make([]byte, 0, 64*1024), 20*1024*1024)120+ // (image blocks, big tool outputs, compacted history) that exceed
116- for sc2.Scan() {121+ // Scanner's token limit and fail with "token too long".
117- line := sc2.Bytes()122+ r := bufio.NewReader(src)
118- var h sessionLineHead123+ for {
119- if err := json.Unmarshal(line, &h); err != nil {124+ line, err := r.ReadBytes('\n')
120- continue125+ if len(line) > 0 {
121- }126+ line = bytes.TrimRight(line, "\r\n")
122- if h.Type == "meta" {127+ var h sessionLineHead
123- continue // already wrote a rewritten copy above128+ if err := json.Unmarshal(line, &h); err == nil && h.Type != "meta" {
129+ if _, werr := bw.Write(line); werr != nil {
130+ return "", werr
131+ }
132+ if werr := bw.WriteByte('\n'); werr != nil {
133+ return "", werr
134+ }
135+ }
124 }136 }
125- if _, err := bw.Write(line); err != nil {137+ if err == io.EOF {
126- return "", err138+ break
127 }139 }
128- if err := bw.WriteByte('\n'); err != nil {140+ if err != nil {
129- return "", err141+ return "", fmt.Errorf("export: read source: %w", err)
130 }142 }
131 }143 }
132- if err := sc2.Err(); err != nil {
133- return "", fmt.Errorf("export: read source: %w", err)
134- }
135 if err := bw.Flush(); err != nil {144 if err := bw.Flush(); err != nil {
136 return "", err145 return "", err
137 }146 }
@@ -207,29 +216,22 @@ func ImportSession(srcPath, root, cwd, version string) (string, error) {
207 return "", err216 return "", err
208 }217 }
209 218
210- // Rewind the source and stream every non-meta row.219+ // Rewind the source and stream every non-meta row. Avoid
220+ // bufio.Scanner so exported sessions with huge JSONL rows import
221+ // cleanly.
211 if _, err := src.Seek(0, io.SeekStart); err != nil {222 if _, err := src.Seek(0, io.SeekStart); err != nil {
212 return "", fmt.Errorf("import: rewind: %w", err)223 return "", fmt.Errorf("import: rewind: %w", err)
213 }224 }
214- sc2 := bufio.NewScanner(src)225+ if err := forEachJSONLLine(src, func(line []byte) error {
215- sc2.Buffer(make([]byte, 0, 64*1024), 20*1024*1024)
216- for sc2.Scan() {
217- line := sc2.Bytes()
218 var h sessionLineHead226 var h sessionLineHead
219- if err := json.Unmarshal(line, &h); err != nil {227+ if err := json.Unmarshal(line, &h); err != nil || h.Type == "meta" {
220- continue228+ return nil
221- }
222- if h.Type == "meta" {
223- continue
224 }229 }
225 if _, err := bw.Write(line); err != nil {230 if _, err := bw.Write(line); err != nil {
226- return "", err231+ return err
227 }232 }
228- if err := bw.WriteByte('\n'); err != nil {233+ return bw.WriteByte('\n')
229- return "", err234+ }); err != nil {
230- }
231- }
232- if err := sc2.Err(); err != nil {
233 return "", fmt.Errorf("import: read source: %w", err)235 return "", fmt.Errorf("import: read source: %w", err)
234 }236 }
235 if err := bw.Flush(); err != nil {237 if err := bw.Flush(); err != nil {
@@ -321,34 +323,39 @@ func BranchSession(parentPath, root, cwd, version string, upToMessageIdx int) (s
321 323
322 // Copy message rows up to the cut point, plus all usage rows324 // Copy message rows up to the cut point, plus all usage rows
323 // that land before the cut (they describe the cost of those325 // that land before the cut (they describe the cost of those
324- // messages).326+ // messages). Rewind and use the large-row-safe JSONL reader.
327+ if _, err := src.Seek(0, io.SeekStart); err != nil {
328+ return "", fmt.Errorf("branch: rewind parent: %w", err)
329+ }
325 msgCount := 0330 msgCount := 0
326- for sc.Scan() && msgCount < upToMessageIdx {331+ if err := forEachJSONLLine(src, func(line []byte) error {
327- line := sc.Bytes()332+ if msgCount >= upToMessageIdx {
333+ return io.EOF
334+ }
328 var h sessionLineHead335 var h sessionLineHead
329 if err := json.Unmarshal(line, &h); err != nil {336 if err := json.Unmarshal(line, &h); err != nil {
330- continue337+ return nil
331 }338 }
332 switch h.Type {339 switch h.Type {
333 case "message":340 case "message":
334 if _, err := bw.Write(line); err != nil {341 if _, err := bw.Write(line); err != nil {
335- return "", err342+ return err
336 }343 }
337 if err := bw.WriteByte('\n'); err != nil {344 if err := bw.WriteByte('\n'); err != nil {
338- return "", err345+ return err
339 }346 }
340 msgCount++347 msgCount++
341 case "usage":348 case "usage":
342 if _, err := bw.Write(line); err != nil {349 if _, err := bw.Write(line); err != nil {
343- return "", err350+ return err
344 }351 }
345 if err := bw.WriteByte('\n'); err != nil {352 if err := bw.WriteByte('\n'); err != nil {
346- return "", err353+ return err
347 }354 }
348 // don't increment msgCount for usage rows355 // don't increment msgCount for usage rows
349 }356 }
350- }357+ return nil
351- if err := sc.Err(); err != nil {358+ }); err != nil && err != io.EOF {
352 return "", fmt.Errorf("branch: read parent: %w", err)359 return "", fmt.Errorf("branch: read parent: %w", err)
353 }360 }
354 if err := bw.Flush(); err != nil {361 if err := bw.Flush(); err != nil {
@@ -456,43 +463,42 @@ func FindSessionByID(root, cwd, id string) string {
456 return ""463 return ""
457 }464 }
458 465
459-// firstUserPrompt scans forward from the current scanner position466+// firstUserPrompt scans forward from the current source position
460-// looking for the first user-role message and returns its text467+// looking for the first user-role message and returns its text.
461-// (trimmed, short). Used to build a humane export filename.468+// Used to build a humane export filename. Uses Reader instead of
462-func firstUserPrompt(sc *bufio.Scanner) string {469+// Scanner so a very large JSONL row before the first user prompt
463- for sc.Scan() {470+// cannot trip Scanner's token limit.
464- var line sessionLine471+func firstUserPrompt(src io.Reader) (string, error) {
465- if err := json.Unmarshal(sc.Bytes(), &line); err != nil {472+ r := bufio.NewReader(src)
466- continue473+ for {
467- }474+ lineBytes, err := r.ReadBytes('\n')
468- if line.Type != "message" || line.Message == nil || line.Message.Role != "user" {475+ if len(lineBytes) > 0 {
469- continue476+ lineBytes = bytes.TrimRight(lineBytes, "\r\n")
470- }477+ var line sessionLine
471- for _, c := range line.Message.Content {478+ if err := json.Unmarshal(lineBytes, &line); err == nil {
472- // Use type name to avoid an import of provider here beyond479+ if line.Type == "message" && line.Message != nil && line.Message.Role == "user" {
473- // the already-imported alias; TextBlock is the only content480+ b, _ := json.Marshal(line.Message)
474- // shape that yields a useful preview, so we just look for481+ var m struct {
475- // something with a reasonable string form.482+ Content []struct {
476- s := fmt.Sprintf("%v", c)483+ Text string `json:"text"`
477- _ = s // formatted value is too noisy; go straight to typed path484+ } `json:"content"`
485+ }
486+ _ = json.Unmarshal(b, &m)
487+ for _, c := range m.Content {
488+ if c.Text != "" {
489+ return c.Text, nil
490+ }
491+ }
492+ }
493+ }
478 }494 }
479- // Simpler: marshal the message and fish out the first "text"495+ if err == io.EOF {
480- // we can find. Avoids reaching into the provider package just496+ return "", nil
481- // for an interface type check.
482- b, _ := json.Marshal(line.Message)
483- var m struct {
484- Content []struct {
485- Text string `json:"text"`
486- } `json:"content"`
487 }497 }
488- _ = json.Unmarshal(b, &m)498+ if err != nil {
489- for _, c := range m.Content {499+ return "", err
490- if c.Text != "" {
491- return c.Text
492- }
493 }500 }
494 }501 }
495- return ""
496 }502 }
497 503
498 // filenameFor builds a descriptive .zotsession filename from the504 // filenameFor builds a descriptive .zotsession filename from the
M internal/core/session_portable_test.go +50 -0
@@ -1,6 +1,7 @@
11 package core
22
33 import (
4+ "encoding/json"
45 "os"
56 "path/filepath"
67 "strings"
@@ -109,6 +110,55 @@ func TestExportToFilePath(t *testing.T) {
109110
110111 // TestExportStripsCWDFromMeta verifies the exported meta no longer
111112 // carries the source user's cwd (not useful to the recipient).
113+func TestExportSessionHandlesHugeJSONLRows(t *testing.T) {
114+ root := t.TempDir()
115+ sess, err := NewSession(root, "/cwd", "anthropic", "claude-opus-4-7", "0.0.0-test")
116+ if err != nil {
117+ t.Fatal(err)
118+ }
119+ _ = sess.AppendMessage(provider.Message{
120+ Role: provider.RoleUser,
121+ Content: []provider.Content{provider.TextBlock{Text: "first prompt names export"}},
122+ })
123+ _ = sess.AppendMessage(provider.Message{
124+ Role: provider.RoleAssistant,
125+ Content: []provider.Content{provider.TextBlock{Text: strings.Repeat("x", 22*1024*1024)}},
126+ })
127+ _ = sess.Close()
128+
129+ exportDir := t.TempDir()
130+ out, err := ExportSession(sess.Path, exportDir)
131+ if err != nil {
132+ t.Fatalf("ExportSession with huge row: %v", err)
133+ }
134+ if _, err := os.Stat(out); err != nil {
135+ t.Fatalf("exported file missing: %v", err)
136+ }
137+ opened, msgs, err := OpenSession(sess.Path)
138+ if err != nil {
139+ t.Fatalf("OpenSession with huge row: %v", err)
140+ }
141+ _ = opened.Close()
142+ if len(msgs) != 2 {
143+ t.Fatalf("OpenSession messages=%d, want 2", len(msgs))
144+ }
145+ // Ensure the exported file is still readable JSONL and contains the
146+ // huge assistant message row, not just the meta/header.
147+ b, err := os.ReadFile(out)
148+ if err != nil {
149+ t.Fatal(err)
150+ }
151+ if !strings.Contains(string(b), strings.Repeat("x", 1024)) {
152+ t.Fatalf("export does not appear to contain huge assistant row")
153+ }
154+ for n, line := range strings.Split(strings.TrimSpace(string(b)), "\n") {
155+ var raw map[string]any
156+ if err := json.Unmarshal([]byte(line), &raw); err != nil {
157+ t.Fatalf("exported line %d is invalid json: %v", n+1, err)
158+ }
159+ }
160+}
161+
112162 func TestExportStripsCWDFromMeta(t *testing.T) {
113163 root := t.TempDir()
114164 sess, err := NewSession(root, "/original/cwd", "anthropic", "claude-opus-4-7", "0.0.0-test")
@@ -1,6 +1,7 @@
1 package core1 package core
2 2
3 import (3 import (
4+ "encoding/json"
4 "os"5 "os"
5 "path/filepath"6 "path/filepath"
6 "strings"7 "strings"
@@ -109,6 +110,55 @@ func TestExportToFilePath(t *testing.T) {
109 110
110 // TestExportStripsCWDFromMeta verifies the exported meta no longer111 // TestExportStripsCWDFromMeta verifies the exported meta no longer
111 // carries the source user's cwd (not useful to the recipient).112 // carries the source user's cwd (not useful to the recipient).
113+func TestExportSessionHandlesHugeJSONLRows(t *testing.T) {
114+ root := t.TempDir()
115+ sess, err := NewSession(root, "/cwd", "anthropic", "claude-opus-4-7", "0.0.0-test")
116+ if err != nil {
117+ t.Fatal(err)
118+ }
119+ _ = sess.AppendMessage(provider.Message{
120+ Role: provider.RoleUser,
121+ Content: []provider.Content{provider.TextBlock{Text: "first prompt names export"}},
122+ })
123+ _ = sess.AppendMessage(provider.Message{
124+ Role: provider.RoleAssistant,
125+ Content: []provider.Content{provider.TextBlock{Text: strings.Repeat("x", 22*1024*1024)}},
126+ })
127+ _ = sess.Close()
128+
129+ exportDir := t.TempDir()
130+ out, err := ExportSession(sess.Path, exportDir)
131+ if err != nil {
132+ t.Fatalf("ExportSession with huge row: %v", err)
133+ }
134+ if _, err := os.Stat(out); err != nil {
135+ t.Fatalf("exported file missing: %v", err)
136+ }
137+ opened, msgs, err := OpenSession(sess.Path)
138+ if err != nil {
139+ t.Fatalf("OpenSession with huge row: %v", err)
140+ }
141+ _ = opened.Close()
142+ if len(msgs) != 2 {
143+ t.Fatalf("OpenSession messages=%d, want 2", len(msgs))
144+ }
145+ // Ensure the exported file is still readable JSONL and contains the
146+ // huge assistant message row, not just the meta/header.
147+ b, err := os.ReadFile(out)
148+ if err != nil {
149+ t.Fatal(err)
150+ }
151+ if !strings.Contains(string(b), strings.Repeat("x", 1024)) {
152+ t.Fatalf("export does not appear to contain huge assistant row")
153+ }
154+ for n, line := range strings.Split(strings.TrimSpace(string(b)), "\n") {
155+ var raw map[string]any
156+ if err := json.Unmarshal([]byte(line), &raw); err != nil {
157+ t.Fatalf("exported line %d is invalid json: %v", n+1, err)
158+ }
159+ }
160+}
161+
112 func TestExportStripsCWDFromMeta(t *testing.T) {162 func TestExportStripsCWDFromMeta(t *testing.T) {
113 root := t.TempDir()163 root := t.TempDir()
114 sess, err := NewSession(root, "/original/cwd", "anthropic", "claude-opus-4-7", "0.0.0-test")164 sess, err := NewSession(root, "/original/cwd", "anthropic", "claude-opus-4-7", "0.0.0-test")

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 1

Tags 476

↑↓ moveenter openesc close