chore(callbacks): add callback ledger instrumentation
- commit
- 8e5e32bd313835a8e128ad7730db83f699ac9175
- author
- brice · · 2026-06-09 12:34 UTC
- parent
- 2dfd053
- signature
- GPG Verified
CD88F9E564E0102646FB6077A1A07C7C7C3E349D· arch-gpg Verified for the committer email
1 file · +196 -7
M syscall_unix.go +196 -7
| @@ -7,13 +7,19 @@ | |||
| 7 | 7 | package purego | |
| 8 | 8 | ||
| 9 | 9 | import ( | |
| 10 | + | "encoding/json" | |
| 10 | 11 | "errors" | |
| 12 | + | "fmt" | |
| 13 | + | "os" | |
| 11 | 14 | "reflect" | |
| 12 | 15 | "runtime" | |
| 16 | + | "runtime/debug" | |
| 17 | + | "strings" | |
| 13 | 18 | "sync" | |
| 19 | + | "time" | |
| 14 | 20 | "unsafe" | |
| 15 | 21 | ||
| 16 | - | "github.com/bnema/purego/internal/strings" | |
| 22 | + | internalstrings "github.com/bnema/purego/internal/strings" | |
| 17 | 23 | ) | |
| 18 | 24 | ||
| 19 | 25 | var syscall15XABI0 uintptr | |
| @@ -60,7 +66,10 @@ func NewCallbackFnPtr(fnPtr any) uintptr { | |||
| 60 | 66 | if val.Kind() != reflect.Ptr || val.Elem().Kind() != reflect.Func { | |
| 61 | 67 | panic("purego: the type must be a function pointer but was not") | |
| 62 | 68 | } | |
| 63 | - | if addr, ok := getCallbackByFnPtr(val); ok { | |
| 69 | + | if addr, idx, ok := getCallbackByFnPtr(val); ok { | |
| 70 | + | cbs.lock.Lock() | |
| 71 | + | recordCallbackLedger("dedupe", idx, addr, val.Pointer(), val.Elem(), false, "") | |
| 72 | + | cbs.lock.Unlock() | |
| 64 | 73 | return addr | |
| 65 | 74 | } | |
| 66 | 75 | addr := compileCallback(val.Elem().Interface()) | |
| @@ -68,6 +77,7 @@ func NewCallbackFnPtr(fnPtr any) uintptr { | |||
| 68 | 77 | cbs.knownFnPtr[val.Pointer()] = addr | |
| 69 | 78 | if idx, ok := cbs.knownIdx[addr]; ok { | |
| 70 | 79 | cbs.fnPtrKeys[idx] = val.Pointer() | |
| 80 | + | recordCallbackLedger("fnptr-bind", idx, addr, val.Pointer(), val.Elem(), false, "") | |
| 71 | 81 | } | |
| 72 | 82 | cbs.lock.Unlock() | |
| 73 | 83 | return addr | |
| @@ -79,9 +89,12 @@ func UnrefCallback(cb uintptr) error { | |||
| 79 | 89 | defer cbs.lock.Unlock() | |
| 80 | 90 | idx, ok := cbs.knownIdx[cb] | |
| 81 | 91 | if !ok { | |
| 92 | + | recordCallbackLedger("release-miss", -1, cb, 0, reflect.Value{}, false, "callback not found") | |
| 82 | 93 | return errors.New("callback not found") | |
| 83 | 94 | } | |
| 84 | - | if key := cbs.fnPtrKeys[idx]; key != 0 { | |
| 95 | + | val := cbs.funcs[idx] | |
| 96 | + | key := cbs.fnPtrKeys[idx] | |
| 97 | + | if key != 0 { | |
| 85 | 98 | delete(cbs.knownFnPtr, key) | |
| 86 | 99 | cbs.fnPtrKeys[idx] = 0 | |
| 87 | 100 | } | |
| @@ -89,6 +102,7 @@ func UnrefCallback(cb uintptr) error { | |||
| 89 | 102 | cbs.holes[idx] = struct{}{} | |
| 90 | 103 | cbs.funcs[idx] = reflect.Value{} | |
| 91 | 104 | cbs.argPools[idx] = nil | |
| 105 | + | recordCallbackLedger("release", idx, cb, key, val, false, "") | |
| 92 | 106 | return nil | |
| 93 | 107 | } | |
| 94 | 108 | ||
| @@ -105,19 +119,23 @@ func UnrefCallbackFnPtr(fnPtr any) error { | |||
| 105 | 119 | defer cbs.lock.Unlock() | |
| 106 | 120 | addr, ok := cbs.knownFnPtr[val.Pointer()] | |
| 107 | 121 | if !ok { | |
| 122 | + | recordCallbackLedger("release-fnptr-miss", -1, 0, val.Pointer(), val.Elem(), false, "callback not found") | |
| 108 | 123 | return errors.New("callback not found") | |
| 109 | 124 | } | |
| 110 | 125 | idx, ok := cbs.knownIdx[addr] | |
| 111 | 126 | if !ok { | |
| 112 | 127 | delete(cbs.knownFnPtr, val.Pointer()) | |
| 128 | + | recordCallbackLedger("release-fnptr-miss", -1, addr, val.Pointer(), val.Elem(), false, "callback not found") | |
| 113 | 129 | return errors.New("callback not found") | |
| 114 | 130 | } | |
| 131 | + | callbackVal := cbs.funcs[idx] | |
| 115 | 132 | delete(cbs.knownFnPtr, val.Pointer()) | |
| 116 | 133 | delete(cbs.knownIdx, addr) | |
| 117 | 134 | cbs.fnPtrKeys[idx] = 0 | |
| 118 | 135 | cbs.holes[idx] = struct{}{} | |
| 119 | 136 | cbs.funcs[idx] = reflect.Value{} | |
| 120 | 137 | cbs.argPools[idx] = nil | |
| 138 | + | recordCallbackLedger("release-fnptr", idx, addr, val.Pointer(), callbackVal, false, "") | |
| 121 | 139 | return nil | |
| 122 | 140 | } | |
| 123 | 141 | ||
| @@ -134,17 +152,175 @@ var cbs = struct { | |||
| 134 | 152 | knownIdx map[uintptr]int // callback address -> slot index | |
| 135 | 153 | knownFnPtr map[uintptr]uintptr // function pointer variable address -> callback address | |
| 136 | 154 | fnPtrKeys [maxCB]uintptr // slot index -> function pointer variable address | |
| 155 | + | ledgerSeq uint64 // monotonic callback ledger event sequence | |
| 137 | 156 | }{ | |
| 138 | 157 | holes: make(map[int]struct{}), | |
| 139 | 158 | knownIdx: make(map[uintptr]int, maxCB), | |
| 140 | 159 | knownFnPtr: make(map[uintptr]uintptr, maxCB), | |
| 141 | 160 | } | |
| 142 | 161 | ||
| 143 | - | func getCallbackByFnPtr(val reflect.Value) (uintptr, bool) { | |
| 162 | + | func getCallbackByFnPtr(val reflect.Value) (uintptr, int, bool) { | |
| 144 | 163 | cbs.lock.RLock() | |
| 145 | 164 | defer cbs.lock.RUnlock() | |
| 146 | 165 | addr, ok := cbs.knownFnPtr[val.Pointer()] | |
| 147 | - | return addr, ok | |
| 166 | + | if !ok { | |
| 167 | + | return 0, -1, false | |
| 168 | + | } | |
| 169 | + | idx, ok := cbs.knownIdx[addr] | |
| 170 | + | if !ok { | |
| 171 | + | return addr, -1, true | |
| 172 | + | } | |
| 173 | + | return addr, idx, true | |
| 174 | + | } | |
| 175 | + | ||
| 176 | + | type callbackLedgerEvent struct { | |
| 177 | + | Marker string `json:"marker"` | |
| 178 | + | PID int `json:"pid"` | |
| 179 | + | Seq uint64 `json:"seq"` | |
| 180 | + | Time string `json:"time"` | |
| 181 | + | Event string `json:"event"` | |
| 182 | + | Index int `json:"index"` | |
| 183 | + | Addr string `json:"addr,omitempty"` | |
| 184 | + | FnPtrKey string `json:"fn_ptr_key,omitempty"` | |
| 185 | + | Type string `json:"type,omitempty"` | |
| 186 | + | Family string `json:"family"` | |
| 187 | + | StackKey string `json:"stack_key"` | |
| 188 | + | NumFn int `json:"num_fn"` | |
| 189 | + | Occupied int `json:"occupied"` | |
| 190 | + | Reusable int `json:"reusable"` | |
| 191 | + | KnownFnPtr int `json:"known_fn_ptr"` | |
| 192 | + | Remaining int `json:"remaining"` | |
| 193 | + | Max int `json:"max"` | |
| 194 | + | ReusedSlot bool `json:"reused_slot,omitempty"` | |
| 195 | + | Stack string `json:"stack,omitempty"` | |
| 196 | + | Error string `json:"error,omitempty"` | |
| 197 | + | } | |
| 198 | + | ||
| 199 | + | func callbackLedgerEnabled() bool { | |
| 200 | + | return os.Getenv("PUREGO_CALLBACK_LEDGER") != "0" | |
| 201 | + | } | |
| 202 | + | ||
| 203 | + | func callbackLedgerPath() string { | |
| 204 | + | if path := os.Getenv("PUREGO_CALLBACK_LEDGER_FILE"); path != "" { | |
| 205 | + | return path | |
| 206 | + | } | |
| 207 | + | return fmt.Sprintf("/tmp/purego-callback-ledger-%d.jsonl", os.Getpid()) | |
| 208 | + | } | |
| 209 | + | ||
| 210 | + | func typeName(val reflect.Value) string { | |
| 211 | + | if !val.IsValid() { | |
| 212 | + | return "" | |
| 213 | + | } | |
| 214 | + | return val.Type().String() | |
| 215 | + | } | |
| 216 | + | ||
| 217 | + | func hexUintptr(v uintptr) string { | |
| 218 | + | if v == 0 { | |
| 219 | + | return "" | |
| 220 | + | } | |
| 221 | + | return fmt.Sprintf("0x%x", v) | |
| 222 | + | } | |
| 223 | + | ||
| 224 | + | func classifyCallbackStack(stack string) (family, stackKey string) { | |
| 225 | + | family = "unknown" | |
| 226 | + | stackKey = "unknown" | |
| 227 | + | lines := strings.Split(stack, "\n") | |
| 228 | + | for i, line := range lines { | |
| 229 | + | if line == "" || strings.HasPrefix(line, "goroutine ") || strings.Contains(line, "runtime/debug.Stack") || strings.Contains(line, "recordCallbackLedger") || strings.Contains(line, "traceCallbackAllocation") { | |
| 230 | + | continue | |
| 231 | + | } | |
| 232 | + | if strings.Contains(line, "github.com/bnema/purego.") || strings.Contains(line, "github.com/ebitengine/purego.") { | |
| 233 | + | continue | |
| 234 | + | } | |
| 235 | + | if strings.HasPrefix(strings.TrimSpace(line), "/") || strings.HasPrefix(strings.TrimSpace(line), "../") { | |
| 236 | + | continue | |
| 237 | + | } | |
| 238 | + | stackKey = strings.TrimSpace(line) | |
| 239 | + | for j := i + 1; j < len(lines); j++ { | |
| 240 | + | fileLine := strings.TrimSpace(lines[j]) | |
| 241 | + | if fileLine == "" { | |
| 242 | + | continue | |
| 243 | + | } | |
| 244 | + | family = classifyCallbackFamily(fileLine + " " + stackKey) | |
| 245 | + | return family, stackKey | |
| 246 | + | } | |
| 247 | + | family = classifyCallbackFamily(stackKey) | |
| 248 | + | return family, stackKey | |
| 249 | + | } | |
| 250 | + | return family, stackKey | |
| 251 | + | } | |
| 252 | + | ||
| 253 | + | func classifyCallbackFamily(s string) string { | |
| 254 | + | switch { | |
| 255 | + | case strings.Contains(s, "purego-cef2gtk"): | |
| 256 | + | return "purego-cef2gtk" | |
| 257 | + | case strings.Contains(s, "purego-cef"): | |
| 258 | + | return "purego-cef" | |
| 259 | + | case strings.Contains(s, "puregotk"): | |
| 260 | + | return "puregotk" | |
| 261 | + | case strings.Contains(s, "dumber"): | |
| 262 | + | return "dumber" | |
| 263 | + | case strings.Contains(s, "github.com/bnema/purego") || strings.Contains(s, "/purego/"): | |
| 264 | + | return "purego" | |
| 265 | + | default: | |
| 266 | + | return "unknown" | |
| 267 | + | } | |
| 268 | + | } | |
| 269 | + | ||
| 270 | + | func recordCallbackLedger(event string, idx int, addr uintptr, fnPtrKey uintptr, val reflect.Value, reused bool, errText string) { | |
| 271 | + | if !callbackLedgerEnabled() { | |
| 272 | + | return | |
| 273 | + | } | |
| 274 | + | stack := string(debug.Stack()) | |
| 275 | + | family, stackKey := classifyCallbackStack(stack) | |
| 276 | + | cbs.ledgerSeq++ | |
| 277 | + | occupied := cbs.numFn - len(cbs.holes) | |
| 278 | + | entry := callbackLedgerEvent{ | |
| 279 | + | Marker: "PUREGO-CALLBACK-LEDGER", | |
| 280 | + | PID: os.Getpid(), | |
| 281 | + | Seq: cbs.ledgerSeq, | |
| 282 | + | Time: time.Now().Format(time.RFC3339Nano), | |
| 283 | + | Event: event, | |
| 284 | + | Index: idx, | |
| 285 | + | Addr: hexUintptr(addr), | |
| 286 | + | FnPtrKey: hexUintptr(fnPtrKey), | |
| 287 | + | Type: typeName(val), | |
| 288 | + | Family: family, | |
| 289 | + | StackKey: stackKey, | |
| 290 | + | NumFn: cbs.numFn, | |
| 291 | + | Occupied: occupied, | |
| 292 | + | Reusable: len(cbs.holes), | |
| 293 | + | KnownFnPtr: len(cbs.knownFnPtr), | |
| 294 | + | Remaining: maxCB - occupied, | |
| 295 | + | Max: maxCB, | |
| 296 | + | ReusedSlot: reused, | |
| 297 | + | Error: errText, | |
| 298 | + | } | |
| 299 | + | if os.Getenv("PUREGO_CALLBACK_LEDGER_STACK") != "0" { | |
| 300 | + | entry.Stack = stack | |
| 301 | + | } | |
| 302 | + | payload, err := json.Marshal(entry) | |
| 303 | + | if err != nil { | |
| 304 | + | return | |
| 305 | + | } | |
| 306 | + | payload = append(payload, '\n') | |
| 307 | + | if f, err := os.OpenFile(callbackLedgerPath(), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600); err == nil { | |
| 308 | + | _, _ = f.Write(payload) | |
| 309 | + | _ = f.Close() | |
| 310 | + | } | |
| 311 | + | } | |
| 312 | + | ||
| 313 | + | func traceCallbackAllocation(event string, val reflect.Value, remaining int) { | |
| 314 | + | message := fmt.Sprintf("PUREGO-CALLBACK-TRACE event=%s remaining=%d live=%d type=%s\n%s\n", event, remaining, maxCB-remaining, val.Type(), debug.Stack()) | |
| 315 | + | fmt.Fprint(os.Stderr, message) | |
| 316 | + | traceFile := os.Getenv("PUREGO_CALLBACK_TRACE_FILE") | |
| 317 | + | if traceFile == "" { | |
| 318 | + | traceFile = fmt.Sprintf("/tmp/purego-callback-trace-%d.log", os.Getpid()) | |
| 319 | + | } | |
| 320 | + | if f, err := os.OpenFile(traceFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600); err == nil { | |
| 321 | + | _, _ = f.WriteString(message) | |
| 322 | + | _ = f.Close() | |
| 323 | + | } | |
| 148 | 324 | } | |
| 149 | 325 | ||
| 150 | 326 | func compileCallback(fn any) uintptr { | |
| @@ -188,15 +364,23 @@ output: | |||
| 188 | 364 | cbs.lock.Lock() | |
| 189 | 365 | defer cbs.lock.Unlock() | |
| 190 | 366 | index := -1 | |
| 367 | + | reused := false | |
| 191 | 368 | for i := range cbs.holes { | |
| 192 | 369 | index = i | |
| 193 | 370 | delete(cbs.holes, i) | |
| 371 | + | reused = true | |
| 194 | 372 | break | |
| 195 | 373 | } | |
| 196 | 374 | if index < 0 { | |
| 197 | - | if cbs.numFn >= maxCB { | |
| 375 | + | remaining := maxCB - cbs.numFn | |
| 376 | + | if remaining <= 0 { | |
| 377 | + | recordCallbackLedger("exhausted", -1, 0, 0, val, false, "maximum callbacks reached") | |
| 378 | + | traceCallbackAllocation("exhausted", val, 0) | |
| 198 | 379 | panic("purego: the maximum number of callbacks has been reached") | |
| 199 | 380 | } | |
| 381 | + | if remaining <= 100 || remaining%250 == 0 { | |
| 382 | + | traceCallbackAllocation("allocated", val, remaining) | |
| 383 | + | } | |
| 200 | 384 | index = cbs.numFn | |
| 201 | 385 | cbs.numFn++ | |
| 202 | 386 | } | |
| @@ -209,6 +393,11 @@ output: | |||
| 209 | 393 | } | |
| 210 | 394 | addr := callbackasmAddr(index) | |
| 211 | 395 | cbs.knownIdx[addr] = index | |
| 396 | + | if reused { | |
| 397 | + | recordCallbackLedger("reuse", index, addr, 0, val, true, "") | |
| 398 | + | } else { | |
| 399 | + | recordCallbackLedger("alloc", index, addr, 0, val, false, "") | |
| 400 | + | } | |
| 212 | 401 | return addr | |
| 213 | 402 | } | |
| 214 | 403 | ||
| @@ -329,7 +518,7 @@ func callbackWrap(a *callbackArgs) { | |||
| 329 | 518 | ptr = frame[pos] | |
| 330 | 519 | } | |
| 331 | 520 | intsN += slots | |
| 332 | - | args[i] = reflect.ValueOf(strings.GoString(ptr)) | |
| 521 | + | args[i] = reflect.ValueOf(internalstrings.GoString(ptr)) | |
| 333 | 522 | continue | |
| 334 | 523 | case reflect.Struct: | |
| 335 | 524 | if i == 0 && inType.AssignableTo(reflect.TypeOf(CDecl{})) { | |
| @@ -7,13 +7,19 @@ | |||||
| 7 | package purego | 7 | package purego | ||
| 8 | 8 | ||||
| 9 | import ( | 9 | import ( | ||
| 10 | + | "encoding/json" | |||
| 10 | "errors" | 11 | "errors" | ||
| 12 | + | "fmt" | |||
| 13 | + | "os" | |||
| 11 | "reflect" | 14 | "reflect" | ||
| 12 | "runtime" | 15 | "runtime" | ||
| 16 | + | "runtime/debug" | |||
| 17 | + | "strings" | |||
| 13 | "sync" | 18 | "sync" | ||
| 19 | + | "time" | |||
| 14 | "unsafe" | 20 | "unsafe" | ||
| 15 | 21 | ||||
| 16 | - | "github.com/bnema/purego/internal/strings" | 22 | + | internalstrings "github.com/bnema/purego/internal/strings" |
| 17 | ) | 23 | ) | ||
| 18 | 24 | ||||
| 19 | var syscall15XABI0 uintptr | 25 | var syscall15XABI0 uintptr | ||
| @@ -60,7 +66,10 @@ func NewCallbackFnPtr(fnPtr any) uintptr { | |||||
| 60 | if val.Kind() != reflect.Ptr || val.Elem().Kind() != reflect.Func { | 66 | if val.Kind() != reflect.Ptr || val.Elem().Kind() != reflect.Func { | ||
| 61 | panic("purego: the type must be a function pointer but was not") | 67 | panic("purego: the type must be a function pointer but was not") | ||
| 62 | } | 68 | } | ||
| 63 | - | if addr, ok := getCallbackByFnPtr(val); ok { | 69 | + | if addr, idx, ok := getCallbackByFnPtr(val); ok { |
| 70 | + | cbs.lock.Lock() | |||
| 71 | + | recordCallbackLedger("dedupe", idx, addr, val.Pointer(), val.Elem(), false, "") | |||
| 72 | + | cbs.lock.Unlock() | |||
| 64 | return addr | 73 | return addr | ||
| 65 | } | 74 | } | ||
| 66 | addr := compileCallback(val.Elem().Interface()) | 75 | addr := compileCallback(val.Elem().Interface()) | ||
| @@ -68,6 +77,7 @@ func NewCallbackFnPtr(fnPtr any) uintptr { | |||||
| 68 | cbs.knownFnPtr[val.Pointer()] = addr | 77 | cbs.knownFnPtr[val.Pointer()] = addr | ||
| 69 | if idx, ok := cbs.knownIdx[addr]; ok { | 78 | if idx, ok := cbs.knownIdx[addr]; ok { | ||
| 70 | cbs.fnPtrKeys[idx] = val.Pointer() | 79 | cbs.fnPtrKeys[idx] = val.Pointer() | ||
| 80 | + | recordCallbackLedger("fnptr-bind", idx, addr, val.Pointer(), val.Elem(), false, "") | |||
| 71 | } | 81 | } | ||
| 72 | cbs.lock.Unlock() | 82 | cbs.lock.Unlock() | ||
| 73 | return addr | 83 | return addr | ||
| @@ -79,9 +89,12 @@ func UnrefCallback(cb uintptr) error { | |||||
| 79 | defer cbs.lock.Unlock() | 89 | defer cbs.lock.Unlock() | ||
| 80 | idx, ok := cbs.knownIdx[cb] | 90 | idx, ok := cbs.knownIdx[cb] | ||
| 81 | if !ok { | 91 | if !ok { | ||
| 92 | + | recordCallbackLedger("release-miss", -1, cb, 0, reflect.Value{}, false, "callback not found") | |||
| 82 | return errors.New("callback not found") | 93 | return errors.New("callback not found") | ||
| 83 | } | 94 | } | ||
| 84 | - | if key := cbs.fnPtrKeys[idx]; key != 0 { | 95 | + | val := cbs.funcs[idx] |
| 96 | + | key := cbs.fnPtrKeys[idx] | |||
| 97 | + | if key != 0 { | |||
| 85 | delete(cbs.knownFnPtr, key) | 98 | delete(cbs.knownFnPtr, key) | ||
| 86 | cbs.fnPtrKeys[idx] = 0 | 99 | cbs.fnPtrKeys[idx] = 0 | ||
| 87 | } | 100 | } | ||
| @@ -89,6 +102,7 @@ func UnrefCallback(cb uintptr) error { | |||||
| 89 | cbs.holes[idx] = struct{}{} | 102 | cbs.holes[idx] = struct{}{} | ||
| 90 | cbs.funcs[idx] = reflect.Value{} | 103 | cbs.funcs[idx] = reflect.Value{} | ||
| 91 | cbs.argPools[idx] = nil | 104 | cbs.argPools[idx] = nil | ||
| 105 | + | recordCallbackLedger("release", idx, cb, key, val, false, "") | |||
| 92 | return nil | 106 | return nil | ||
| 93 | } | 107 | } | ||
| 94 | 108 | ||||
| @@ -105,19 +119,23 @@ func UnrefCallbackFnPtr(fnPtr any) error { | |||||
| 105 | defer cbs.lock.Unlock() | 119 | defer cbs.lock.Unlock() | ||
| 106 | addr, ok := cbs.knownFnPtr[val.Pointer()] | 120 | addr, ok := cbs.knownFnPtr[val.Pointer()] | ||
| 107 | if !ok { | 121 | if !ok { | ||
| 122 | + | recordCallbackLedger("release-fnptr-miss", -1, 0, val.Pointer(), val.Elem(), false, "callback not found") | |||
| 108 | return errors.New("callback not found") | 123 | return errors.New("callback not found") | ||
| 109 | } | 124 | } | ||
| 110 | idx, ok := cbs.knownIdx[addr] | 125 | idx, ok := cbs.knownIdx[addr] | ||
| 111 | if !ok { | 126 | if !ok { | ||
| 112 | delete(cbs.knownFnPtr, val.Pointer()) | 127 | delete(cbs.knownFnPtr, val.Pointer()) | ||
| 128 | + | recordCallbackLedger("release-fnptr-miss", -1, addr, val.Pointer(), val.Elem(), false, "callback not found") | |||
| 113 | return errors.New("callback not found") | 129 | return errors.New("callback not found") | ||
| 114 | } | 130 | } | ||
| 131 | + | callbackVal := cbs.funcs[idx] | |||
| 115 | delete(cbs.knownFnPtr, val.Pointer()) | 132 | delete(cbs.knownFnPtr, val.Pointer()) | ||
| 116 | delete(cbs.knownIdx, addr) | 133 | delete(cbs.knownIdx, addr) | ||
| 117 | cbs.fnPtrKeys[idx] = 0 | 134 | cbs.fnPtrKeys[idx] = 0 | ||
| 118 | cbs.holes[idx] = struct{}{} | 135 | cbs.holes[idx] = struct{}{} | ||
| 119 | cbs.funcs[idx] = reflect.Value{} | 136 | cbs.funcs[idx] = reflect.Value{} | ||
| 120 | cbs.argPools[idx] = nil | 137 | cbs.argPools[idx] = nil | ||
| 138 | + | recordCallbackLedger("release-fnptr", idx, addr, val.Pointer(), callbackVal, false, "") | |||
| 121 | return nil | 139 | return nil | ||
| 122 | } | 140 | } | ||
| 123 | 141 | ||||
| @@ -134,17 +152,175 @@ var cbs = struct { | |||||
| 134 | knownIdx map[uintptr]int // callback address -> slot index | 152 | knownIdx map[uintptr]int // callback address -> slot index | ||
| 135 | knownFnPtr map[uintptr]uintptr // function pointer variable address -> callback address | 153 | knownFnPtr map[uintptr]uintptr // function pointer variable address -> callback address | ||
| 136 | fnPtrKeys [maxCB]uintptr // slot index -> function pointer variable address | 154 | fnPtrKeys [maxCB]uintptr // slot index -> function pointer variable address | ||
| 155 | + | ledgerSeq uint64 // monotonic callback ledger event sequence | |||
| 137 | }{ | 156 | }{ | ||
| 138 | holes: make(map[int]struct{}), | 157 | holes: make(map[int]struct{}), | ||
| 139 | knownIdx: make(map[uintptr]int, maxCB), | 158 | knownIdx: make(map[uintptr]int, maxCB), | ||
| 140 | knownFnPtr: make(map[uintptr]uintptr, maxCB), | 159 | knownFnPtr: make(map[uintptr]uintptr, maxCB), | ||
| 141 | } | 160 | } | ||
| 142 | 161 | ||||
| 143 | - | func getCallbackByFnPtr(val reflect.Value) (uintptr, bool) { | 162 | + | func getCallbackByFnPtr(val reflect.Value) (uintptr, int, bool) { |
| 144 | cbs.lock.RLock() | 163 | cbs.lock.RLock() | ||
| 145 | defer cbs.lock.RUnlock() | 164 | defer cbs.lock.RUnlock() | ||
| 146 | addr, ok := cbs.knownFnPtr[val.Pointer()] | 165 | addr, ok := cbs.knownFnPtr[val.Pointer()] | ||
| 147 | - | return addr, ok | 166 | + | if !ok { |
| 167 | + | return 0, -1, false | |||
| 168 | + | } | |||
| 169 | + | idx, ok := cbs.knownIdx[addr] | |||
| 170 | + | if !ok { | |||
| 171 | + | return addr, -1, true | |||
| 172 | + | } | |||
| 173 | + | return addr, idx, true | |||
| 174 | + | } | |||
| 175 | + | ||||
| 176 | + | type callbackLedgerEvent struct { | |||
| 177 | + | Marker string `json:"marker"` | |||
| 178 | + | PID int `json:"pid"` | |||
| 179 | + | Seq uint64 `json:"seq"` | |||
| 180 | + | Time string `json:"time"` | |||
| 181 | + | Event string `json:"event"` | |||
| 182 | + | Index int `json:"index"` | |||
| 183 | + | Addr string `json:"addr,omitempty"` | |||
| 184 | + | FnPtrKey string `json:"fn_ptr_key,omitempty"` | |||
| 185 | + | Type string `json:"type,omitempty"` | |||
| 186 | + | Family string `json:"family"` | |||
| 187 | + | StackKey string `json:"stack_key"` | |||
| 188 | + | NumFn int `json:"num_fn"` | |||
| 189 | + | Occupied int `json:"occupied"` | |||
| 190 | + | Reusable int `json:"reusable"` | |||
| 191 | + | KnownFnPtr int `json:"known_fn_ptr"` | |||
| 192 | + | Remaining int `json:"remaining"` | |||
| 193 | + | Max int `json:"max"` | |||
| 194 | + | ReusedSlot bool `json:"reused_slot,omitempty"` | |||
| 195 | + | Stack string `json:"stack,omitempty"` | |||
| 196 | + | Error string `json:"error,omitempty"` | |||
| 197 | + | } | |||
| 198 | + | ||||
| 199 | + | func callbackLedgerEnabled() bool { | |||
| 200 | + | return os.Getenv("PUREGO_CALLBACK_LEDGER") != "0" | |||
| 201 | + | } | |||
| 202 | + | ||||
| 203 | + | func callbackLedgerPath() string { | |||
| 204 | + | if path := os.Getenv("PUREGO_CALLBACK_LEDGER_FILE"); path != "" { | |||
| 205 | + | return path | |||
| 206 | + | } | |||
| 207 | + | return fmt.Sprintf("/tmp/purego-callback-ledger-%d.jsonl", os.Getpid()) | |||
| 208 | + | } | |||
| 209 | + | ||||
| 210 | + | func typeName(val reflect.Value) string { | |||
| 211 | + | if !val.IsValid() { | |||
| 212 | + | return "" | |||
| 213 | + | } | |||
| 214 | + | return val.Type().String() | |||
| 215 | + | } | |||
| 216 | + | ||||
| 217 | + | func hexUintptr(v uintptr) string { | |||
| 218 | + | if v == 0 { | |||
| 219 | + | return "" | |||
| 220 | + | } | |||
| 221 | + | return fmt.Sprintf("0x%x", v) | |||
| 222 | + | } | |||
| 223 | + | ||||
| 224 | + | func classifyCallbackStack(stack string) (family, stackKey string) { | |||
| 225 | + | family = "unknown" | |||
| 226 | + | stackKey = "unknown" | |||
| 227 | + | lines := strings.Split(stack, "\n") | |||
| 228 | + | for i, line := range lines { | |||
| 229 | + | if line == "" || strings.HasPrefix(line, "goroutine ") || strings.Contains(line, "runtime/debug.Stack") || strings.Contains(line, "recordCallbackLedger") || strings.Contains(line, "traceCallbackAllocation") { | |||
| 230 | + | continue | |||
| 231 | + | } | |||
| 232 | + | if strings.Contains(line, "github.com/bnema/purego.") || strings.Contains(line, "github.com/ebitengine/purego.") { | |||
| 233 | + | continue | |||
| 234 | + | } | |||
| 235 | + | if strings.HasPrefix(strings.TrimSpace(line), "/") || strings.HasPrefix(strings.TrimSpace(line), "../") { | |||
| 236 | + | continue | |||
| 237 | + | } | |||
| 238 | + | stackKey = strings.TrimSpace(line) | |||
| 239 | + | for j := i + 1; j < len(lines); j++ { | |||
| 240 | + | fileLine := strings.TrimSpace(lines[j]) | |||
| 241 | + | if fileLine == "" { | |||
| 242 | + | continue | |||
| 243 | + | } | |||
| 244 | + | family = classifyCallbackFamily(fileLine + " " + stackKey) | |||
| 245 | + | return family, stackKey | |||
| 246 | + | } | |||
| 247 | + | family = classifyCallbackFamily(stackKey) | |||
| 248 | + | return family, stackKey | |||
| 249 | + | } | |||
| 250 | + | return family, stackKey | |||
| 251 | + | } | |||
| 252 | + | ||||
| 253 | + | func classifyCallbackFamily(s string) string { | |||
| 254 | + | switch { | |||
| 255 | + | case strings.Contains(s, "purego-cef2gtk"): | |||
| 256 | + | return "purego-cef2gtk" | |||
| 257 | + | case strings.Contains(s, "purego-cef"): | |||
| 258 | + | return "purego-cef" | |||
| 259 | + | case strings.Contains(s, "puregotk"): | |||
| 260 | + | return "puregotk" | |||
| 261 | + | case strings.Contains(s, "dumber"): | |||
| 262 | + | return "dumber" | |||
| 263 | + | case strings.Contains(s, "github.com/bnema/purego") || strings.Contains(s, "/purego/"): | |||
| 264 | + | return "purego" | |||
| 265 | + | default: | |||
| 266 | + | return "unknown" | |||
| 267 | + | } | |||
| 268 | + | } | |||
| 269 | + | ||||
| 270 | + | func recordCallbackLedger(event string, idx int, addr uintptr, fnPtrKey uintptr, val reflect.Value, reused bool, errText string) { | |||
| 271 | + | if !callbackLedgerEnabled() { | |||
| 272 | + | return | |||
| 273 | + | } | |||
| 274 | + | stack := string(debug.Stack()) | |||
| 275 | + | family, stackKey := classifyCallbackStack(stack) | |||
| 276 | + | cbs.ledgerSeq++ | |||
| 277 | + | occupied := cbs.numFn - len(cbs.holes) | |||
| 278 | + | entry := callbackLedgerEvent{ | |||
| 279 | + | Marker: "PUREGO-CALLBACK-LEDGER", | |||
| 280 | + | PID: os.Getpid(), | |||
| 281 | + | Seq: cbs.ledgerSeq, | |||
| 282 | + | Time: time.Now().Format(time.RFC3339Nano), | |||
| 283 | + | Event: event, | |||
| 284 | + | Index: idx, | |||
| 285 | + | Addr: hexUintptr(addr), | |||
| 286 | + | FnPtrKey: hexUintptr(fnPtrKey), | |||
| 287 | + | Type: typeName(val), | |||
| 288 | + | Family: family, | |||
| 289 | + | StackKey: stackKey, | |||
| 290 | + | NumFn: cbs.numFn, | |||
| 291 | + | Occupied: occupied, | |||
| 292 | + | Reusable: len(cbs.holes), | |||
| 293 | + | KnownFnPtr: len(cbs.knownFnPtr), | |||
| 294 | + | Remaining: maxCB - occupied, | |||
| 295 | + | Max: maxCB, | |||
| 296 | + | ReusedSlot: reused, | |||
| 297 | + | Error: errText, | |||
| 298 | + | } | |||
| 299 | + | if os.Getenv("PUREGO_CALLBACK_LEDGER_STACK") != "0" { | |||
| 300 | + | entry.Stack = stack | |||
| 301 | + | } | |||
| 302 | + | payload, err := json.Marshal(entry) | |||
| 303 | + | if err != nil { | |||
| 304 | + | return | |||
| 305 | + | } | |||
| 306 | + | payload = append(payload, '\n') | |||
| 307 | + | if f, err := os.OpenFile(callbackLedgerPath(), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600); err == nil { | |||
| 308 | + | _, _ = f.Write(payload) | |||
| 309 | + | _ = f.Close() | |||
| 310 | + | } | |||
| 311 | + | } | |||
| 312 | + | ||||
| 313 | + | func traceCallbackAllocation(event string, val reflect.Value, remaining int) { | |||
| 314 | + | message := fmt.Sprintf("PUREGO-CALLBACK-TRACE event=%s remaining=%d live=%d type=%s\n%s\n", event, remaining, maxCB-remaining, val.Type(), debug.Stack()) | |||
| 315 | + | fmt.Fprint(os.Stderr, message) | |||
| 316 | + | traceFile := os.Getenv("PUREGO_CALLBACK_TRACE_FILE") | |||
| 317 | + | if traceFile == "" { | |||
| 318 | + | traceFile = fmt.Sprintf("/tmp/purego-callback-trace-%d.log", os.Getpid()) | |||
| 319 | + | } | |||
| 320 | + | if f, err := os.OpenFile(traceFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600); err == nil { | |||
| 321 | + | _, _ = f.WriteString(message) | |||
| 322 | + | _ = f.Close() | |||
| 323 | + | } | |||
| 148 | } | 324 | } | ||
| 149 | 325 | ||||
| 150 | func compileCallback(fn any) uintptr { | 326 | func compileCallback(fn any) uintptr { | ||
| @@ -188,15 +364,23 @@ output: | |||||
| 188 | cbs.lock.Lock() | 364 | cbs.lock.Lock() | ||
| 189 | defer cbs.lock.Unlock() | 365 | defer cbs.lock.Unlock() | ||
| 190 | index := -1 | 366 | index := -1 | ||
| 367 | + | reused := false | |||
| 191 | for i := range cbs.holes { | 368 | for i := range cbs.holes { | ||
| 192 | index = i | 369 | index = i | ||
| 193 | delete(cbs.holes, i) | 370 | delete(cbs.holes, i) | ||
| 371 | + | reused = true | |||
| 194 | break | 372 | break | ||
| 195 | } | 373 | } | ||
| 196 | if index < 0 { | 374 | if index < 0 { | ||
| 197 | - | if cbs.numFn >= maxCB { | 375 | + | remaining := maxCB - cbs.numFn |
| 376 | + | if remaining <= 0 { | |||
| 377 | + | recordCallbackLedger("exhausted", -1, 0, 0, val, false, "maximum callbacks reached") | |||
| 378 | + | traceCallbackAllocation("exhausted", val, 0) | |||
| 198 | panic("purego: the maximum number of callbacks has been reached") | 379 | panic("purego: the maximum number of callbacks has been reached") | ||
| 199 | } | 380 | } | ||
| 381 | + | if remaining <= 100 || remaining%250 == 0 { | |||
| 382 | + | traceCallbackAllocation("allocated", val, remaining) | |||
| 383 | + | } | |||
| 200 | index = cbs.numFn | 384 | index = cbs.numFn | ||
| 201 | cbs.numFn++ | 385 | cbs.numFn++ | ||
| 202 | } | 386 | } | ||
| @@ -209,6 +393,11 @@ output: | |||||
| 209 | } | 393 | } | ||
| 210 | addr := callbackasmAddr(index) | 394 | addr := callbackasmAddr(index) | ||
| 211 | cbs.knownIdx[addr] = index | 395 | cbs.knownIdx[addr] = index | ||
| 396 | + | if reused { | |||
| 397 | + | recordCallbackLedger("reuse", index, addr, 0, val, true, "") | |||
| 398 | + | } else { | |||
| 399 | + | recordCallbackLedger("alloc", index, addr, 0, val, false, "") | |||
| 400 | + | } | |||
| 212 | return addr | 401 | return addr | ||
| 213 | } | 402 | } | ||
| 214 | 403 | ||||
| @@ -329,7 +518,7 @@ func callbackWrap(a *callbackArgs) { | |||||
| 329 | ptr = frame[pos] | 518 | ptr = frame[pos] | ||
| 330 | } | 519 | } | ||
| 331 | intsN += slots | 520 | intsN += slots | ||
| 332 | - | args[i] = reflect.ValueOf(strings.GoString(ptr)) | 521 | + | args[i] = reflect.ValueOf(internalstrings.GoString(ptr)) |
| 333 | continue | 522 | continue | ||
| 334 | case reflect.Struct: | 523 | case reflect.Struct: | ||
| 335 | if i == 0 && inType.AssignableTo(reflect.TypeOf(CDecl{})) { | 524 | if i == 0 && inType.AssignableTo(reflect.TypeOf(CDecl{})) { | ||