1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: 2022 The Ebitengine Authors
3
4//go:build darwin || freebsd || (linux && (386 || amd64 || arm || arm64 || loong64 || ppc64le || riscv64 || (s390x && (cgo || go1.27)))) || netbsd
5
6package purego
7
8import (
9 "encoding/json"
10 "errors"
11 "fmt"
12 "math"
13 "os"
14 "reflect"
15 "runtime"
16 "runtime/debug"
17 "strings"
18 "sync"
19 "time"
20 "unsafe"
21
22 internalstrings "github.com/bnema/purego/internal/strings"
23)
24
25var syscallXABI0 uintptr
26
27func syscall_syscallN(fn uintptr, args ...uintptr) (r1, r2, err uintptr) {
28 panic("purego: syscall_syscallN is only supported on windows")
29}
30
31// NewCallback converts a Go function to a function pointer conforming to the C calling convention.
32// This is useful when interoperating with C code requiring callbacks. The argument is expected to be a
33// function with zero or one uintptr-sized result. The function must not have arguments with size larger than the size
34// of uintptr. Only a limited number of callbacks may be created in a single Go process, and any memory allocated
35// for these callbacks is never released. At least 1024 callbacks can always be created. Although this function
36// provides similar functionality to windows.NewCallback it is distinct.
37//
38// Every call to NewCallback creates a new callback even for the same function value, so passing a Go callback to C
39// inside a loop (e.g. a qsort comparator) keeps consuming callbacks and eventually panics once they are exhausted.
40// The same happens when a func value is passed to a C function, as [RegisterFunc] creates a new callback for each
41// call. Create the callback once with NewCallback and reuse the returned pointer instead.
42func NewCallback(fn any) uintptr {
43 ty := reflect.TypeOf(fn)
44 if ty == nil || ty.Kind() != reflect.Func {
45 panic("purego: the type must be a function but was not")
46 }
47 for i := range ty.NumIn() {
48 in := ty.In(i)
49 if !in.AssignableTo(reflect.TypeFor[CDecl]()) {
50 continue
51 }
52 if i != 0 {
53 panic("purego: CDecl must be the first argument")
54 }
55 }
56 return compileCallback(fn)
57}
58
59// CallbackArgs is the integer-class argument block of one C call into a
60// NewCallbackInts callback. It is only valid during the callback.
61type CallbackArgs struct {
62 frame unsafe.Pointer
63 intFrame unsafe.Pointer
64 stackFrame unsafe.Pointer
65}
66
67// Int returns the i-th integer-class argument (0-based), following the
68// platform C ABI: integer registers first, then stack slots.
69func (a *CallbackArgs) Int(i int) uintptr {
70 if i < numOfIntegerRegisters() {
71 if a.intFrame != nil {
72 return (*[callbackMaxFrame]uintptr)(a.intFrame)[i]
73 }
74 return (*[callbackMaxFrame]uintptr)(a.frame)[numOfFloatRegisters()+i]
75 }
76 k := i - numOfIntegerRegisters()
77 if a.stackFrame != nil {
78 return (*[callbackMaxFrame]uintptr)(a.stackFrame)[k]
79 }
80 return (*[callbackMaxFrame]uintptr)(a.frame)[numOfFloatRegisters()+numOfIntegerRegisters()+k]
81}
82
83// NewCallbackInts returns a C function pointer that calls fn with the raw
84// integer-class arguments of the C call and returns its result in the C
85// integer return register. Only integer-class C parameters (integers,
86// pointers) are supported; float arguments are not visible. The call path
87// does no reflection and no allocation. Like NewCallback, the callback is
88// never freed and counts against the callback limit.
89func NewCallbackInts(fn func(a *CallbackArgs) uintptr) uintptr {
90 return compileFastCallback(fn)
91}
92
93// NewCallbackFnPtr converts a Go function pointer to a function pointer conforming to the C calling convention.
94// Calling this function multiple times with the same function pointer returns the original callback address.
95func NewCallbackFnPtr(fnPtr any) uintptr {
96 val := reflect.ValueOf(fnPtr)
97 if val.IsNil() {
98 panic("purego: function must not be nil")
99 }
100 if val.Kind() != reflect.Ptr || val.Elem().Kind() != reflect.Func {
101 panic("purego: the type must be a function pointer but was not")
102 }
103 if addr, idx, ok := getCallbackByFnPtr(val); ok {
104 cbs.lock.Lock()
105 recordCallbackLedger("dedupe", idx, addr, val.Pointer(), val.Elem(), false, "")
106 cbs.lock.Unlock()
107 return addr
108 }
109 addr := compileCallback(val.Elem().Interface())
110 cbs.lock.Lock()
111 cbs.knownFnPtr[val.Pointer()] = addr
112 if idx, ok := cbs.knownIdx[addr]; ok {
113 cbs.fnPtrKeys[idx] = val.Pointer()
114 recordCallbackLedger("fnptr-bind", idx, addr, val.Pointer(), val.Elem(), false, "")
115 }
116 cbs.lock.Unlock()
117 return addr
118}
119
120// UnrefCallback unreferences a callback created by NewCallback or NewCallbackFnPtr and frees its slot.
121func UnrefCallback(cb uintptr) error {
122 cbs.lock.Lock()
123 defer cbs.lock.Unlock()
124 idx, ok := cbs.knownIdx[cb]
125 if !ok {
126 recordCallbackLedger("release-miss", -1, cb, 0, reflect.Value{}, false, "callback not found")
127 return errors.New("callback not found")
128 }
129 val := cbs.funcs[idx]
130 key := cbs.fnPtrKeys[idx]
131 if key != 0 {
132 delete(cbs.knownFnPtr, key)
133 cbs.fnPtrKeys[idx] = 0
134 }
135 delete(cbs.knownIdx, cb)
136 cbs.holes[idx] = struct{}{}
137 cbs.funcs[idx] = reflect.Value{}
138 cbs.fast[idx] = nil
139 recordCallbackLedger("release", idx, cb, key, val, false, "")
140 return nil
141}
142
143// UnrefCallbackFnPtr unreferences a callback previously created via NewCallbackFnPtr.
144func UnrefCallbackFnPtr(fnPtr any) error {
145 val := reflect.ValueOf(fnPtr)
146 if val.IsNil() {
147 panic("purego: function must not be nil")
148 }
149 if val.Kind() != reflect.Ptr || val.Elem().Kind() != reflect.Func {
150 panic("purego: the type must be a function pointer but was not")
151 }
152 cbs.lock.Lock()
153 defer cbs.lock.Unlock()
154 addr, ok := cbs.knownFnPtr[val.Pointer()]
155 if !ok {
156 recordCallbackLedger("release-fnptr-miss", -1, 0, val.Pointer(), val.Elem(), false, "callback not found")
157 return errors.New("callback not found")
158 }
159 idx, ok := cbs.knownIdx[addr]
160 if !ok {
161 delete(cbs.knownFnPtr, val.Pointer())
162 recordCallbackLedger("release-fnptr-miss", -1, addr, val.Pointer(), val.Elem(), false, "callback not found")
163 return errors.New("callback not found")
164 }
165 callbackVal := cbs.funcs[idx]
166 delete(cbs.knownFnPtr, val.Pointer())
167 delete(cbs.knownIdx, addr)
168 cbs.fnPtrKeys[idx] = 0
169 cbs.holes[idx] = struct{}{}
170 cbs.funcs[idx] = reflect.Value{}
171 recordCallbackLedger("release-fnptr", idx, addr, val.Pointer(), callbackVal, false, "")
172 return nil
173}
174
175// maxCb is the maximum number of callbacks
176// only increase this if you have added more to the callbackasm function
177const maxCB = 2000
178
179var cbs = struct {
180 lock sync.RWMutex
181 numFn int // the highest allocated callback index + 1
182 holes map[int]struct{} // reusable callback slots
183 funcs [maxCB]reflect.Value // the saved callbacks
184 fast [maxCB]func(*CallbackArgs) uintptr // integer-only callbacks
185 knownIdx map[uintptr]int // callback address -> slot index
186 knownFnPtr map[uintptr]uintptr // function pointer variable address -> callback address
187 fnPtrKeys [maxCB]uintptr // slot index -> function pointer variable address
188 ledgerSeq uint64 // monotonic callback ledger event sequence
189}{
190 holes: make(map[int]struct{}),
191 knownIdx: make(map[uintptr]int, maxCB),
192 knownFnPtr: make(map[uintptr]uintptr, maxCB),
193}
194
195func getCallbackByFnPtr(val reflect.Value) (uintptr, int, bool) {
196 cbs.lock.RLock()
197 defer cbs.lock.RUnlock()
198 addr, ok := cbs.knownFnPtr[val.Pointer()]
199 if !ok {
200 return 0, -1, false
201 }
202 idx, ok := cbs.knownIdx[addr]
203 if !ok {
204 return addr, -1, true
205 }
206 return addr, idx, true
207}
208
209type callbackLedgerEvent struct {
210 Marker string `json:"marker"`
211 PID int `json:"pid"`
212 Seq uint64 `json:"seq"`
213 Time string `json:"time"`
214 Event string `json:"event"`
215 Index int `json:"index"`
216 Addr string `json:"addr,omitempty"`
217 FnPtrKey string `json:"fn_ptr_key,omitempty"`
218 Type string `json:"type,omitempty"`
219 Family string `json:"family"`
220 StackKey string `json:"stack_key"`
221 NumFn int `json:"num_fn"`
222 Occupied int `json:"occupied"`
223 Reusable int `json:"reusable"`
224 KnownFnPtr int `json:"known_fn_ptr"`
225 Remaining int `json:"remaining"`
226 Max int `json:"max"`
227 ReusedSlot bool `json:"reused_slot,omitempty"`
228 Stack string `json:"stack,omitempty"`
229 Error string `json:"error,omitempty"`
230}
231
232func callbackEnvEnabled(name string) bool {
233 switch strings.ToLower(os.Getenv(name)) {
234 case "1", "true", "yes", "on":
235 return true
236 default:
237 return false
238 }
239}
240
241func callbackLedgerEnabled() bool {
242 return callbackEnvEnabled("PUREGO_CALLBACK_LEDGER")
243}
244
245func callbackLedgerStackEnabled() bool {
246 return callbackEnvEnabled("PUREGO_CALLBACK_LEDGER_STACK")
247}
248
249func callbackTraceEnabled() bool {
250 return callbackEnvEnabled("PUREGO_CALLBACK_TRACE")
251}
252
253func callbackLedgerPath() string {
254 if path := os.Getenv("PUREGO_CALLBACK_LEDGER_FILE"); path != "" {
255 return path
256 }
257 return fmt.Sprintf("/tmp/purego-callback-ledger-%d.jsonl", os.Getpid())
258}
259
260func typeName(val reflect.Value) string {
261 if !val.IsValid() {
262 return ""
263 }
264 return val.Type().String()
265}
266
267func hexUintptr(v uintptr) string {
268 if v == 0 {
269 return ""
270 }
271 return fmt.Sprintf("0x%x", v)
272}
273
274func classifyCallbackStack(stack string) (family, stackKey string) {
275 family = "unknown"
276 stackKey = "unknown"
277 lines := strings.Split(stack, "\n")
278 for i, line := range lines {
279 if line == "" || strings.HasPrefix(line, "goroutine ") || strings.Contains(line, "runtime/debug.Stack") || strings.Contains(line, "recordCallbackLedger") || strings.Contains(line, "traceCallbackAllocation") {
280 continue
281 }
282 if strings.Contains(line, "github.com/bnema/purego.") || strings.Contains(line, "github.com/ebitengine/purego.") {
283 continue
284 }
285 if strings.HasPrefix(strings.TrimSpace(line), "/") || strings.HasPrefix(strings.TrimSpace(line), "../") {
286 continue
287 }
288 stackKey = strings.TrimSpace(line)
289 for j := i + 1; j < len(lines); j++ {
290 fileLine := strings.TrimSpace(lines[j])
291 if fileLine == "" {
292 continue
293 }
294 family = classifyCallbackFamily(fileLine + " " + stackKey)
295 return family, stackKey
296 }
297 family = classifyCallbackFamily(stackKey)
298 return family, stackKey
299 }
300 return family, stackKey
301}
302
303func classifyCallbackFamily(s string) string {
304 switch {
305 case strings.Contains(s, "purego-cef2gtk"):
306 return "purego-cef2gtk"
307 case strings.Contains(s, "purego-cef"):
308 return "purego-cef"
309 case strings.Contains(s, "puregotk"):
310 return "puregotk"
311 case strings.Contains(s, "dumber"):
312 return "dumber"
313 case strings.Contains(s, "github.com/bnema/purego") || strings.Contains(s, "/purego/"):
314 return "purego"
315 default:
316 return "unknown"
317 }
318}
319
320func recordCallbackLedger(event string, idx int, addr uintptr, fnPtrKey uintptr, val reflect.Value, reused bool, errText string) {
321 if !callbackLedgerEnabled() {
322 return
323 }
324 stack := ""
325 family := "unknown"
326 stackKey := "unknown"
327 if callbackLedgerStackEnabled() {
328 stack = string(debug.Stack())
329 family, stackKey = classifyCallbackStack(stack)
330 }
331 cbs.ledgerSeq++
332 occupied := cbs.numFn - len(cbs.holes)
333 entry := callbackLedgerEvent{
334 Marker: "PUREGO-CALLBACK-LEDGER",
335 PID: os.Getpid(),
336 Seq: cbs.ledgerSeq,
337 Time: time.Now().Format(time.RFC3339Nano),
338 Event: event,
339 Index: idx,
340 Addr: hexUintptr(addr),
341 FnPtrKey: hexUintptr(fnPtrKey),
342 Type: typeName(val),
343 Family: family,
344 StackKey: stackKey,
345 NumFn: cbs.numFn,
346 Occupied: occupied,
347 Reusable: len(cbs.holes),
348 KnownFnPtr: len(cbs.knownFnPtr),
349 Remaining: maxCB - occupied,
350 Max: maxCB,
351 ReusedSlot: reused,
352 Error: errText,
353 }
354 if callbackLedgerStackEnabled() {
355 entry.Stack = stack
356 }
357 payload, err := json.Marshal(entry)
358 if err != nil {
359 return
360 }
361 payload = append(payload, '\n')
362 if f, err := os.OpenFile(callbackLedgerPath(), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600); err == nil {
363 _, _ = f.Write(payload)
364 _ = f.Close()
365 }
366}
367
368func traceCallbackAllocation(event string, val reflect.Value, remaining int) {
369 if !callbackTraceEnabled() {
370 return
371 }
372 message := fmt.Sprintf("PUREGO-CALLBACK-TRACE event=%s remaining=%d live=%d type=%s\n%s\n", event, remaining, maxCB-remaining, val.Type(), debug.Stack())
373 fmt.Fprint(os.Stderr, message)
374 traceFile := os.Getenv("PUREGO_CALLBACK_TRACE_FILE")
375 if traceFile == "" {
376 traceFile = fmt.Sprintf("/tmp/purego-callback-trace-%d.log", os.Getpid())
377 }
378 if f, err := os.OpenFile(traceFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600); err == nil {
379 _, _ = f.WriteString(message)
380 _ = f.Close()
381 }
382}
383
384func compileCallback(fn any) uintptr {
385 val := reflect.ValueOf(fn)
386 if val.Kind() != reflect.Func {
387 panic("purego: the type must be a function but was not")
388 }
389 if val.IsNil() {
390 panic("purego: function must not be nil")
391 }
392 ty := val.Type()
393 for i := range ty.NumIn() {
394 in := ty.In(i)
395 switch in.Kind() {
396 case reflect.Struct:
397 if i == 0 && in.AssignableTo(reflect.TypeFor[CDecl]()) {
398 continue
399 }
400 ensureCallbackStructSupported()
401 checkStructFieldsSupported(in)
402 continue
403 case reflect.Interface, reflect.Func, reflect.Slice,
404 reflect.Chan, reflect.Complex64, reflect.Complex128,
405 reflect.Map, reflect.Invalid:
406 panic("purego: unsupported argument type: " + in.Kind().String())
407 }
408 }
409output:
410 switch {
411 case ty.NumOut() == 1:
412 switch ty.Out(0).Kind() {
413 case reflect.Struct:
414 ensureCallbackStructSupported()
415 checkStructFieldsSupported(ty.Out(0))
416 break output
417 case reflect.Pointer, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
418 reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
419 reflect.Bool, reflect.UnsafePointer:
420 break output
421 }
422 panic("purego: unsupported return type: " + ty.String())
423 case ty.NumOut() > 1:
424 panic("purego: callbacks can only have one return")
425 }
426 cbs.lock.Lock()
427 defer cbs.lock.Unlock()
428 return allocateCallback(val, nil)
429}
430
431func compileFastCallback(fn func(*CallbackArgs) uintptr) uintptr {
432 if fn == nil {
433 panic("purego: function must not be nil")
434 }
435 cbs.lock.Lock()
436 defer cbs.lock.Unlock()
437 return allocateCallback(reflect.Value{}, fn)
438}
439
440// allocateCallback is called with cbs.lock held.
441func allocateCallback(val reflect.Value, fast func(*CallbackArgs) uintptr) uintptr {
442 index := -1
443 reused := false
444 for i := range cbs.holes {
445 index = i
446 delete(cbs.holes, i)
447 reused = true
448 break
449 }
450 if index < 0 {
451 remaining := maxCB - cbs.numFn
452 if remaining <= 0 {
453 recordCallbackLedger("exhausted", -1, 0, 0, val, false, "maximum callbacks reached")
454 if val.IsValid() {
455 traceCallbackAllocation("exhausted", val, 0)
456 }
457 panic("purego: the maximum number of callbacks has been reached")
458 }
459 if val.IsValid() && (remaining <= 100 || remaining%250 == 0) {
460 traceCallbackAllocation("allocated", val, remaining)
461 }
462 index = cbs.numFn
463 cbs.numFn++
464 }
465 cbs.funcs[index] = val
466 cbs.fast[index] = fast
467 addr := callbackasmAddr(index)
468 cbs.knownIdx[addr] = index
469 if reused {
470 recordCallbackLedger("reuse", index, addr, 0, val, true, "")
471 } else {
472 recordCallbackLedger("alloc", index, addr, 0, val, false, "")
473 }
474 return addr
475}
476
477const ptrSize = unsafe.Sizeof((*int)(nil))
478
479const callbackMaxFrame = 64 * ptrSize
480
481// callbackasm is implemented in zcallback_GOOS_GOARCH.s
482//
483//go:linkname __callbackasm callbackasm
484var __callbackasm byte
485var callbackasmABI0 = uintptr(unsafe.Pointer(&__callbackasm))
486
487// callbackWrap_call allows the calling of the ABIInternal wrapper
488// which is required for runtime.cgocallback without the
489// <ABIInternal> tag which is only allowed in the runtime.
490// This closure is used inside sys_darwin_GOARCH.s
491var callbackWrap_call = callbackWrap
492
493// noescape is runtime.noescape; the synchronous callee must not retain p.
494//
495//go:noescape
496//go:linkname noescape runtime.noescape
497func noescape(p unsafe.Pointer) unsafe.Pointer
498
499// callbackWrap is called by assembly code which determines which Go function to call.
500// This function takes the arguments and passes them to the Go function and returns the result.
501func callbackWrap(a *callbackArgs) {
502 // Slots are published under the lock before the C pointer is handed out.
503 // A caller must not invoke a released callback while UnrefCallback reuses
504 // its slot; while callable, fast slots are never cleared.
505 if f := cbs.fast[a.index]; f != nil {
506 ca := CallbackArgs{frame: a.args, intFrame: a.intFrame(), stackFrame: a.stackFrame()}
507 a.result[0] = f((*CallbackArgs)(noescape(unsafe.Pointer(&ca))))
508 return
509 }
510 cbs.lock.Lock()
511 fn := cbs.funcs[a.index]
512 cbs.lock.Unlock()
513 fnType := fn.Type()
514 args := make([]reflect.Value, fnType.NumIn())
515 frame := (*[callbackMaxFrame]uintptr)(a.args)
516 // stackFrame points to stack-passed arguments. On most architectures this is
517 // contiguous with frame (after register args), but on ppc64le it's separate.
518 var stackFrame *[callbackMaxFrame]uintptr
519 var intFrame *[callbackMaxFrame]uintptr
520 if sf := a.stackFrame(); sf != nil {
521 // Only ppc64le uses separate stackArgs pointer due to NOSPLIT constraints
522 stackFrame = (*[callbackMaxFrame]uintptr)(sf)
523 }
524 if intf := a.intFrame(); intf != nil {
525 intFrame = (*[callbackMaxFrame]uintptr)(intf)
526 }
527 // floatsN and intsN track the number of register slots used, not argument count.
528 // This distinction matters on ARM32 where float64 uses 2 slots (32-bit registers).
529 var floatsN int
530 var intsN int
531 // On amd64/loong64/ppc64le/riscv64/s390x, when returning a struct larger than
532 // maxRegAllocStructSize, the caller passes a hidden pointer in the first integer
533 // register. Skip it to avoid misreading it as the first function argument.
534 if (runtime.GOARCH == "amd64" || runtime.GOARCH == "loong64" || runtime.GOARCH == "riscv64" || runtime.GOARCH == "s390x") &&
535 fnType.NumOut() == 1 && fnType.Out(0).Kind() == reflect.Struct &&
536 fnType.Out(0).Size() > maxRegAllocStructSize {
537 intsN = 1
538 }
539 // stackSlot points to the index into frame (or stackFrame) of the current stack element.
540 // When stackFrame is nil, stack begins after float and integer registers in frame.
541 // When stackFrame is not nil (ppc64le), stackSlot indexes into stackFrame starting at 0.
542 stackSlot := numOfIntegerRegisters() + numOfFloatRegisters()
543 if stackFrame != nil {
544 // ppc64le: stackArgs is a separate pointer, indices start at 0
545 stackSlot = 0
546 }
547 // stackByteOffset tracks the byte offset within the stack area for Darwin ARM64
548 // tight packing. On Darwin ARM64, C passes small types packed on the stack.
549 stackByteOffset := uintptr(0)
550 for i := range args {
551 // slots is the number of pointer-sized slots the argument takes
552 inType := fnType.In(i)
553 slots := int((inType.Size() + ptrSize - 1) / ptrSize)
554 switch inType.Kind() {
555 case reflect.Float32, reflect.Float64:
556 if isARMSoftFloat() {
557 // we should restore from integer slot, can skip unnecessary branching here
558 if isARMPaddingNeeded(inType, -1, intsN) {
559 intsN++
560 }
561 if intsN+slots <= numOfIntegerRegisters() {
562 // the integers begin after the floats in frame
563 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[intsN+numOfFloatRegisters()])).Elem()
564 intsN += slots
565 continue
566 }
567 if isARMPaddingNeeded(inType, -1, stackSlot) {
568 stackSlot++
569 }
570 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
571 stackSlot += slots
572 intsN += slots
573 continue
574 }
575
576 if floatsN+slots > numOfFloatRegisters() {
577 if isDarwin && runtime.GOARCH == "arm64" {
578 // Darwin ARM64: read from packed stack with proper alignment
579 args[i] = callbackArgFromStack(a.args, stackSlot, &stackByteOffset, inType)
580 } else if stackFrame != nil {
581 // ppc64le/s390x: stack args are in separate stackFrame
582 switch runtime.GOARCH {
583 case "ppc64le":
584 args[i] = callbackFloatFromDoubleSlot(unsafe.Pointer(&stackFrame[stackSlot]), inType)
585 case "s390x":
586 // s390x big-endian: sub-8-byte values are right-justified
587 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&stackFrame[stackSlot]), inType)
588 default:
589 args[i] = reflect.NewAt(inType, unsafe.Pointer(&stackFrame[stackSlot])).Elem()
590 }
591 stackSlot += slots
592 } else if isARMFloatPaddingNeeded(inType, -1, stackSlot) {
593 stackSlot++
594 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
595 stackSlot += slots
596 } else {
597 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
598 stackSlot += slots
599 }
600 } else {
601 switch runtime.GOARCH {
602 case "ppc64le":
603 args[i] = callbackFloatFromDoubleSlot(unsafe.Pointer(&frame[floatsN]), inType)
604 case "s390x":
605 // s390x big-endian: float32 is right-justified in 8-byte FPR slot
606 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&frame[floatsN]), inType)
607 default:
608 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[floatsN])).Elem()
609 }
610 }
611 floatsN += slots
612 case reflect.String:
613 slots = 1
614 var ptr uintptr
615 if intsN+slots > numOfIntegerRegisters() {
616 if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
617 ptr = uintptr(callbackArgFromStack(a.args, stackSlot, &stackByteOffset, reflect.TypeOf(uintptr(0))).Uint())
618 } else if stackFrame != nil {
619 if runtime.GOARCH == "s390x" {
620 ptr = uintptr(callbackArgFromSlotBigEndian(unsafe.Pointer(&stackFrame[stackSlot]), reflect.TypeOf(uintptr(0))).Uint())
621 } else {
622 ptr = *(*uintptr)(unsafe.Pointer(&stackFrame[stackSlot]))
623 }
624 stackSlot += slots
625 } else {
626 ptr = frame[stackSlot]
627 stackSlot += slots
628 }
629 } else {
630 pos := intsN + numOfFloatRegisters()
631 ptr = frame[pos]
632 }
633 intsN += slots
634 args[i] = reflect.ValueOf(internalstrings.GoString(ptr))
635 continue
636 case reflect.Struct:
637 if i == 0 && inType.AssignableTo(reflect.TypeFor[CDecl]()) {
638 args[i] = reflect.Zero(inType)
639 continue
640 }
641 if inType.Size() == 0 {
642 args[i] = reflect.New(inType).Elem()
643 continue
644 }
645 args[i] = getCallbackStruct(inType, a.args, &floatsN, &intsN, &stackSlot, &stackByteOffset)
646 continue
647 default:
648 if isARMPaddingNeeded(inType, -1, intsN) {
649 intsN++
650 }
651 if intsN+slots > numOfIntegerRegisters() {
652 if isDarwin && runtime.GOARCH == "arm64" {
653 // Darwin ARM64: read from packed stack with proper alignment
654 args[i] = callbackArgFromStack(a.args, stackSlot, &stackByteOffset, inType)
655 } else if stackFrame != nil {
656 // ppc64le/s390x: stack args are in separate stackFrame
657 if runtime.GOARCH == "s390x" {
658 // s390x big-endian: sub-8-byte values are right-justified
659 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&stackFrame[stackSlot]), inType)
660 } else {
661 args[i] = reflect.NewAt(inType, unsafe.Pointer(&stackFrame[stackSlot])).Elem()
662 }
663 stackSlot += slots
664 } else if isARMPaddingNeeded(inType, -1, stackSlot) {
665 stackSlot++
666 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
667 stackSlot += slots
668 } else {
669 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
670 stackSlot += slots
671 }
672 } else {
673 if intFrame != nil {
674 args[i] = reflect.NewAt(inType, unsafe.Pointer(&intFrame[intsN])).Elem()
675 } else {
676 // the integers begin after the floats in frame
677 pos := intsN + numOfFloatRegisters()
678 if runtime.GOARCH == "s390x" {
679 // s390x big-endian: sub-8-byte values are right-justified in GPR slot
680 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&frame[pos]), inType)
681 } else {
682 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[pos])).Elem()
683 }
684 }
685 }
686 intsN += slots
687 }
688 }
689 ret := fn.Call(args)
690 if len(ret) > 0 {
691 switch k := ret[0].Kind(); k {
692 case reflect.Uint64:
693 a.setUint64Result(ret[0].Uint())
694 case reflect.Uint, reflect.Uint32, reflect.Uint16, reflect.Uint8, reflect.Uintptr:
695 a.result[0] = uintptr(ret[0].Uint())
696 case reflect.Int64:
697 a.setInt64Result(ret[0].Int())
698 case reflect.Int, reflect.Int32, reflect.Int16, reflect.Int8:
699 a.result[0] = uintptr(ret[0].Int())
700 case reflect.Bool:
701 if ret[0].Bool() {
702 a.result[0] = 1
703 } else {
704 a.result[0] = 0
705 }
706 case reflect.Pointer:
707 a.result[0] = ret[0].Pointer()
708 case reflect.UnsafePointer:
709 a.result[0] = ret[0].Pointer()
710 case reflect.Struct:
711 setStruct(a, ret[0])
712 default:
713 panic("purego: unsupported kind: " + k.String())
714 }
715 }
716}
717
718// callbackArgFromStack reads an argument from the tightly-packed stack area on Darwin ARM64.
719// The C ABI on Darwin ARM64 packs small types on the stack without padding to 8 bytes.
720// This function handles proper alignment and advances stackByteOffset accordingly.
721func callbackArgFromStack(argsBase unsafe.Pointer, stackSlot int, stackByteOffset *uintptr, inType reflect.Type) reflect.Value {
722 // Calculate base address of stack area (after float and int registers)
723 stackBase := unsafe.Add(argsBase, stackSlot*int(ptrSize))
724
725 // Get type's natural alignment
726 align := uintptr(inType.Align())
727 size := inType.Size()
728
729 // Align the offset
730 if *stackByteOffset%align != 0 {
731 *stackByteOffset = (*stackByteOffset + align - 1) &^ (align - 1)
732 }
733
734 // Read value at aligned offset
735 ptr := unsafe.Add(stackBase, *stackByteOffset)
736 *stackByteOffset += size
737
738 return reflect.NewAt(inType, ptr).Elem()
739}
740
741// callbackFloatFromDoubleSlot reads a floating-point callback argument from an
742// 8-byte register or stack slot on ppc64le, where a single-precision value is
743// held in double-precision format.
744func callbackFloatFromDoubleSlot(slotPtr unsafe.Pointer, inType reflect.Type) reflect.Value {
745 if inType.Kind() != reflect.Float32 {
746 return reflect.NewAt(inType, slotPtr).Elem()
747 }
748 v := reflect.New(inType).Elem()
749 v.SetFloat(math.Float64frombits(*(*uint64)(slotPtr)))
750 return v
751}
752
753// callbackArgFromSlotBigEndian reads an argument from an 8-byte slot on big-endian architectures.
754// On s390x:
755// - Integer types are right-justified in GPRs: sub-8-byte values are at offset (8 - size)
756// - Float32 in FPRs is left-justified: stored in upper 32 bits, so at offset 0
757// - Float64 occupies the full 8-byte slot
758func callbackArgFromSlotBigEndian(slotPtr unsafe.Pointer, inType reflect.Type) reflect.Value {
759 size := inType.Size()
760 if size >= 8 {
761 // 8-byte values occupy the entire slot
762 return reflect.NewAt(inType, slotPtr).Elem()
763 }
764 // Float32 is left-justified in FPRs (upper 32 bits), so offset is 0
765 if inType.Kind() == reflect.Float32 {
766 return reflect.NewAt(inType, slotPtr).Elem()
767 }
768 // Integer types are right-justified: offset = 8 - size
769 offset := 8 - size
770 ptr := unsafe.Add(slotPtr, offset)
771 return reflect.NewAt(inType, ptr).Elem()
772}
773
774// callbackasmAddr returns address of runtime.callbackasm
775// function adjusted by i.
776// On x86 and amd64, runtime.callbackasm is a series of CALL instructions,
777// and we want callback to arrive at
778// correspondent call instruction instead of start of
779// runtime.callbackasm.
780// On ARM, runtime.callbackasm is a series of mov and branch instructions.
781// R12 is loaded with the callback index. Each entry is two instructions,
782// hence 8 bytes.
783func callbackasmAddr(i int) uintptr {
784 var entrySize int
785 switch runtime.GOARCH {
786 default:
787 panic("purego: unsupported architecture")
788 case "amd64":
789 // On amd64, each callback entry is just a CALL instruction (5 bytes)
790 entrySize = 5
791 case "386":
792 // On 386, each callback entry is MOVL $imm, CX (5 bytes) + JMP (5 bytes)
793 entrySize = 10
794 case "arm", "arm64", "loong64", "ppc64le", "riscv64":
795 // On ARM, ARM64, Loong64, PPC64LE and RISCV64, each entry is a MOV instruction
796 // followed by a branch instruction
797 entrySize = 8
798 case "s390x":
799 // On S390X, each entry is LGHI (4 bytes) + JG (6 bytes)
800 entrySize = 10
801 }
802 return callbackasmABI0 + uintptr(i*entrySize)
803}