1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: 2022 The Ebitengine Authors
3
4//go:build darwin || freebsd || (linux && (386 || amd64 || arm || arm64 || loong64 || ppc64le || riscv64 || (s390x && (cgo || go1.27)))) || netbsd
5
6package purego
7
8import (
9 "encoding/json"
10 "errors"
11 "fmt"
12 "math"
13 "os"
14 "reflect"
15 "runtime"
16 "runtime/debug"
17 "strings"
18 "sync"
19 "time"
20 "unsafe"
21
22 internalstrings "github.com/bnema/purego/internal/strings"
23)
24
25var syscallXABI0 uintptr
26
27func syscall_syscallN(fn uintptr, args ...uintptr) (r1, r2, err uintptr) {
28 panic("purego: syscall_syscallN is only supported on windows")
29}
30
31// NewCallback converts a Go function to a function pointer conforming to the C calling convention.
32// This is useful when interoperating with C code requiring callbacks. The argument is expected to be a
33// function with zero or one uintptr-sized result. The function must not have arguments with size larger than the size
34// of uintptr. Only a limited number of callbacks may be created in a single Go process, and any memory allocated
35// for these callbacks is never released. At least 1024 callbacks can always be created. Although this function
36// provides similar functionality to windows.NewCallback it is distinct.
37//
38// Every call to NewCallback creates a new callback even for the same function value, so passing a Go callback to C
39// inside a loop (e.g. a qsort comparator) keeps consuming callbacks and eventually panics once they are exhausted.
40// The same happens when a func value is passed to a C function, as [RegisterFunc] creates a new callback for each
41// call. Create the callback once with NewCallback and reuse the returned pointer instead.
42func NewCallback(fn any) uintptr {
43 ty := reflect.TypeOf(fn)
44 if ty == nil || ty.Kind() != reflect.Func {
45 panic("purego: the type must be a function but was not")
46 }
47 for i := range ty.NumIn() {
48 in := ty.In(i)
49 if !in.AssignableTo(reflect.TypeFor[CDecl]()) {
50 continue
51 }
52 if i != 0 {
53 panic("purego: CDecl must be the first argument")
54 }
55 }
56 return compileCallback(fn)
57}
58
59// CallbackArgs is the integer-class argument block of one C call into a
60// NewCallbackInts callback. It is only valid during the callback.
61type CallbackArgs struct {
62 frame unsafe.Pointer
63 intFrame unsafe.Pointer
64 stackFrame unsafe.Pointer
65}
66
67// Int returns the i-th integer-class argument (0-based), following the
68// platform C ABI: integer registers first, then stack slots.
69func (a *CallbackArgs) Int(i int) uintptr {
70 if i < numOfIntegerRegisters() {
71 if a.intFrame != nil {
72 return (*[callbackMaxFrame]uintptr)(a.intFrame)[i]
73 }
74 return (*[callbackMaxFrame]uintptr)(a.frame)[numOfFloatRegisters()+i]
75 }
76 k := i - numOfIntegerRegisters()
77 if a.stackFrame != nil {
78 return (*[callbackMaxFrame]uintptr)(a.stackFrame)[k]
79 }
80 return (*[callbackMaxFrame]uintptr)(a.frame)[numOfFloatRegisters()+numOfIntegerRegisters()+k]
81}
82
83// NewCallbackInts returns a C function pointer that calls fn with the raw
84// integer-class arguments of the C call and returns its result in the C
85// integer return register. Only integer-class C parameters (integers,
86// pointers) are supported; float arguments are not visible. The call path
87// does no reflection and no allocation. Like NewCallback, the callback is
88// never freed and counts against the callback limit.
89func NewCallbackInts(fn func(a *CallbackArgs) uintptr) uintptr {
90 return compileFastCallback(fn)
91}
92
93// NewCallbackFnPtr converts a Go function pointer to a function pointer conforming to the C calling convention.
94// Calling this function multiple times with the same function pointer returns the original callback address.
95func NewCallbackFnPtr(fnPtr any) uintptr {
96 val := reflect.ValueOf(fnPtr)
97 if val.IsNil() {
98 panic("purego: function must not be nil")
99 }
100 if val.Kind() != reflect.Ptr || val.Elem().Kind() != reflect.Func {
101 panic("purego: the type must be a function pointer but was not")
102 }
103 if addr, idx, ok := getCallbackByFnPtr(val); ok {
104 cbs.lock.Lock()
105 recordCallbackLedger("dedupe", idx, addr, val.Pointer(), val.Elem(), false, "")
106 cbs.lock.Unlock()
107 return addr
108 }
109 addr := compileCallback(val.Elem().Interface())
110 cbs.lock.Lock()
111 cbs.knownFnPtr[val.Pointer()] = addr
112 if idx, ok := cbs.knownIdx[addr]; ok {
113 cbs.fnPtrKeys[idx] = val.Pointer()
114 recordCallbackLedger("fnptr-bind", idx, addr, val.Pointer(), val.Elem(), false, "")
115 }
116 cbs.lock.Unlock()
117 return addr
118}
119
120// UnrefCallback unreferences a callback created by NewCallback or NewCallbackFnPtr and frees its slot.
121func UnrefCallback(cb uintptr) error {
122 cbs.lock.Lock()
123 defer cbs.lock.Unlock()
124 idx, ok := cbs.knownIdx[cb]
125 if !ok {
126 recordCallbackLedger("release-miss", -1, cb, 0, reflect.Value{}, false, "callback not found")
127 return errors.New("callback not found")
128 }
129 val := cbs.funcs[idx]
130 key := cbs.fnPtrKeys[idx]
131 if key != 0 {
132 delete(cbs.knownFnPtr, key)
133 cbs.fnPtrKeys[idx] = 0
134 }
135 delete(cbs.knownIdx, cb)
136 cbs.holes[idx] = struct{}{}
137 cbs.funcs[idx] = reflect.Value{}
138 cbs.fast[idx] = nil
139 recordCallbackLedger("release", idx, cb, key, val, false, "")
140 return nil
141}
142
143// UnrefCallbackFnPtr unreferences a callback previously created via NewCallbackFnPtr.
144func UnrefCallbackFnPtr(fnPtr any) error {
145 val := reflect.ValueOf(fnPtr)
146 if val.IsNil() {
147 panic("purego: function must not be nil")
148 }
149 if val.Kind() != reflect.Ptr || val.Elem().Kind() != reflect.Func {
150 panic("purego: the type must be a function pointer but was not")
151 }
152 cbs.lock.Lock()
153 defer cbs.lock.Unlock()
154 addr, ok := cbs.knownFnPtr[val.Pointer()]
155 if !ok {
156 recordCallbackLedger("release-fnptr-miss", -1, 0, val.Pointer(), val.Elem(), false, "callback not found")
157 return errors.New("callback not found")
158 }
159 idx, ok := cbs.knownIdx[addr]
160 if !ok {
161 delete(cbs.knownFnPtr, val.Pointer())
162 recordCallbackLedger("release-fnptr-miss", -1, addr, val.Pointer(), val.Elem(), false, "callback not found")
163 return errors.New("callback not found")
164 }
165 callbackVal := cbs.funcs[idx]
166 delete(cbs.knownFnPtr, val.Pointer())
167 delete(cbs.knownIdx, addr)
168 cbs.fnPtrKeys[idx] = 0
169 cbs.holes[idx] = struct{}{}
170 cbs.funcs[idx] = reflect.Value{}
171 recordCallbackLedger("release-fnptr", idx, addr, val.Pointer(), callbackVal, false, "")
172 return nil
173}
174
175// maxCb is the maximum number of callbacks
176// only increase this if you have added more to the callbackasm function
177const maxCB = 2000
178
179var cbs = struct {
180 lock sync.RWMutex
181 numFn int // the highest allocated callback index + 1
182 holes map[int]struct{} // reusable callback slots
183 funcs [maxCB]reflect.Value // the saved callbacks
184 fast [maxCB]func(*CallbackArgs) uintptr // integer-only callbacks
185 knownIdx map[uintptr]int // callback address -> slot index
186 knownFnPtr map[uintptr]uintptr // function pointer variable address -> callback address
187 fnPtrKeys [maxCB]uintptr // slot index -> function pointer variable address
188 ledgerSeq uint64 // monotonic callback ledger event sequence
189}{
190 holes: make(map[int]struct{}),
191 knownIdx: make(map[uintptr]int, maxCB),
192 knownFnPtr: make(map[uintptr]uintptr, maxCB),
193}
194
195func getCallbackByFnPtr(val reflect.Value) (uintptr, int, bool) {
196 cbs.lock.RLock()
197 defer cbs.lock.RUnlock()
198 addr, ok := cbs.knownFnPtr[val.Pointer()]
199 if !ok {
200 return 0, -1, false
201 }
202 idx, ok := cbs.knownIdx[addr]
203 if !ok {
204 return addr, -1, true
205 }
206 return addr, idx, true
207}
208
209type callbackLedgerEvent struct {
210 Marker string `json:"marker"`
211 PID int `json:"pid"`
212 Seq uint64 `json:"seq"`
213 Time string `json:"time"`
214 Event string `json:"event"`
215 Index int `json:"index"`
216 Addr string `json:"addr,omitempty"`
217 FnPtrKey string `json:"fn_ptr_key,omitempty"`
218 Type string `json:"type,omitempty"`
219 Family string `json:"family"`
220 StackKey string `json:"stack_key"`
221 NumFn int `json:"num_fn"`
222 Occupied int `json:"occupied"`
223 Reusable int `json:"reusable"`
224 KnownFnPtr int `json:"known_fn_ptr"`
225 Remaining int `json:"remaining"`
226 Max int `json:"max"`
227 ReusedSlot bool `json:"reused_slot,omitempty"`
228 Stack string `json:"stack,omitempty"`
229 Error string `json:"error,omitempty"`
230}
231
232func callbackEnvEnabled(name string) bool {
233 switch strings.ToLower(os.Getenv(name)) {
234 case "1", "true", "yes", "on":
235 return true
236 default:
237 return false
238 }
239}
240
241func callbackLedgerEnabled() bool {
242 return callbackEnvEnabled("PUREGO_CALLBACK_LEDGER")
243}
244
245func callbackLedgerStackEnabled() bool {
246 return callbackEnvEnabled("PUREGO_CALLBACK_LEDGER_STACK")
247}
248
249func callbackTraceEnabled() bool {
250 return callbackEnvEnabled("PUREGO_CALLBACK_TRACE")
251}
252
253func callbackLedgerPath() string {
254 if path := os.Getenv("PUREGO_CALLBACK_LEDGER_FILE"); path != "" {
255 return path
256 }
257 return fmt.Sprintf("/tmp/purego-callback-ledger-%d.jsonl", os.Getpid())
258}
259
260func typeName(val reflect.Value) string {
261 if !val.IsValid() {
262 return ""
263 }
264 return val.Type().String()
265}
266
267func hexUintptr(v uintptr) string {
268 if v == 0 {
269 return ""
270 }
271 return fmt.Sprintf("0x%x", v)
272}
273
274func classifyCallbackStack(stack string) (family, stackKey string) {
275 family = "unknown"
276 stackKey = "unknown"
277 lines := strings.Split(stack, "\n")
278 for i, line := range lines {
279 if line == "" || strings.HasPrefix(line, "goroutine ") || strings.Contains(line, "runtime/debug.Stack") || strings.Contains(line, "recordCallbackLedger") || strings.Contains(line, "traceCallbackAllocation") {
280 continue
281 }
282 if strings.Contains(line, "github.com/bnema/purego.") || strings.Contains(line, "github.com/ebitengine/purego.") {
283 continue
284 }
285 if strings.HasPrefix(strings.TrimSpace(line), "/") || strings.HasPrefix(strings.TrimSpace(line), "../") {
286 continue
287 }
288 stackKey = strings.TrimSpace(line)
289 for j := i + 1; j < len(lines); j++ {
290 fileLine := strings.TrimSpace(lines[j])
291 if fileLine == "" {
292 continue
293 }
294 family = classifyCallbackFamily(fileLine + " " + stackKey)
295 return family, stackKey
296 }
297 family = classifyCallbackFamily(stackKey)
298 return family, stackKey
299 }
300 return family, stackKey
301}
302
303func classifyCallbackFamily(s string) string {
304 switch {
305 case strings.Contains(s, "purego-cef2gtk"):
306 return "purego-cef2gtk"
307 case strings.Contains(s, "purego-cef"):
308 return "purego-cef"
309 case strings.Contains(s, "puregotk"):
310 return "puregotk"
311 case strings.Contains(s, "dumber"):
312 return "dumber"
313 case strings.Contains(s, "github.com/bnema/purego") || strings.Contains(s, "/purego/"):
314 return "purego"
315 default:
316 return "unknown"
317 }
318}
319
320func recordCallbackLedger(event string, idx int, addr uintptr, fnPtrKey uintptr, val reflect.Value, reused bool, errText string) {
321 if !callbackLedgerEnabled() {
322 return
323 }
324 stack := ""
325 family := "unknown"
326 stackKey := "unknown"
327 if callbackLedgerStackEnabled() {
328 stack = string(debug.Stack())
329 family, stackKey = classifyCallbackStack(stack)
330 }
331 cbs.ledgerSeq++
332 occupied := cbs.numFn - len(cbs.holes)
333 entry := callbackLedgerEvent{
334 Marker: "PUREGO-CALLBACK-LEDGER",
335 PID: os.Getpid(),
336 Seq: cbs.ledgerSeq,
337 Time: time.Now().Format(time.RFC3339Nano),
338 Event: event,
339 Index: idx,
340 Addr: hexUintptr(addr),
341 FnPtrKey: hexUintptr(fnPtrKey),
342 Type: typeName(val),
343 Family: family,
344 StackKey: stackKey,
345 NumFn: cbs.numFn,
346 Occupied: occupied,
347 Reusable: len(cbs.holes),
348 KnownFnPtr: len(cbs.knownFnPtr),
349 Remaining: maxCB - occupied,
350 Max: maxCB,
351 ReusedSlot: reused,
352 Error: errText,
353 }
354 if callbackLedgerStackEnabled() {
355 entry.Stack = stack
356 }
357 payload, err := json.Marshal(entry)
358 if err != nil {
359 return
360 }
361 payload = append(payload, '\n')
362 if f, err := os.OpenFile(callbackLedgerPath(), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600); err == nil {
363 _, _ = f.Write(payload)
364 _ = f.Close()
365 }
366}
367
368func traceCallbackAllocation(event string, val reflect.Value, remaining int) {
369 if !callbackTraceEnabled() {
370 return
371 }
372 message := fmt.Sprintf("PUREGO-CALLBACK-TRACE event=%s remaining=%d live=%d type=%s\n%s\n", event, remaining, maxCB-remaining, val.Type(), debug.Stack())
373 fmt.Fprint(os.Stderr, message)
374 traceFile := os.Getenv("PUREGO_CALLBACK_TRACE_FILE")
375 if traceFile == "" {
376 traceFile = fmt.Sprintf("/tmp/purego-callback-trace-%d.log", os.Getpid())
377 }
378 if f, err := os.OpenFile(traceFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600); err == nil {
379 _, _ = f.WriteString(message)
380 _ = f.Close()
381 }
382}
383
384func compileCallback(fn any) uintptr {
385 val := reflect.ValueOf(fn)
386 if val.Kind() != reflect.Func {
387 panic("purego: the type must be a function but was not")
388 }
389 if val.IsNil() {
390 panic("purego: function must not be nil")
391 }
392 ty := val.Type()
393 for i := range ty.NumIn() {
394 in := ty.In(i)
395 switch in.Kind() {
396 case reflect.Struct:
397 if i == 0 && in.AssignableTo(reflect.TypeFor[CDecl]()) {
398 continue
399 }
400 ensureCallbackStructSupported()
401 checkStructFieldsSupported(in)
402 continue
403 case reflect.Interface, reflect.Func, reflect.Slice,
404 reflect.Chan, reflect.Complex64, reflect.Complex128,
405 reflect.Map, reflect.Invalid:
406 panic("purego: unsupported argument type: " + in.Kind().String())
407 }
408 }
409output:
410 switch {
411 case ty.NumOut() == 1:
412 switch ty.Out(0).Kind() {
413 case reflect.Struct:
414 ensureCallbackStructSupported()
415 checkStructFieldsSupported(ty.Out(0))
416 break output
417 case reflect.Pointer, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
418 reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
419 reflect.Bool, reflect.UnsafePointer:
420 break output
421 }
422 panic("purego: unsupported return type: " + ty.String())
423 case ty.NumOut() > 1:
424 panic("purego: callbacks can only have one return")
425 }
426 cbs.lock.Lock()
427 defer cbs.lock.Unlock()
428 return allocateCallback(val, nil)
429}
430
431func compileFastCallback(fn func(*CallbackArgs) uintptr) uintptr {
432 if fn == nil {
433 panic("purego: function must not be nil")
434 }
435 cbs.lock.Lock()
436 defer cbs.lock.Unlock()
437 return allocateCallback(reflect.Value{}, fn)
438}
439
440// allocateCallback is called with cbs.lock held.
441func allocateCallback(val reflect.Value, fast func(*CallbackArgs) uintptr) uintptr {
442 index := -1
443 reused := false
444 for i := range cbs.holes {
445 index = i
446 delete(cbs.holes, i)
447 reused = true
448 break
449 }
450 if index < 0 {
451 remaining := maxCB - cbs.numFn
452 if remaining <= 0 {
453 recordCallbackLedger("exhausted", -1, 0, 0, val, false, "maximum callbacks reached")
454 if val.IsValid() {
455 traceCallbackAllocation("exhausted", val, 0)
456 }
457 panic("purego: the maximum number of callbacks has been reached")
458 }
459 if val.IsValid() && (remaining <= 100 || remaining%250 == 0) {
460 traceCallbackAllocation("allocated", val, remaining)
461 }
462 index = cbs.numFn
463 cbs.numFn++
464 }
465 cbs.funcs[index] = val
466 cbs.fast[index] = fast
467 addr := callbackasmAddr(index)
468 cbs.knownIdx[addr] = index
469 if reused {
470 recordCallbackLedger("reuse", index, addr, 0, val, true, "")
471 } else {
472 recordCallbackLedger("alloc", index, addr, 0, val, false, "")
473 }
474 return addr
475}
476
477const ptrSize = unsafe.Sizeof((*int)(nil))
478
479const callbackMaxFrame = 64 * ptrSize
480
481// callbackasm is implemented in zcallback_GOOS_GOARCH.s
482//
483//go:linkname __callbackasm callbackasm
484var __callbackasm byte
485var callbackasmABI0 = uintptr(unsafe.Pointer(&__callbackasm))
486
487// callbackWrap_call allows the calling of the ABIInternal wrapper
488// which is required for runtime.cgocallback without the
489// <ABIInternal> tag which is only allowed in the runtime.
490// This closure is used inside sys_darwin_GOARCH.s
491var callbackWrap_call = callbackWrap
492
493// noescape is runtime.noescape; the synchronous callee must not retain p.
494//go:noescape
495//go:linkname noescape runtime.noescape
496func noescape(p unsafe.Pointer) unsafe.Pointer
497
498// callbackWrap is called by assembly code which determines which Go function to call.
499// This function takes the arguments and passes them to the Go function and returns the result.
500func callbackWrap(a *callbackArgs) {
501 // Slots are published under the lock before the C pointer is handed out.
502 // A caller must not invoke a released callback while UnrefCallback reuses
503 // its slot; while callable, fast slots are never cleared.
504 if f := cbs.fast[a.index]; f != nil {
505 ca := CallbackArgs{frame: a.args, intFrame: a.intFrame(), stackFrame: a.stackFrame()}
506 a.result[0] = f((*CallbackArgs)(noescape(unsafe.Pointer(&ca))))
507 return
508 }
509 cbs.lock.Lock()
510 fn := cbs.funcs[a.index]
511 cbs.lock.Unlock()
512 fnType := fn.Type()
513 args := make([]reflect.Value, fnType.NumIn())
514 frame := (*[callbackMaxFrame]uintptr)(a.args)
515 // stackFrame points to stack-passed arguments. On most architectures this is
516 // contiguous with frame (after register args), but on ppc64le it's separate.
517 var stackFrame *[callbackMaxFrame]uintptr
518 var intFrame *[callbackMaxFrame]uintptr
519 if sf := a.stackFrame(); sf != nil {
520 // Only ppc64le uses separate stackArgs pointer due to NOSPLIT constraints
521 stackFrame = (*[callbackMaxFrame]uintptr)(sf)
522 }
523 if intf := a.intFrame(); intf != nil {
524 intFrame = (*[callbackMaxFrame]uintptr)(intf)
525 }
526 // floatsN and intsN track the number of register slots used, not argument count.
527 // This distinction matters on ARM32 where float64 uses 2 slots (32-bit registers).
528 var floatsN int
529 var intsN int
530 // On amd64/loong64/ppc64le/riscv64/s390x, when returning a struct larger than
531 // maxRegAllocStructSize, the caller passes a hidden pointer in the first integer
532 // register. Skip it to avoid misreading it as the first function argument.
533 if (runtime.GOARCH == "amd64" || runtime.GOARCH == "loong64" || runtime.GOARCH == "riscv64" || runtime.GOARCH == "s390x") &&
534 fnType.NumOut() == 1 && fnType.Out(0).Kind() == reflect.Struct &&
535 fnType.Out(0).Size() > maxRegAllocStructSize {
536 intsN = 1
537 }
538 // stackSlot points to the index into frame (or stackFrame) of the current stack element.
539 // When stackFrame is nil, stack begins after float and integer registers in frame.
540 // When stackFrame is not nil (ppc64le), stackSlot indexes into stackFrame starting at 0.
541 stackSlot := numOfIntegerRegisters() + numOfFloatRegisters()
542 if stackFrame != nil {
543 // ppc64le: stackArgs is a separate pointer, indices start at 0
544 stackSlot = 0
545 }
546 // stackByteOffset tracks the byte offset within the stack area for Darwin ARM64
547 // tight packing. On Darwin ARM64, C passes small types packed on the stack.
548 stackByteOffset := uintptr(0)
549 for i := range args {
550 // slots is the number of pointer-sized slots the argument takes
551 inType := fnType.In(i)
552 slots := int((inType.Size() + ptrSize - 1) / ptrSize)
553 switch inType.Kind() {
554 case reflect.Float32, reflect.Float64:
555 if isARMSoftFloat() {
556 // we should restore from integer slot, can skip unnecessary branching here
557 if isARMPaddingNeeded(inType, -1, intsN) {
558 intsN++
559 }
560 if intsN+slots <= numOfIntegerRegisters() {
561 // the integers begin after the floats in frame
562 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[intsN+numOfFloatRegisters()])).Elem()
563 intsN += slots
564 continue
565 }
566 if isARMPaddingNeeded(inType, -1, stackSlot) {
567 stackSlot++
568 }
569 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
570 stackSlot += slots
571 intsN += slots
572 continue
573 }
574
575 if floatsN+slots > numOfFloatRegisters() {
576 if isDarwin && runtime.GOARCH == "arm64" {
577 // Darwin ARM64: read from packed stack with proper alignment
578 args[i] = callbackArgFromStack(a.args, stackSlot, &stackByteOffset, inType)
579 } else if stackFrame != nil {
580 // ppc64le/s390x: stack args are in separate stackFrame
581 switch runtime.GOARCH {
582 case "ppc64le":
583 args[i] = callbackFloatFromDoubleSlot(unsafe.Pointer(&stackFrame[stackSlot]), inType)
584 case "s390x":
585 // s390x big-endian: sub-8-byte values are right-justified
586 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&stackFrame[stackSlot]), inType)
587 default:
588 args[i] = reflect.NewAt(inType, unsafe.Pointer(&stackFrame[stackSlot])).Elem()
589 }
590 stackSlot += slots
591 } else if isARMFloatPaddingNeeded(inType, -1, stackSlot) {
592 stackSlot++
593 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
594 stackSlot += slots
595 } else {
596 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
597 stackSlot += slots
598 }
599 } else {
600 switch runtime.GOARCH {
601 case "ppc64le":
602 args[i] = callbackFloatFromDoubleSlot(unsafe.Pointer(&frame[floatsN]), inType)
603 case "s390x":
604 // s390x big-endian: float32 is right-justified in 8-byte FPR slot
605 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&frame[floatsN]), inType)
606 default:
607 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[floatsN])).Elem()
608 }
609 }
610 floatsN += slots
611 case reflect.String:
612 slots = 1
613 var ptr uintptr
614 if intsN+slots > numOfIntegerRegisters() {
615 if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
616 ptr = uintptr(callbackArgFromStack(a.args, stackSlot, &stackByteOffset, reflect.TypeOf(uintptr(0))).Uint())
617 } else if stackFrame != nil {
618 if runtime.GOARCH == "s390x" {
619 ptr = uintptr(callbackArgFromSlotBigEndian(unsafe.Pointer(&stackFrame[stackSlot]), reflect.TypeOf(uintptr(0))).Uint())
620 } else {
621 ptr = *(*uintptr)(unsafe.Pointer(&stackFrame[stackSlot]))
622 }
623 stackSlot += slots
624 } else {
625 ptr = frame[stackSlot]
626 stackSlot += slots
627 }
628 } else {
629 pos := intsN + numOfFloatRegisters()
630 ptr = frame[pos]
631 }
632 intsN += slots
633 args[i] = reflect.ValueOf(internalstrings.GoString(ptr))
634 continue
635 case reflect.Struct:
636 if i == 0 && inType.AssignableTo(reflect.TypeFor[CDecl]()) {
637 args[i] = reflect.Zero(inType)
638 continue
639 }
640 if inType.Size() == 0 {
641 args[i] = reflect.New(inType).Elem()
642 continue
643 }
644 args[i] = getCallbackStruct(inType, a.args, &floatsN, &intsN, &stackSlot, &stackByteOffset)
645 continue
646 default:
647 if isARMPaddingNeeded(inType, -1, intsN) {
648 intsN++
649 }
650 if intsN+slots > numOfIntegerRegisters() {
651 if isDarwin && runtime.GOARCH == "arm64" {
652 // Darwin ARM64: read from packed stack with proper alignment
653 args[i] = callbackArgFromStack(a.args, stackSlot, &stackByteOffset, inType)
654 } else if stackFrame != nil {
655 // ppc64le/s390x: stack args are in separate stackFrame
656 if runtime.GOARCH == "s390x" {
657 // s390x big-endian: sub-8-byte values are right-justified
658 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&stackFrame[stackSlot]), inType)
659 } else {
660 args[i] = reflect.NewAt(inType, unsafe.Pointer(&stackFrame[stackSlot])).Elem()
661 }
662 stackSlot += slots
663 } else if isARMPaddingNeeded(inType, -1, stackSlot) {
664 stackSlot++
665 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
666 stackSlot += slots
667 } else {
668 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
669 stackSlot += slots
670 }
671 } else {
672 if intFrame != nil {
673 args[i] = reflect.NewAt(inType, unsafe.Pointer(&intFrame[intsN])).Elem()
674 } else {
675 // the integers begin after the floats in frame
676 pos := intsN + numOfFloatRegisters()
677 if runtime.GOARCH == "s390x" {
678 // s390x big-endian: sub-8-byte values are right-justified in GPR slot
679 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&frame[pos]), inType)
680 } else {
681 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[pos])).Elem()
682 }
683 }
684 }
685 intsN += slots
686 }
687 }
688 ret := fn.Call(args)
689 if len(ret) > 0 {
690 switch k := ret[0].Kind(); k {
691 case reflect.Uint64:
692 a.setUint64Result(ret[0].Uint())
693 case reflect.Uint, reflect.Uint32, reflect.Uint16, reflect.Uint8, reflect.Uintptr:
694 a.result[0] = uintptr(ret[0].Uint())
695 case reflect.Int64:
696 a.setInt64Result(ret[0].Int())
697 case reflect.Int, reflect.Int32, reflect.Int16, reflect.Int8:
698 a.result[0] = uintptr(ret[0].Int())
699 case reflect.Bool:
700 if ret[0].Bool() {
701 a.result[0] = 1
702 } else {
703 a.result[0] = 0
704 }
705 case reflect.Pointer:
706 a.result[0] = ret[0].Pointer()
707 case reflect.UnsafePointer:
708 a.result[0] = ret[0].Pointer()
709 case reflect.Struct:
710 setStruct(a, ret[0])
711 default:
712 panic("purego: unsupported kind: " + k.String())
713 }
714 }
715}
716
717// callbackArgFromStack reads an argument from the tightly-packed stack area on Darwin ARM64.
718// The C ABI on Darwin ARM64 packs small types on the stack without padding to 8 bytes.
719// This function handles proper alignment and advances stackByteOffset accordingly.
720func callbackArgFromStack(argsBase unsafe.Pointer, stackSlot int, stackByteOffset *uintptr, inType reflect.Type) reflect.Value {
721 // Calculate base address of stack area (after float and int registers)
722 stackBase := unsafe.Add(argsBase, stackSlot*int(ptrSize))
723
724 // Get type's natural alignment
725 align := uintptr(inType.Align())
726 size := inType.Size()
727
728 // Align the offset
729 if *stackByteOffset%align != 0 {
730 *stackByteOffset = (*stackByteOffset + align - 1) &^ (align - 1)
731 }
732
733 // Read value at aligned offset
734 ptr := unsafe.Add(stackBase, *stackByteOffset)
735 *stackByteOffset += size
736
737 return reflect.NewAt(inType, ptr).Elem()
738}
739
740// callbackFloatFromDoubleSlot reads a floating-point callback argument from an
741// 8-byte register or stack slot on ppc64le, where a single-precision value is
742// held in double-precision format.
743func callbackFloatFromDoubleSlot(slotPtr unsafe.Pointer, inType reflect.Type) reflect.Value {
744 if inType.Kind() != reflect.Float32 {
745 return reflect.NewAt(inType, slotPtr).Elem()
746 }
747 v := reflect.New(inType).Elem()
748 v.SetFloat(math.Float64frombits(*(*uint64)(slotPtr)))
749 return v
750}
751
752// callbackArgFromSlotBigEndian reads an argument from an 8-byte slot on big-endian architectures.
753// On s390x:
754// - Integer types are right-justified in GPRs: sub-8-byte values are at offset (8 - size)
755// - Float32 in FPRs is left-justified: stored in upper 32 bits, so at offset 0
756// - Float64 occupies the full 8-byte slot
757func callbackArgFromSlotBigEndian(slotPtr unsafe.Pointer, inType reflect.Type) reflect.Value {
758 size := inType.Size()
759 if size >= 8 {
760 // 8-byte values occupy the entire slot
761 return reflect.NewAt(inType, slotPtr).Elem()
762 }
763 // Float32 is left-justified in FPRs (upper 32 bits), so offset is 0
764 if inType.Kind() == reflect.Float32 {
765 return reflect.NewAt(inType, slotPtr).Elem()
766 }
767 // Integer types are right-justified: offset = 8 - size
768 offset := 8 - size
769 ptr := unsafe.Add(slotPtr, offset)
770 return reflect.NewAt(inType, ptr).Elem()
771}
772
773// callbackasmAddr returns address of runtime.callbackasm
774// function adjusted by i.
775// On x86 and amd64, runtime.callbackasm is a series of CALL instructions,
776// and we want callback to arrive at
777// correspondent call instruction instead of start of
778// runtime.callbackasm.
779// On ARM, runtime.callbackasm is a series of mov and branch instructions.
780// R12 is loaded with the callback index. Each entry is two instructions,
781// hence 8 bytes.
782func callbackasmAddr(i int) uintptr {
783 var entrySize int
784 switch runtime.GOARCH {
785 default:
786 panic("purego: unsupported architecture")
787 case "amd64":
788 // On amd64, each callback entry is just a CALL instruction (5 bytes)
789 entrySize = 5
790 case "386":
791 // On 386, each callback entry is MOVL $imm, CX (5 bytes) + JMP (5 bytes)
792 entrySize = 10
793 case "arm", "arm64", "loong64", "ppc64le", "riscv64":
794 // On ARM, ARM64, Loong64, PPC64LE and RISCV64, each entry is a MOV instruction
795 // followed by a branch instruction
796 entrySize = 8
797 case "s390x":
798 // On S390X, each entry is LGHI (4 bytes) + JG (6 bytes)
799 entrySize = 10
800 }
801 return callbackasmABI0 + uintptr(i*entrySize)
802}