1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: 2022 The Ebitengine Authors
3
4//go:build darwin || freebsd || (linux && (386 || amd64 || arm || arm64 || loong64 || ppc64le || riscv64 || (s390x && (cgo || go1.27)))) || netbsd
5
6package purego
7
8import (
9 "encoding/json"
10 "errors"
11 "fmt"
12 "math"
13 "os"
14 "reflect"
15 "runtime"
16 "runtime/debug"
17 "strings"
18 "sync"
19 "time"
20 "unsafe"
21
22 internalstrings "github.com/bnema/purego/internal/strings"
23)
24
25var syscallXABI0 uintptr
26
27func syscall_syscallN(fn uintptr, args ...uintptr) (r1, r2, err uintptr) {
28 panic("purego: syscall_syscallN is only supported on windows")
29}
30
31// NewCallback converts a Go function to a function pointer conforming to the C calling convention.
32// This is useful when interoperating with C code requiring callbacks. The argument is expected to be a
33// function with zero or one uintptr-sized result. The function must not have arguments with size larger than the size
34// of uintptr. Only a limited number of callbacks may be created in a single Go process, and any memory allocated
35// for these callbacks is never released. At least 1024 callbacks can always be created. Although this function
36// provides similar functionality to windows.NewCallback it is distinct.
37//
38// Every call to NewCallback creates a new callback even for the same function value, so passing a Go callback to C
39// inside a loop (e.g. a qsort comparator) keeps consuming callbacks and eventually panics once they are exhausted.
40// The same happens when a func value is passed to a C function, as [RegisterFunc] creates a new callback for each
41// call. Create the callback once with NewCallback and reuse the returned pointer instead.
42func NewCallback(fn any) uintptr {
43 ty := reflect.TypeOf(fn)
44 if ty == nil || ty.Kind() != reflect.Func {
45 panic("purego: the type must be a function but was not")
46 }
47 for i := range ty.NumIn() {
48 in := ty.In(i)
49 if !in.AssignableTo(reflect.TypeFor[CDecl]()) {
50 continue
51 }
52 if i != 0 {
53 panic("purego: CDecl must be the first argument")
54 }
55 }
56 return compileCallback(fn)
57}
58
59// NewCallbackFnPtr converts a Go function pointer to a function pointer conforming to the C calling convention.
60// Calling this function multiple times with the same function pointer returns the original callback address.
61func NewCallbackFnPtr(fnPtr any) uintptr {
62 val := reflect.ValueOf(fnPtr)
63 if val.IsNil() {
64 panic("purego: function must not be nil")
65 }
66 if val.Kind() != reflect.Ptr || val.Elem().Kind() != reflect.Func {
67 panic("purego: the type must be a function pointer but was not")
68 }
69 if addr, idx, ok := getCallbackByFnPtr(val); ok {
70 cbs.lock.Lock()
71 recordCallbackLedger("dedupe", idx, addr, val.Pointer(), val.Elem(), false, "")
72 cbs.lock.Unlock()
73 return addr
74 }
75 addr := compileCallback(val.Elem().Interface())
76 cbs.lock.Lock()
77 cbs.knownFnPtr[val.Pointer()] = addr
78 if idx, ok := cbs.knownIdx[addr]; ok {
79 cbs.fnPtrKeys[idx] = val.Pointer()
80 recordCallbackLedger("fnptr-bind", idx, addr, val.Pointer(), val.Elem(), false, "")
81 }
82 cbs.lock.Unlock()
83 return addr
84}
85
86// UnrefCallback unreferences a callback created by NewCallback or NewCallbackFnPtr and frees its slot.
87func UnrefCallback(cb uintptr) error {
88 cbs.lock.Lock()
89 defer cbs.lock.Unlock()
90 idx, ok := cbs.knownIdx[cb]
91 if !ok {
92 recordCallbackLedger("release-miss", -1, cb, 0, reflect.Value{}, false, "callback not found")
93 return errors.New("callback not found")
94 }
95 val := cbs.funcs[idx]
96 key := cbs.fnPtrKeys[idx]
97 if key != 0 {
98 delete(cbs.knownFnPtr, key)
99 cbs.fnPtrKeys[idx] = 0
100 }
101 delete(cbs.knownIdx, cb)
102 cbs.holes[idx] = struct{}{}
103 cbs.funcs[idx] = reflect.Value{}
104 recordCallbackLedger("release", idx, cb, key, val, false, "")
105 return nil
106}
107
108// UnrefCallbackFnPtr unreferences a callback previously created via NewCallbackFnPtr.
109func UnrefCallbackFnPtr(fnPtr any) error {
110 val := reflect.ValueOf(fnPtr)
111 if val.IsNil() {
112 panic("purego: function must not be nil")
113 }
114 if val.Kind() != reflect.Ptr || val.Elem().Kind() != reflect.Func {
115 panic("purego: the type must be a function pointer but was not")
116 }
117 cbs.lock.Lock()
118 defer cbs.lock.Unlock()
119 addr, ok := cbs.knownFnPtr[val.Pointer()]
120 if !ok {
121 recordCallbackLedger("release-fnptr-miss", -1, 0, val.Pointer(), val.Elem(), false, "callback not found")
122 return errors.New("callback not found")
123 }
124 idx, ok := cbs.knownIdx[addr]
125 if !ok {
126 delete(cbs.knownFnPtr, val.Pointer())
127 recordCallbackLedger("release-fnptr-miss", -1, addr, val.Pointer(), val.Elem(), false, "callback not found")
128 return errors.New("callback not found")
129 }
130 callbackVal := cbs.funcs[idx]
131 delete(cbs.knownFnPtr, val.Pointer())
132 delete(cbs.knownIdx, addr)
133 cbs.fnPtrKeys[idx] = 0
134 cbs.holes[idx] = struct{}{}
135 cbs.funcs[idx] = reflect.Value{}
136 recordCallbackLedger("release-fnptr", idx, addr, val.Pointer(), callbackVal, false, "")
137 return nil
138}
139
140// maxCb is the maximum number of callbacks
141// only increase this if you have added more to the callbackasm function
142const maxCB = 2000
143
144var cbs = struct {
145 lock sync.RWMutex
146 numFn int // the highest allocated callback index + 1
147 holes map[int]struct{} // reusable callback slots
148 funcs [maxCB]reflect.Value // the saved callbacks
149 knownIdx map[uintptr]int // callback address -> slot index
150 knownFnPtr map[uintptr]uintptr // function pointer variable address -> callback address
151 fnPtrKeys [maxCB]uintptr // slot index -> function pointer variable address
152 ledgerSeq uint64 // monotonic callback ledger event sequence
153}{
154 holes: make(map[int]struct{}),
155 knownIdx: make(map[uintptr]int, maxCB),
156 knownFnPtr: make(map[uintptr]uintptr, maxCB),
157}
158
159func getCallbackByFnPtr(val reflect.Value) (uintptr, int, bool) {
160 cbs.lock.RLock()
161 defer cbs.lock.RUnlock()
162 addr, ok := cbs.knownFnPtr[val.Pointer()]
163 if !ok {
164 return 0, -1, false
165 }
166 idx, ok := cbs.knownIdx[addr]
167 if !ok {
168 return addr, -1, true
169 }
170 return addr, idx, true
171}
172
173type callbackLedgerEvent struct {
174 Marker string `json:"marker"`
175 PID int `json:"pid"`
176 Seq uint64 `json:"seq"`
177 Time string `json:"time"`
178 Event string `json:"event"`
179 Index int `json:"index"`
180 Addr string `json:"addr,omitempty"`
181 FnPtrKey string `json:"fn_ptr_key,omitempty"`
182 Type string `json:"type,omitempty"`
183 Family string `json:"family"`
184 StackKey string `json:"stack_key"`
185 NumFn int `json:"num_fn"`
186 Occupied int `json:"occupied"`
187 Reusable int `json:"reusable"`
188 KnownFnPtr int `json:"known_fn_ptr"`
189 Remaining int `json:"remaining"`
190 Max int `json:"max"`
191 ReusedSlot bool `json:"reused_slot,omitempty"`
192 Stack string `json:"stack,omitempty"`
193 Error string `json:"error,omitempty"`
194}
195
196func callbackEnvEnabled(name string) bool {
197 switch strings.ToLower(os.Getenv(name)) {
198 case "1", "true", "yes", "on":
199 return true
200 default:
201 return false
202 }
203}
204
205func callbackLedgerEnabled() bool {
206 return callbackEnvEnabled("PUREGO_CALLBACK_LEDGER")
207}
208
209func callbackLedgerStackEnabled() bool {
210 return callbackEnvEnabled("PUREGO_CALLBACK_LEDGER_STACK")
211}
212
213func callbackTraceEnabled() bool {
214 return callbackEnvEnabled("PUREGO_CALLBACK_TRACE")
215}
216
217func callbackLedgerPath() string {
218 if path := os.Getenv("PUREGO_CALLBACK_LEDGER_FILE"); path != "" {
219 return path
220 }
221 return fmt.Sprintf("/tmp/purego-callback-ledger-%d.jsonl", os.Getpid())
222}
223
224func typeName(val reflect.Value) string {
225 if !val.IsValid() {
226 return ""
227 }
228 return val.Type().String()
229}
230
231func hexUintptr(v uintptr) string {
232 if v == 0 {
233 return ""
234 }
235 return fmt.Sprintf("0x%x", v)
236}
237
238func classifyCallbackStack(stack string) (family, stackKey string) {
239 family = "unknown"
240 stackKey = "unknown"
241 lines := strings.Split(stack, "\n")
242 for i, line := range lines {
243 if line == "" || strings.HasPrefix(line, "goroutine ") || strings.Contains(line, "runtime/debug.Stack") || strings.Contains(line, "recordCallbackLedger") || strings.Contains(line, "traceCallbackAllocation") {
244 continue
245 }
246 if strings.Contains(line, "github.com/bnema/purego.") || strings.Contains(line, "github.com/ebitengine/purego.") {
247 continue
248 }
249 if strings.HasPrefix(strings.TrimSpace(line), "/") || strings.HasPrefix(strings.TrimSpace(line), "../") {
250 continue
251 }
252 stackKey = strings.TrimSpace(line)
253 for j := i + 1; j < len(lines); j++ {
254 fileLine := strings.TrimSpace(lines[j])
255 if fileLine == "" {
256 continue
257 }
258 family = classifyCallbackFamily(fileLine + " " + stackKey)
259 return family, stackKey
260 }
261 family = classifyCallbackFamily(stackKey)
262 return family, stackKey
263 }
264 return family, stackKey
265}
266
267func classifyCallbackFamily(s string) string {
268 switch {
269 case strings.Contains(s, "purego-cef2gtk"):
270 return "purego-cef2gtk"
271 case strings.Contains(s, "purego-cef"):
272 return "purego-cef"
273 case strings.Contains(s, "puregotk"):
274 return "puregotk"
275 case strings.Contains(s, "dumber"):
276 return "dumber"
277 case strings.Contains(s, "github.com/bnema/purego") || strings.Contains(s, "/purego/"):
278 return "purego"
279 default:
280 return "unknown"
281 }
282}
283
284func recordCallbackLedger(event string, idx int, addr uintptr, fnPtrKey uintptr, val reflect.Value, reused bool, errText string) {
285 if !callbackLedgerEnabled() {
286 return
287 }
288 stack := ""
289 family := "unknown"
290 stackKey := "unknown"
291 if callbackLedgerStackEnabled() {
292 stack = string(debug.Stack())
293 family, stackKey = classifyCallbackStack(stack)
294 }
295 cbs.ledgerSeq++
296 occupied := cbs.numFn - len(cbs.holes)
297 entry := callbackLedgerEvent{
298 Marker: "PUREGO-CALLBACK-LEDGER",
299 PID: os.Getpid(),
300 Seq: cbs.ledgerSeq,
301 Time: time.Now().Format(time.RFC3339Nano),
302 Event: event,
303 Index: idx,
304 Addr: hexUintptr(addr),
305 FnPtrKey: hexUintptr(fnPtrKey),
306 Type: typeName(val),
307 Family: family,
308 StackKey: stackKey,
309 NumFn: cbs.numFn,
310 Occupied: occupied,
311 Reusable: len(cbs.holes),
312 KnownFnPtr: len(cbs.knownFnPtr),
313 Remaining: maxCB - occupied,
314 Max: maxCB,
315 ReusedSlot: reused,
316 Error: errText,
317 }
318 if callbackLedgerStackEnabled() {
319 entry.Stack = stack
320 }
321 payload, err := json.Marshal(entry)
322 if err != nil {
323 return
324 }
325 payload = append(payload, '\n')
326 if f, err := os.OpenFile(callbackLedgerPath(), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600); err == nil {
327 _, _ = f.Write(payload)
328 _ = f.Close()
329 }
330}
331
332func traceCallbackAllocation(event string, val reflect.Value, remaining int) {
333 if !callbackTraceEnabled() {
334 return
335 }
336 message := fmt.Sprintf("PUREGO-CALLBACK-TRACE event=%s remaining=%d live=%d type=%s\n%s\n", event, remaining, maxCB-remaining, val.Type(), debug.Stack())
337 fmt.Fprint(os.Stderr, message)
338 traceFile := os.Getenv("PUREGO_CALLBACK_TRACE_FILE")
339 if traceFile == "" {
340 traceFile = fmt.Sprintf("/tmp/purego-callback-trace-%d.log", os.Getpid())
341 }
342 if f, err := os.OpenFile(traceFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600); err == nil {
343 _, _ = f.WriteString(message)
344 _ = f.Close()
345 }
346}
347
348func compileCallback(fn any) uintptr {
349 val := reflect.ValueOf(fn)
350 if val.Kind() != reflect.Func {
351 panic("purego: the type must be a function but was not")
352 }
353 if val.IsNil() {
354 panic("purego: function must not be nil")
355 }
356 ty := val.Type()
357 for i := range ty.NumIn() {
358 in := ty.In(i)
359 switch in.Kind() {
360 case reflect.Struct:
361 if i == 0 && in.AssignableTo(reflect.TypeFor[CDecl]()) {
362 continue
363 }
364 ensureCallbackStructSupported()
365 checkStructFieldsSupported(in)
366 continue
367 case reflect.Interface, reflect.Func, reflect.Slice,
368 reflect.Chan, reflect.Complex64, reflect.Complex128,
369 reflect.Map, reflect.Invalid:
370 panic("purego: unsupported argument type: " + in.Kind().String())
371 }
372 }
373output:
374 switch {
375 case ty.NumOut() == 1:
376 switch ty.Out(0).Kind() {
377 case reflect.Struct:
378 ensureCallbackStructSupported()
379 checkStructFieldsSupported(ty.Out(0))
380 break output
381 case reflect.Pointer, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
382 reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
383 reflect.Bool, reflect.UnsafePointer:
384 break output
385 }
386 panic("purego: unsupported return type: " + ty.String())
387 case ty.NumOut() > 1:
388 panic("purego: callbacks can only have one return")
389 }
390 cbs.lock.Lock()
391 defer cbs.lock.Unlock()
392 index := -1
393 reused := false
394 for i := range cbs.holes {
395 index = i
396 delete(cbs.holes, i)
397 reused = true
398 break
399 }
400 if index < 0 {
401 remaining := maxCB - cbs.numFn
402 if remaining <= 0 {
403 recordCallbackLedger("exhausted", -1, 0, 0, val, false, "maximum callbacks reached")
404 traceCallbackAllocation("exhausted", val, 0)
405 panic("purego: the maximum number of callbacks has been reached")
406 }
407 if remaining <= 100 || remaining%250 == 0 {
408 traceCallbackAllocation("allocated", val, remaining)
409 }
410 index = cbs.numFn
411 cbs.numFn++
412 }
413 cbs.funcs[index] = val
414 addr := callbackasmAddr(index)
415 cbs.knownIdx[addr] = index
416 if reused {
417 recordCallbackLedger("reuse", index, addr, 0, val, true, "")
418 } else {
419 recordCallbackLedger("alloc", index, addr, 0, val, false, "")
420 }
421 return addr
422}
423
424const ptrSize = unsafe.Sizeof((*int)(nil))
425
426const callbackMaxFrame = 64 * ptrSize
427
428// callbackasm is implemented in zcallback_GOOS_GOARCH.s
429//
430//go:linkname __callbackasm callbackasm
431var __callbackasm byte
432var callbackasmABI0 = uintptr(unsafe.Pointer(&__callbackasm))
433
434// callbackWrap_call allows the calling of the ABIInternal wrapper
435// which is required for runtime.cgocallback without the
436// <ABIInternal> tag which is only allowed in the runtime.
437// This closure is used inside sys_darwin_GOARCH.s
438var callbackWrap_call = callbackWrap
439
440// callbackWrap is called by assembly code which determines which Go function to call.
441// This function takes the arguments and passes them to the Go function and returns the result.
442func callbackWrap(a *callbackArgs) {
443 cbs.lock.Lock()
444 fn := cbs.funcs[a.index]
445 cbs.lock.Unlock()
446 fnType := fn.Type()
447 args := make([]reflect.Value, fnType.NumIn())
448 frame := (*[callbackMaxFrame]uintptr)(a.args)
449 // stackFrame points to stack-passed arguments. On most architectures this is
450 // contiguous with frame (after register args), but on ppc64le it's separate.
451 var stackFrame *[callbackMaxFrame]uintptr
452 var intFrame *[callbackMaxFrame]uintptr
453 if sf := a.stackFrame(); sf != nil {
454 // Only ppc64le uses separate stackArgs pointer due to NOSPLIT constraints
455 stackFrame = (*[callbackMaxFrame]uintptr)(sf)
456 }
457 if intf := a.intFrame(); intf != nil {
458 intFrame = (*[callbackMaxFrame]uintptr)(intf)
459 }
460 // floatsN and intsN track the number of register slots used, not argument count.
461 // This distinction matters on ARM32 where float64 uses 2 slots (32-bit registers).
462 var floatsN int
463 var intsN int
464 // On amd64/loong64/ppc64le/riscv64/s390x, when returning a struct larger than
465 // maxRegAllocStructSize, the caller passes a hidden pointer in the first integer
466 // register. Skip it to avoid misreading it as the first function argument.
467 if (runtime.GOARCH == "amd64" || runtime.GOARCH == "loong64" || runtime.GOARCH == "riscv64" || runtime.GOARCH == "s390x") &&
468 fnType.NumOut() == 1 && fnType.Out(0).Kind() == reflect.Struct &&
469 fnType.Out(0).Size() > maxRegAllocStructSize {
470 intsN = 1
471 }
472 // stackSlot points to the index into frame (or stackFrame) of the current stack element.
473 // When stackFrame is nil, stack begins after float and integer registers in frame.
474 // When stackFrame is not nil (ppc64le), stackSlot indexes into stackFrame starting at 0.
475 stackSlot := numOfIntegerRegisters() + numOfFloatRegisters()
476 if stackFrame != nil {
477 // ppc64le: stackArgs is a separate pointer, indices start at 0
478 stackSlot = 0
479 }
480 // stackByteOffset tracks the byte offset within the stack area for Darwin ARM64
481 // tight packing. On Darwin ARM64, C passes small types packed on the stack.
482 stackByteOffset := uintptr(0)
483 for i := range args {
484 // slots is the number of pointer-sized slots the argument takes
485 inType := fnType.In(i)
486 slots := int((inType.Size() + ptrSize - 1) / ptrSize)
487 switch inType.Kind() {
488 case reflect.Float32, reflect.Float64:
489 if isARMSoftFloat() {
490 // we should restore from integer slot, can skip unnecessary branching here
491 if isARMPaddingNeeded(inType, -1, intsN) {
492 intsN++
493 }
494 if intsN+slots <= numOfIntegerRegisters() {
495 // the integers begin after the floats in frame
496 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[intsN+numOfFloatRegisters()])).Elem()
497 intsN += slots
498 continue
499 }
500 if isARMPaddingNeeded(inType, -1, stackSlot) {
501 stackSlot++
502 }
503 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
504 stackSlot += slots
505 intsN += slots
506 continue
507 }
508
509 if floatsN+slots > numOfFloatRegisters() {
510 if isDarwin && runtime.GOARCH == "arm64" {
511 // Darwin ARM64: read from packed stack with proper alignment
512 args[i] = callbackArgFromStack(a.args, stackSlot, &stackByteOffset, inType)
513 } else if stackFrame != nil {
514 // ppc64le/s390x: stack args are in separate stackFrame
515 switch runtime.GOARCH {
516 case "ppc64le":
517 args[i] = callbackFloatFromDoubleSlot(unsafe.Pointer(&stackFrame[stackSlot]), inType)
518 case "s390x":
519 // s390x big-endian: sub-8-byte values are right-justified
520 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&stackFrame[stackSlot]), inType)
521 default:
522 args[i] = reflect.NewAt(inType, unsafe.Pointer(&stackFrame[stackSlot])).Elem()
523 }
524 stackSlot += slots
525 } else if isARMFloatPaddingNeeded(inType, -1, stackSlot) {
526 stackSlot++
527 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
528 stackSlot += slots
529 } else {
530 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
531 stackSlot += slots
532 }
533 } else {
534 switch runtime.GOARCH {
535 case "ppc64le":
536 args[i] = callbackFloatFromDoubleSlot(unsafe.Pointer(&frame[floatsN]), inType)
537 case "s390x":
538 // s390x big-endian: float32 is right-justified in 8-byte FPR slot
539 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&frame[floatsN]), inType)
540 default:
541 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[floatsN])).Elem()
542 }
543 }
544 floatsN += slots
545 case reflect.String:
546 slots = 1
547 var ptr uintptr
548 if intsN+slots > numOfIntegerRegisters() {
549 if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
550 ptr = uintptr(callbackArgFromStack(a.args, stackSlot, &stackByteOffset, reflect.TypeOf(uintptr(0))).Uint())
551 } else if stackFrame != nil {
552 if runtime.GOARCH == "s390x" {
553 ptr = uintptr(callbackArgFromSlotBigEndian(unsafe.Pointer(&stackFrame[stackSlot]), reflect.TypeOf(uintptr(0))).Uint())
554 } else {
555 ptr = *(*uintptr)(unsafe.Pointer(&stackFrame[stackSlot]))
556 }
557 stackSlot += slots
558 } else {
559 ptr = frame[stackSlot]
560 stackSlot += slots
561 }
562 } else {
563 pos := intsN + numOfFloatRegisters()
564 ptr = frame[pos]
565 }
566 intsN += slots
567 args[i] = reflect.ValueOf(internalstrings.GoString(ptr))
568 continue
569 case reflect.Struct:
570 if i == 0 && inType.AssignableTo(reflect.TypeFor[CDecl]()) {
571 args[i] = reflect.Zero(inType)
572 continue
573 }
574 if inType.Size() == 0 {
575 args[i] = reflect.New(inType).Elem()
576 continue
577 }
578 args[i] = getCallbackStruct(inType, a.args, &floatsN, &intsN, &stackSlot, &stackByteOffset)
579 continue
580 default:
581 if isARMPaddingNeeded(inType, -1, intsN) {
582 intsN++
583 }
584 if intsN+slots > numOfIntegerRegisters() {
585 if isDarwin && runtime.GOARCH == "arm64" {
586 // Darwin ARM64: read from packed stack with proper alignment
587 args[i] = callbackArgFromStack(a.args, stackSlot, &stackByteOffset, inType)
588 } else if stackFrame != nil {
589 // ppc64le/s390x: stack args are in separate stackFrame
590 if runtime.GOARCH == "s390x" {
591 // s390x big-endian: sub-8-byte values are right-justified
592 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&stackFrame[stackSlot]), inType)
593 } else {
594 args[i] = reflect.NewAt(inType, unsafe.Pointer(&stackFrame[stackSlot])).Elem()
595 }
596 stackSlot += slots
597 } else if isARMPaddingNeeded(inType, -1, stackSlot) {
598 stackSlot++
599 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
600 stackSlot += slots
601 } else {
602 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
603 stackSlot += slots
604 }
605 } else {
606 if intFrame != nil {
607 args[i] = reflect.NewAt(inType, unsafe.Pointer(&intFrame[intsN])).Elem()
608 } else {
609 // the integers begin after the floats in frame
610 pos := intsN + numOfFloatRegisters()
611 if runtime.GOARCH == "s390x" {
612 // s390x big-endian: sub-8-byte values are right-justified in GPR slot
613 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&frame[pos]), inType)
614 } else {
615 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[pos])).Elem()
616 }
617 }
618 }
619 intsN += slots
620 }
621 }
622 ret := fn.Call(args)
623 if len(ret) > 0 {
624 switch k := ret[0].Kind(); k {
625 case reflect.Uint64:
626 a.setUint64Result(ret[0].Uint())
627 case reflect.Uint, reflect.Uint32, reflect.Uint16, reflect.Uint8, reflect.Uintptr:
628 a.result[0] = uintptr(ret[0].Uint())
629 case reflect.Int64:
630 a.setInt64Result(ret[0].Int())
631 case reflect.Int, reflect.Int32, reflect.Int16, reflect.Int8:
632 a.result[0] = uintptr(ret[0].Int())
633 case reflect.Bool:
634 if ret[0].Bool() {
635 a.result[0] = 1
636 } else {
637 a.result[0] = 0
638 }
639 case reflect.Pointer:
640 a.result[0] = ret[0].Pointer()
641 case reflect.UnsafePointer:
642 a.result[0] = ret[0].Pointer()
643 case reflect.Struct:
644 setStruct(a, ret[0])
645 default:
646 panic("purego: unsupported kind: " + k.String())
647 }
648 }
649}
650
651// callbackArgFromStack reads an argument from the tightly-packed stack area on Darwin ARM64.
652// The C ABI on Darwin ARM64 packs small types on the stack without padding to 8 bytes.
653// This function handles proper alignment and advances stackByteOffset accordingly.
654func callbackArgFromStack(argsBase unsafe.Pointer, stackSlot int, stackByteOffset *uintptr, inType reflect.Type) reflect.Value {
655 // Calculate base address of stack area (after float and int registers)
656 stackBase := unsafe.Add(argsBase, stackSlot*int(ptrSize))
657
658 // Get type's natural alignment
659 align := uintptr(inType.Align())
660 size := inType.Size()
661
662 // Align the offset
663 if *stackByteOffset%align != 0 {
664 *stackByteOffset = (*stackByteOffset + align - 1) &^ (align - 1)
665 }
666
667 // Read value at aligned offset
668 ptr := unsafe.Add(stackBase, *stackByteOffset)
669 *stackByteOffset += size
670
671 return reflect.NewAt(inType, ptr).Elem()
672}
673
674// callbackFloatFromDoubleSlot reads a floating-point callback argument from an
675// 8-byte register or stack slot on ppc64le, where a single-precision value is
676// held in double-precision format.
677func callbackFloatFromDoubleSlot(slotPtr unsafe.Pointer, inType reflect.Type) reflect.Value {
678 if inType.Kind() != reflect.Float32 {
679 return reflect.NewAt(inType, slotPtr).Elem()
680 }
681 v := reflect.New(inType).Elem()
682 v.SetFloat(math.Float64frombits(*(*uint64)(slotPtr)))
683 return v
684}
685
686// callbackArgFromSlotBigEndian reads an argument from an 8-byte slot on big-endian architectures.
687// On s390x:
688// - Integer types are right-justified in GPRs: sub-8-byte values are at offset (8 - size)
689// - Float32 in FPRs is left-justified: stored in upper 32 bits, so at offset 0
690// - Float64 occupies the full 8-byte slot
691func callbackArgFromSlotBigEndian(slotPtr unsafe.Pointer, inType reflect.Type) reflect.Value {
692 size := inType.Size()
693 if size >= 8 {
694 // 8-byte values occupy the entire slot
695 return reflect.NewAt(inType, slotPtr).Elem()
696 }
697 // Float32 is left-justified in FPRs (upper 32 bits), so offset is 0
698 if inType.Kind() == reflect.Float32 {
699 return reflect.NewAt(inType, slotPtr).Elem()
700 }
701 // Integer types are right-justified: offset = 8 - size
702 offset := 8 - size
703 ptr := unsafe.Add(slotPtr, offset)
704 return reflect.NewAt(inType, ptr).Elem()
705}
706
707// callbackasmAddr returns address of runtime.callbackasm
708// function adjusted by i.
709// On x86 and amd64, runtime.callbackasm is a series of CALL instructions,
710// and we want callback to arrive at
711// correspondent call instruction instead of start of
712// runtime.callbackasm.
713// On ARM, runtime.callbackasm is a series of mov and branch instructions.
714// R12 is loaded with the callback index. Each entry is two instructions,
715// hence 8 bytes.
716func callbackasmAddr(i int) uintptr {
717 var entrySize int
718 switch runtime.GOARCH {
719 default:
720 panic("purego: unsupported architecture")
721 case "amd64":
722 // On amd64, each callback entry is just a CALL instruction (5 bytes)
723 entrySize = 5
724 case "386":
725 // On 386, each callback entry is MOVL $imm, CX (5 bytes) + JMP (5 bytes)
726 entrySize = 10
727 case "arm", "arm64", "loong64", "ppc64le", "riscv64":
728 // On ARM, ARM64, Loong64, PPC64LE and RISCV64, each entry is a MOV instruction
729 // followed by a branch instruction
730 entrySize = 8
731 case "s390x":
732 // On S390X, each entry is LGHI (4 bytes) + JG (6 bytes)
733 entrySize = 10
734 }
735 return callbackasmABI0 + uintptr(i*entrySize)
736}