1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: 2022 The Ebitengine Authors
3
4// TODO: remove s390x cgo dependency once golang/go#77449 is resolved
5//go:build darwin || freebsd || (linux && (386 || amd64 || arm || arm64 || loong64 || ppc64le || riscv64 || (cgo && s390x))) || netbsd
6
7package purego
8
9import (
10 "errors"
11 "reflect"
12 "runtime"
13 "sync"
14 "unsafe"
15
16 "github.com/bnema/purego/internal/strings"
17)
18
19var syscall15XABI0 uintptr
20
21func syscall_syscall15X(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15 uintptr) (r1, r2, err uintptr) {
22 args := &syscall15Args{
23 fn: fn,
24 a1: a1, a2: a2, a3: a3, a4: a4, a5: a5, a6: a6, a7: a7, a8: a8,
25 a9: a9, a10: a10, a11: a11, a12: a12, a13: a13, a14: a14, a15: a15,
26 f1: a1, f2: a2, f3: a3, f4: a4, f5: a5, f6: a6, f7: a7, f8: a8,
27 }
28
29 runtime_cgocall(syscall15XABI0, unsafe.Pointer(args))
30 return args.a1, args.a2, args.a3
31}
32
33// NewCallback converts a Go function to a function pointer conforming to the C calling convention.
34// This is useful when interoperating with C code requiring callbacks. The argument is expected to be a
35// function with zero or one uintptr-sized result. The function must not have arguments with size larger than the size
36// of uintptr. Only a limited number of callbacks may be created in a single Go process, and any memory allocated
37// for these callbacks is never released. At least 2000 callbacks can always be created. Although this function
38// provides similar functionality to windows.NewCallback it is distinct.
39func NewCallback(fn any) uintptr {
40 ty := reflect.TypeOf(fn)
41 for i := 0; i < ty.NumIn(); i++ {
42 in := ty.In(i)
43 if !in.AssignableTo(reflect.TypeOf(CDecl{})) {
44 continue
45 }
46 if i != 0 {
47 panic("purego: CDecl must be the first argument")
48 }
49 }
50 return compileCallback(fn)
51}
52
53// NewCallbackFnPtr converts a Go function pointer to a function pointer conforming to the C calling convention.
54// Calling this function multiple times with the same function pointer returns the original callback address.
55func NewCallbackFnPtr(fnPtr any) uintptr {
56 val := reflect.ValueOf(fnPtr)
57 if val.IsNil() {
58 panic("purego: function must not be nil")
59 }
60 if val.Kind() != reflect.Ptr || val.Elem().Kind() != reflect.Func {
61 panic("purego: the type must be a function pointer but was not")
62 }
63 if addr, ok := getCallbackByFnPtr(val); ok {
64 return addr
65 }
66 addr := compileCallback(val.Elem().Interface())
67 cbs.lock.Lock()
68 cbs.knownFnPtr[val.Pointer()] = addr
69 if idx, ok := cbs.knownIdx[addr]; ok {
70 cbs.fnPtrKeys[idx] = val.Pointer()
71 }
72 cbs.lock.Unlock()
73 return addr
74}
75
76// UnrefCallback unreferences a callback created by NewCallback or NewCallbackFnPtr and frees its slot.
77func UnrefCallback(cb uintptr) error {
78 cbs.lock.Lock()
79 defer cbs.lock.Unlock()
80 idx, ok := cbs.knownIdx[cb]
81 if !ok {
82 return errors.New("callback not found")
83 }
84 if key := cbs.fnPtrKeys[idx]; key != 0 {
85 delete(cbs.knownFnPtr, key)
86 cbs.fnPtrKeys[idx] = 0
87 }
88 delete(cbs.knownIdx, cb)
89 cbs.holes[idx] = struct{}{}
90 cbs.funcs[idx] = reflect.Value{}
91 cbs.argPools[idx] = nil
92 return nil
93}
94
95// UnrefCallbackFnPtr unreferences a callback previously created via NewCallbackFnPtr.
96func UnrefCallbackFnPtr(fnPtr any) error {
97 val := reflect.ValueOf(fnPtr)
98 if val.IsNil() {
99 panic("purego: function must not be nil")
100 }
101 if val.Kind() != reflect.Ptr || val.Elem().Kind() != reflect.Func {
102 panic("purego: the type must be a function pointer but was not")
103 }
104 cbs.lock.Lock()
105 defer cbs.lock.Unlock()
106 addr, ok := cbs.knownFnPtr[val.Pointer()]
107 if !ok {
108 return errors.New("callback not found")
109 }
110 idx, ok := cbs.knownIdx[addr]
111 if !ok {
112 delete(cbs.knownFnPtr, val.Pointer())
113 return errors.New("callback not found")
114 }
115 delete(cbs.knownFnPtr, val.Pointer())
116 delete(cbs.knownIdx, addr)
117 cbs.fnPtrKeys[idx] = 0
118 cbs.holes[idx] = struct{}{}
119 cbs.funcs[idx] = reflect.Value{}
120 cbs.argPools[idx] = nil
121 return nil
122}
123
124// maxCb is the maximum number of callbacks
125// only increase this if you have added more to the callbackasm function
126const maxCB = 2000
127
128var cbs = struct {
129 lock sync.RWMutex
130 numFn int // the highest allocated callback index + 1
131 holes map[int]struct{} // reusable callback slots
132 funcs [maxCB]reflect.Value // the saved callbacks
133 argPools [maxCB]*sync.Pool // pre-allocated argument buffers per callback
134 knownIdx map[uintptr]int // callback address -> slot index
135 knownFnPtr map[uintptr]uintptr // function pointer variable address -> callback address
136 fnPtrKeys [maxCB]uintptr // slot index -> function pointer variable address
137}{
138 holes: make(map[int]struct{}),
139 knownIdx: make(map[uintptr]int, maxCB),
140 knownFnPtr: make(map[uintptr]uintptr, maxCB),
141}
142
143func getCallbackByFnPtr(val reflect.Value) (uintptr, bool) {
144 cbs.lock.RLock()
145 defer cbs.lock.RUnlock()
146 addr, ok := cbs.knownFnPtr[val.Pointer()]
147 return addr, ok
148}
149
150func compileCallback(fn any) uintptr {
151 val := reflect.ValueOf(fn)
152 if val.Kind() != reflect.Func {
153 panic("purego: the type must be a function but was not")
154 }
155 if val.IsNil() {
156 panic("purego: function must not be nil")
157 }
158 ty := val.Type()
159 for i := 0; i < ty.NumIn(); i++ {
160 in := ty.In(i)
161 switch in.Kind() {
162 case reflect.Struct:
163 if i == 0 && in.AssignableTo(reflect.TypeOf(CDecl{})) {
164 continue
165 }
166 ensureStructSupported()
167 checkStructFieldsSupported(in)
168 continue
169 case reflect.Interface, reflect.Func, reflect.Slice,
170 reflect.Chan, reflect.Complex64, reflect.Complex128,
171 reflect.Map, reflect.Invalid:
172 panic("purego: unsupported argument type: " + in.Kind().String())
173 }
174 }
175output:
176 switch {
177 case ty.NumOut() == 1:
178 switch ty.Out(0).Kind() {
179 case reflect.Pointer, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
180 reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
181 reflect.Bool, reflect.UnsafePointer, reflect.Struct:
182 break output
183 }
184 panic("purego: unsupported return type: " + ty.String())
185 case ty.NumOut() > 1:
186 panic("purego: callbacks can only have one return")
187 }
188 cbs.lock.Lock()
189 defer cbs.lock.Unlock()
190 index := -1
191 for i := range cbs.holes {
192 index = i
193 delete(cbs.holes, i)
194 break
195 }
196 if index < 0 {
197 if cbs.numFn >= maxCB {
198 panic("purego: the maximum number of callbacks has been reached")
199 }
200 index = cbs.numFn
201 cbs.numFn++
202 }
203 cbs.funcs[index] = val
204 numIn := ty.NumIn()
205 cbs.argPools[index] = &sync.Pool{
206 New: func() any {
207 return make([]reflect.Value, numIn)
208 },
209 }
210 addr := callbackasmAddr(index)
211 cbs.knownIdx[addr] = index
212 return addr
213}
214
215const ptrSize = unsafe.Sizeof((*int)(nil))
216
217const callbackMaxFrame = 64 * ptrSize
218
219// callbackasm is implemented in zcallback_GOOS_GOARCH.s
220//
221//go:linkname __callbackasm callbackasm
222var __callbackasm byte
223var callbackasmABI0 = uintptr(unsafe.Pointer(&__callbackasm))
224
225// callbackWrap_call allows the calling of the ABIInternal wrapper
226// which is required for runtime.cgocallback without the
227// <ABIInternal> tag which is only allowed in the runtime.
228// This closure is used inside sys_darwin_GOARCH.s
229var callbackWrap_call = callbackWrap
230
231// callbackWrap is called by assembly code which determines which Go function to call.
232// This function takes the arguments and passes them to the Go function and returns the result.
233func callbackWrap(a *callbackArgs) {
234 cbs.lock.RLock()
235 fn := cbs.funcs[a.index]
236 pool := cbs.argPools[a.index]
237 cbs.lock.RUnlock()
238 fnType := fn.Type()
239 args := pool.Get().([]reflect.Value)
240 defer func() {
241 for i := range args {
242 args[i] = reflect.Value{}
243 }
244 pool.Put(args)
245 }()
246 frame := (*[callbackMaxFrame]uintptr)(a.args)
247 // stackFrame points to stack-passed arguments. On most architectures this is
248 // contiguous with frame (after register args), but on ppc64le it's separate.
249 var stackFrame *[callbackMaxFrame]uintptr
250 if sf := a.stackFrame(); sf != nil {
251 // Only ppc64le uses separate stackArgs pointer due to NOSPLIT constraints
252 stackFrame = (*[callbackMaxFrame]uintptr)(sf)
253 }
254 // floatsN and intsN track the number of register slots used, not argument count.
255 // This distinction matters on ARM32 where float64 uses 2 slots (32-bit registers).
256 var floatsN int
257 var intsN int
258 // On amd64/loong64/ppc64le/riscv64/s390x, when returning a struct larger than
259 // maxRegAllocStructSize, the caller passes a hidden pointer in the first integer
260 // register. Skip it to avoid misreading it as the first function argument.
261 if (runtime.GOARCH == "amd64" || runtime.GOARCH == "loong64" || runtime.GOARCH == "ppc64le" || runtime.GOARCH == "riscv64" || runtime.GOARCH == "s390x") &&
262 fnType.NumOut() == 1 && fnType.Out(0).Kind() == reflect.Struct &&
263 fnType.Out(0).Size() > maxRegAllocStructSize {
264 intsN = 1
265 }
266 // stackSlot points to the index into frame (or stackFrame) of the current stack element.
267 // When stackFrame is nil, stack begins after float and integer registers in frame.
268 // When stackFrame is not nil (ppc64le), stackSlot indexes into stackFrame starting at 0.
269 stackSlot := numOfIntegerRegisters() + numOfFloatRegisters()
270 if stackFrame != nil {
271 // ppc64le: stackArgs is a separate pointer, indices start at 0
272 stackSlot = 0
273 }
274 // stackByteOffset tracks the byte offset within the stack area for Darwin ARM64
275 // tight packing. On Darwin ARM64, C passes small types packed on the stack.
276 stackByteOffset := uintptr(0)
277 for i := range args {
278 // slots is the number of pointer-sized slots the argument takes
279 var slots int
280 inType := fnType.In(i)
281 switch inType.Kind() {
282 case reflect.Float32, reflect.Float64:
283 slots = int((fnType.In(i).Size() + ptrSize - 1) / ptrSize)
284 if floatsN+slots > numOfFloatRegisters() {
285 if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
286 // Darwin ARM64: read from packed stack with proper alignment
287 args[i] = callbackArgFromStack(a.args, stackSlot, &stackByteOffset, inType)
288 } else if stackFrame != nil {
289 // ppc64le/s390x: stack args are in separate stackFrame
290 if runtime.GOARCH == "s390x" {
291 // s390x big-endian: sub-8-byte values are right-justified
292 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&stackFrame[stackSlot]), inType)
293 } else {
294 args[i] = reflect.NewAt(inType, unsafe.Pointer(&stackFrame[stackSlot])).Elem()
295 }
296 stackSlot += slots
297 } else {
298 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
299 stackSlot += slots
300 }
301 } else {
302 if runtime.GOARCH == "s390x" {
303 // s390x big-endian: float32 is right-justified in 8-byte FPR slot
304 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&frame[floatsN]), inType)
305 } else {
306 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[floatsN])).Elem()
307 }
308 }
309 floatsN += slots
310 case reflect.String:
311 slots = 1
312 var ptr uintptr
313 if intsN+slots > numOfIntegerRegisters() {
314 if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
315 ptr = uintptr(callbackArgFromStack(a.args, stackSlot, &stackByteOffset, reflect.TypeOf(uintptr(0))).Uint())
316 } else if stackFrame != nil {
317 if runtime.GOARCH == "s390x" {
318 ptr = uintptr(callbackArgFromSlotBigEndian(unsafe.Pointer(&stackFrame[stackSlot]), reflect.TypeOf(uintptr(0))).Uint())
319 } else {
320 ptr = *(*uintptr)(unsafe.Pointer(&stackFrame[stackSlot]))
321 }
322 stackSlot += slots
323 } else {
324 ptr = frame[stackSlot]
325 stackSlot += slots
326 }
327 } else {
328 pos := intsN + numOfFloatRegisters()
329 ptr = frame[pos]
330 }
331 intsN += slots
332 args[i] = reflect.ValueOf(strings.GoString(ptr))
333 continue
334 case reflect.Struct:
335 if i == 0 && inType.AssignableTo(reflect.TypeOf(CDecl{})) {
336 args[i] = reflect.Zero(inType)
337 continue
338 }
339 if inType.Size() == 0 {
340 args[i] = reflect.New(inType).Elem()
341 continue
342 }
343 args[i] = getCallbackStruct(inType, a.args, &floatsN, &intsN, &stackSlot, &stackByteOffset)
344 continue
345 default:
346 slots = int((inType.Size() + ptrSize - 1) / ptrSize)
347 if intsN+slots > numOfIntegerRegisters() {
348 if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
349 // Darwin ARM64: read from packed stack with proper alignment
350 args[i] = callbackArgFromStack(a.args, stackSlot, &stackByteOffset, inType)
351 } else if stackFrame != nil {
352 // ppc64le/s390x: stack args are in separate stackFrame
353 if runtime.GOARCH == "s390x" {
354 // s390x big-endian: sub-8-byte values are right-justified
355 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&stackFrame[stackSlot]), inType)
356 } else {
357 args[i] = reflect.NewAt(inType, unsafe.Pointer(&stackFrame[stackSlot])).Elem()
358 }
359 stackSlot += slots
360 } else {
361 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
362 stackSlot += slots
363 }
364 } else {
365 // the integers begin after the floats in frame
366 pos := intsN + numOfFloatRegisters()
367 if runtime.GOARCH == "s390x" {
368 // s390x big-endian: sub-8-byte values are right-justified in GPR slot
369 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&frame[pos]), inType)
370 } else {
371 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[pos])).Elem()
372 }
373 }
374 intsN += slots
375 }
376 }
377 ret := fn.Call(args)
378 if len(ret) > 0 {
379 switch k := ret[0].Kind(); k {
380 case reflect.Uint, reflect.Uint64, reflect.Uint32, reflect.Uint16, reflect.Uint8, reflect.Uintptr:
381 a.result[0] = uintptr(ret[0].Uint())
382 case reflect.Int, reflect.Int64, reflect.Int32, reflect.Int16, reflect.Int8:
383 a.result[0] = uintptr(ret[0].Int())
384 case reflect.Bool:
385 if ret[0].Bool() {
386 a.result[0] = 1
387 } else {
388 a.result[0] = 0
389 }
390 case reflect.Pointer:
391 a.result[0] = ret[0].Pointer()
392 case reflect.UnsafePointer:
393 a.result[0] = ret[0].Pointer()
394 case reflect.Struct:
395 setStruct(a, ret[0])
396 default:
397 panic("purego: unsupported kind: " + k.String())
398 }
399 }
400}
401
402// callbackArgFromStack reads an argument from the tightly-packed stack area on Darwin ARM64.
403// The C ABI on Darwin ARM64 packs small types on the stack without padding to 8 bytes.
404// This function handles proper alignment and advances stackByteOffset accordingly.
405func callbackArgFromStack(argsBase unsafe.Pointer, stackSlot int, stackByteOffset *uintptr, inType reflect.Type) reflect.Value {
406 // Calculate base address of stack area (after float and int registers)
407 stackBase := unsafe.Add(argsBase, stackSlot*int(ptrSize))
408
409 // Get type's natural alignment
410 align := uintptr(inType.Align())
411 size := inType.Size()
412
413 // Align the offset
414 if *stackByteOffset%align != 0 {
415 *stackByteOffset = (*stackByteOffset + align - 1) &^ (align - 1)
416 }
417
418 // Read value at aligned offset
419 ptr := unsafe.Add(stackBase, *stackByteOffset)
420 *stackByteOffset += size
421
422 return reflect.NewAt(inType, ptr).Elem()
423}
424
425// callbackArgFromSlotBigEndian reads an argument from an 8-byte slot on big-endian architectures.
426// On s390x:
427// - Integer types are right-justified in GPRs: sub-8-byte values are at offset (8 - size)
428// - Float32 in FPRs is left-justified: stored in upper 32 bits, so at offset 0
429// - Float64 occupies the full 8-byte slot
430func callbackArgFromSlotBigEndian(slotPtr unsafe.Pointer, inType reflect.Type) reflect.Value {
431 size := inType.Size()
432 if size >= 8 {
433 // 8-byte values occupy the entire slot
434 return reflect.NewAt(inType, slotPtr).Elem()
435 }
436 // Float32 is left-justified in FPRs (upper 32 bits), so offset is 0
437 if inType.Kind() == reflect.Float32 {
438 return reflect.NewAt(inType, slotPtr).Elem()
439 }
440 // Integer types are right-justified: offset = 8 - size
441 offset := 8 - size
442 ptr := unsafe.Add(slotPtr, offset)
443 return reflect.NewAt(inType, ptr).Elem()
444}
445
446// callbackasmAddr returns address of runtime.callbackasm
447// function adjusted by i.
448// On x86 and amd64, runtime.callbackasm is a series of CALL instructions,
449// and we want callback to arrive at
450// correspondent call instruction instead of start of
451// runtime.callbackasm.
452// On ARM, runtime.callbackasm is a series of mov and branch instructions.
453// R12 is loaded with the callback index. Each entry is two instructions,
454// hence 8 bytes.
455func callbackasmAddr(i int) uintptr {
456 var entrySize int
457 switch runtime.GOARCH {
458 default:
459 panic("purego: unsupported architecture")
460 case "amd64":
461 // On amd64, each callback entry is just a CALL instruction (5 bytes)
462 entrySize = 5
463 case "386":
464 // On 386, each callback entry is MOVL $imm, CX (5 bytes) + JMP (5 bytes)
465 entrySize = 10
466 case "arm", "arm64", "loong64", "ppc64le", "riscv64":
467 // On ARM, ARM64, Loong64, PPC64LE and RISCV64, each entry is a MOV instruction
468 // followed by a branch instruction
469 entrySize = 8
470 case "s390x":
471 // On S390X, each entry is LGHI (4 bytes) + JG (6 bytes)
472 entrySize = 10
473 }
474 return callbackasmABI0 + uintptr(i*entrySize)
475}