1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: 2022 The Ebitengine Authors
3
4// TODO: remove s390x cgo dependency once golang/go#77449 is resolved
5//go:build darwin || freebsd || (linux && (386 || amd64 || arm || arm64 || loong64 || ppc64le || riscv64 || (cgo && s390x))) || netbsd
6
7package purego
8
9import (
10 "errors"
11 "reflect"
12 "runtime"
13 "sync"
14 "unsafe"
15
16 "github.com/bnema/purego/internal/strings"
17)
18
19var syscall15XABI0 uintptr
20
21func syscall_syscall15X(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15 uintptr) (r1, r2, err uintptr) {
22 args := thePool.Get().(*syscall15Args)
23 defer thePool.Put(args)
24
25 *args = syscall15Args{
26 fn: fn,
27 a1: a1, a2: a2, a3: a3, a4: a4, a5: a5, a6: a6, a7: a7, a8: a8,
28 a9: a9, a10: a10, a11: a11, a12: a12, a13: a13, a14: a14, a15: a15,
29 f1: a1, f2: a2, f3: a3, f4: a4, f5: a5, f6: a6, f7: a7, f8: a8,
30 }
31
32 runtime_cgocall(syscall15XABI0, unsafe.Pointer(args))
33 return args.a1, args.a2, args.a3
34}
35
36// NewCallback converts a Go function to a function pointer conforming to the C calling convention.
37// This is useful when interoperating with C code requiring callbacks. The argument is expected to be a
38// function with zero or one uintptr-sized result. The function must not have arguments with size larger than the size
39// of uintptr. Only a limited number of callbacks may be created in a single Go process, and any memory allocated
40// for these callbacks is never released. At least 2000 callbacks can always be created. Although this function
41// provides similar functionality to windows.NewCallback it is distinct.
42func NewCallback(fn any) uintptr {
43 ty := reflect.TypeOf(fn)
44 for i := 0; i < ty.NumIn(); i++ {
45 in := ty.In(i)
46 if !in.AssignableTo(reflect.TypeOf(CDecl{})) {
47 continue
48 }
49 if i != 0 {
50 panic("purego: CDecl must be the first argument")
51 }
52 }
53 return compileCallback(fn)
54}
55
56// NewCallbackFnPtr converts a Go function pointer to a function pointer conforming to the C calling convention.
57// Calling this function multiple times with the same function pointer returns the original callback address.
58func NewCallbackFnPtr(fnPtr any) uintptr {
59 val := reflect.ValueOf(fnPtr)
60 if val.IsNil() {
61 panic("purego: function must not be nil")
62 }
63 if val.Kind() != reflect.Ptr || val.Elem().Kind() != reflect.Func {
64 panic("purego: the type must be a function pointer but was not")
65 }
66 if addr, ok := getCallbackByFnPtr(val); ok {
67 return addr
68 }
69 addr := compileCallback(val.Elem().Interface())
70 cbs.lock.Lock()
71 cbs.knownFnPtr[val.Pointer()] = addr
72 if idx, ok := cbs.knownIdx[addr]; ok {
73 cbs.fnPtrKeys[idx] = val.Pointer()
74 }
75 cbs.lock.Unlock()
76 return addr
77}
78
79// UnrefCallback unreferences a callback created by NewCallback or NewCallbackFnPtr and frees its slot.
80func UnrefCallback(cb uintptr) error {
81 cbs.lock.Lock()
82 defer cbs.lock.Unlock()
83 idx, ok := cbs.knownIdx[cb]
84 if !ok {
85 return errors.New("callback not found")
86 }
87 if key := cbs.fnPtrKeys[idx]; key != 0 {
88 delete(cbs.knownFnPtr, key)
89 cbs.fnPtrKeys[idx] = 0
90 }
91 delete(cbs.knownIdx, cb)
92 cbs.holes[idx] = struct{}{}
93 cbs.funcs[idx] = reflect.Value{}
94 cbs.argPools[idx] = nil
95 return nil
96}
97
98// UnrefCallbackFnPtr unreferences a callback previously created via NewCallbackFnPtr.
99func UnrefCallbackFnPtr(fnPtr any) error {
100 val := reflect.ValueOf(fnPtr)
101 if val.IsNil() {
102 panic("purego: function must not be nil")
103 }
104 if val.Kind() != reflect.Ptr || val.Elem().Kind() != reflect.Func {
105 panic("purego: the type must be a function pointer but was not")
106 }
107 cbs.lock.Lock()
108 defer cbs.lock.Unlock()
109 addr, ok := cbs.knownFnPtr[val.Pointer()]
110 if !ok {
111 return errors.New("callback not found")
112 }
113 idx, ok := cbs.knownIdx[addr]
114 if !ok {
115 delete(cbs.knownFnPtr, val.Pointer())
116 return errors.New("callback not found")
117 }
118 delete(cbs.knownFnPtr, val.Pointer())
119 delete(cbs.knownIdx, addr)
120 cbs.fnPtrKeys[idx] = 0
121 cbs.holes[idx] = struct{}{}
122 cbs.funcs[idx] = reflect.Value{}
123 cbs.argPools[idx] = nil
124 return nil
125}
126
127// maxCb is the maximum number of callbacks
128// only increase this if you have added more to the callbackasm function
129const maxCB = 2000
130
131var cbs = struct {
132 lock sync.RWMutex
133 numFn int // the highest allocated callback index + 1
134 holes map[int]struct{} // reusable callback slots
135 funcs [maxCB]reflect.Value // the saved callbacks
136 argPools [maxCB]*sync.Pool // pre-allocated argument buffers per callback
137 knownIdx map[uintptr]int // callback address -> slot index
138 knownFnPtr map[uintptr]uintptr // function pointer variable address -> callback address
139 fnPtrKeys [maxCB]uintptr // slot index -> function pointer variable address
140}{
141 holes: make(map[int]struct{}),
142 knownIdx: make(map[uintptr]int, maxCB),
143 knownFnPtr: make(map[uintptr]uintptr, maxCB),
144}
145
146func getCallbackByFnPtr(val reflect.Value) (uintptr, bool) {
147 cbs.lock.RLock()
148 defer cbs.lock.RUnlock()
149 addr, ok := cbs.knownFnPtr[val.Pointer()]
150 return addr, ok
151}
152
153func compileCallback(fn any) uintptr {
154 val := reflect.ValueOf(fn)
155 if val.Kind() != reflect.Func {
156 panic("purego: the type must be a function but was not")
157 }
158 if val.IsNil() {
159 panic("purego: function must not be nil")
160 }
161 ty := val.Type()
162 for i := 0; i < ty.NumIn(); i++ {
163 in := ty.In(i)
164 switch in.Kind() {
165 case reflect.Struct:
166 if i == 0 && in.AssignableTo(reflect.TypeOf(CDecl{})) {
167 continue
168 }
169 ensureStructSupported()
170 checkStructFieldsSupported(in)
171 continue
172 case reflect.Interface, reflect.Func, reflect.Slice,
173 reflect.Chan, reflect.Complex64, reflect.Complex128,
174 reflect.Map, reflect.Invalid:
175 panic("purego: unsupported argument type: " + in.Kind().String())
176 }
177 }
178output:
179 switch {
180 case ty.NumOut() == 1:
181 switch ty.Out(0).Kind() {
182 case reflect.Pointer, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
183 reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
184 reflect.Bool, reflect.UnsafePointer, reflect.Struct:
185 break output
186 }
187 panic("purego: unsupported return type: " + ty.String())
188 case ty.NumOut() > 1:
189 panic("purego: callbacks can only have one return")
190 }
191 cbs.lock.Lock()
192 defer cbs.lock.Unlock()
193 index := -1
194 for i := range cbs.holes {
195 index = i
196 delete(cbs.holes, i)
197 break
198 }
199 if index < 0 {
200 if cbs.numFn >= maxCB {
201 panic("purego: the maximum number of callbacks has been reached")
202 }
203 index = cbs.numFn
204 cbs.numFn++
205 }
206 cbs.funcs[index] = val
207 numIn := ty.NumIn()
208 cbs.argPools[index] = &sync.Pool{
209 New: func() any {
210 return make([]reflect.Value, numIn)
211 },
212 }
213 addr := callbackasmAddr(index)
214 cbs.knownIdx[addr] = index
215 return addr
216}
217
218const ptrSize = unsafe.Sizeof((*int)(nil))
219
220const callbackMaxFrame = 64 * ptrSize
221
222// callbackasm is implemented in zcallback_GOOS_GOARCH.s
223//
224//go:linkname __callbackasm callbackasm
225var __callbackasm byte
226var callbackasmABI0 = uintptr(unsafe.Pointer(&__callbackasm))
227
228// callbackWrap_call allows the calling of the ABIInternal wrapper
229// which is required for runtime.cgocallback without the
230// <ABIInternal> tag which is only allowed in the runtime.
231// This closure is used inside sys_darwin_GOARCH.s
232var callbackWrap_call = callbackWrap
233
234// callbackWrap is called by assembly code which determines which Go function to call.
235// This function takes the arguments and passes them to the Go function and returns the result.
236func callbackWrap(a *callbackArgs) {
237 cbs.lock.RLock()
238 fn := cbs.funcs[a.index]
239 pool := cbs.argPools[a.index]
240 cbs.lock.RUnlock()
241 fnType := fn.Type()
242 args := pool.Get().([]reflect.Value)
243 defer func() {
244 for i := range args {
245 args[i] = reflect.Value{}
246 }
247 pool.Put(args)
248 }()
249 frame := (*[callbackMaxFrame]uintptr)(a.args)
250 // stackFrame points to stack-passed arguments. On most architectures this is
251 // contiguous with frame (after register args), but on ppc64le it's separate.
252 var stackFrame *[callbackMaxFrame]uintptr
253 if sf := a.stackFrame(); sf != nil {
254 // Only ppc64le uses separate stackArgs pointer due to NOSPLIT constraints
255 stackFrame = (*[callbackMaxFrame]uintptr)(sf)
256 }
257 // floatsN and intsN track the number of register slots used, not argument count.
258 // This distinction matters on ARM32 where float64 uses 2 slots (32-bit registers).
259 var floatsN int
260 var intsN int
261 // On amd64/loong64/ppc64le/riscv64/s390x, when returning a struct larger than
262 // maxRegAllocStructSize, the caller passes a hidden pointer in the first integer
263 // register. Skip it to avoid misreading it as the first function argument.
264 if (runtime.GOARCH == "amd64" || runtime.GOARCH == "loong64" || runtime.GOARCH == "ppc64le" || runtime.GOARCH == "riscv64" || runtime.GOARCH == "s390x") &&
265 fnType.NumOut() == 1 && fnType.Out(0).Kind() == reflect.Struct &&
266 fnType.Out(0).Size() > maxRegAllocStructSize {
267 intsN = 1
268 }
269 // stackSlot points to the index into frame (or stackFrame) of the current stack element.
270 // When stackFrame is nil, stack begins after float and integer registers in frame.
271 // When stackFrame is not nil (ppc64le), stackSlot indexes into stackFrame starting at 0.
272 stackSlot := numOfIntegerRegisters() + numOfFloatRegisters()
273 if stackFrame != nil {
274 // ppc64le: stackArgs is a separate pointer, indices start at 0
275 stackSlot = 0
276 }
277 // stackByteOffset tracks the byte offset within the stack area for Darwin ARM64
278 // tight packing. On Darwin ARM64, C passes small types packed on the stack.
279 stackByteOffset := uintptr(0)
280 for i := range args {
281 // slots is the number of pointer-sized slots the argument takes
282 var slots int
283 inType := fnType.In(i)
284 switch inType.Kind() {
285 case reflect.Float32, reflect.Float64:
286 slots = int((fnType.In(i).Size() + ptrSize - 1) / ptrSize)
287 if floatsN+slots > numOfFloatRegisters() {
288 if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
289 // Darwin ARM64: read from packed stack with proper alignment
290 args[i] = callbackArgFromStack(a.args, stackSlot, &stackByteOffset, inType)
291 } else if stackFrame != nil {
292 // ppc64le/s390x: stack args are in separate stackFrame
293 if runtime.GOARCH == "s390x" {
294 // s390x big-endian: sub-8-byte values are right-justified
295 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&stackFrame[stackSlot]), inType)
296 } else {
297 args[i] = reflect.NewAt(inType, unsafe.Pointer(&stackFrame[stackSlot])).Elem()
298 }
299 stackSlot += slots
300 } else {
301 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
302 stackSlot += slots
303 }
304 } else {
305 if runtime.GOARCH == "s390x" {
306 // s390x big-endian: float32 is right-justified in 8-byte FPR slot
307 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&frame[floatsN]), inType)
308 } else {
309 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[floatsN])).Elem()
310 }
311 }
312 floatsN += slots
313 case reflect.String:
314 slots = 1
315 var ptr uintptr
316 if intsN+slots > numOfIntegerRegisters() {
317 if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
318 ptr = uintptr(callbackArgFromStack(a.args, stackSlot, &stackByteOffset, reflect.TypeOf(uintptr(0))).Uint())
319 } else if stackFrame != nil {
320 if runtime.GOARCH == "s390x" {
321 ptr = uintptr(callbackArgFromSlotBigEndian(unsafe.Pointer(&stackFrame[stackSlot]), reflect.TypeOf(uintptr(0))).Uint())
322 } else {
323 ptr = *(*uintptr)(unsafe.Pointer(&stackFrame[stackSlot]))
324 }
325 stackSlot += slots
326 } else {
327 ptr = frame[stackSlot]
328 stackSlot += slots
329 }
330 } else {
331 pos := intsN + numOfFloatRegisters()
332 ptr = frame[pos]
333 }
334 intsN += slots
335 args[i] = reflect.ValueOf(strings.GoString(ptr))
336 continue
337 case reflect.Struct:
338 if i == 0 && inType.AssignableTo(reflect.TypeOf(CDecl{})) {
339 args[i] = reflect.Zero(inType)
340 continue
341 }
342 if inType.Size() == 0 {
343 args[i] = reflect.New(inType).Elem()
344 continue
345 }
346 args[i] = getCallbackStruct(inType, a.args, &floatsN, &intsN, &stackSlot, &stackByteOffset)
347 continue
348 default:
349 slots = int((inType.Size() + ptrSize - 1) / ptrSize)
350 if intsN+slots > numOfIntegerRegisters() {
351 if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
352 // Darwin ARM64: read from packed stack with proper alignment
353 args[i] = callbackArgFromStack(a.args, stackSlot, &stackByteOffset, inType)
354 } else if stackFrame != nil {
355 // ppc64le/s390x: stack args are in separate stackFrame
356 if runtime.GOARCH == "s390x" {
357 // s390x big-endian: sub-8-byte values are right-justified
358 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&stackFrame[stackSlot]), inType)
359 } else {
360 args[i] = reflect.NewAt(inType, unsafe.Pointer(&stackFrame[stackSlot])).Elem()
361 }
362 stackSlot += slots
363 } else {
364 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
365 stackSlot += slots
366 }
367 } else {
368 // the integers begin after the floats in frame
369 pos := intsN + numOfFloatRegisters()
370 if runtime.GOARCH == "s390x" {
371 // s390x big-endian: sub-8-byte values are right-justified in GPR slot
372 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&frame[pos]), inType)
373 } else {
374 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[pos])).Elem()
375 }
376 }
377 intsN += slots
378 }
379 }
380 ret := fn.Call(args)
381 if len(ret) > 0 {
382 switch k := ret[0].Kind(); k {
383 case reflect.Uint, reflect.Uint64, reflect.Uint32, reflect.Uint16, reflect.Uint8, reflect.Uintptr:
384 a.result[0] = uintptr(ret[0].Uint())
385 case reflect.Int, reflect.Int64, reflect.Int32, reflect.Int16, reflect.Int8:
386 a.result[0] = uintptr(ret[0].Int())
387 case reflect.Bool:
388 if ret[0].Bool() {
389 a.result[0] = 1
390 } else {
391 a.result[0] = 0
392 }
393 case reflect.Pointer:
394 a.result[0] = ret[0].Pointer()
395 case reflect.UnsafePointer:
396 a.result[0] = ret[0].Pointer()
397 case reflect.Struct:
398 setStruct(a, ret[0])
399 default:
400 panic("purego: unsupported kind: " + k.String())
401 }
402 }
403}
404
405// callbackArgFromStack reads an argument from the tightly-packed stack area on Darwin ARM64.
406// The C ABI on Darwin ARM64 packs small types on the stack without padding to 8 bytes.
407// This function handles proper alignment and advances stackByteOffset accordingly.
408func callbackArgFromStack(argsBase unsafe.Pointer, stackSlot int, stackByteOffset *uintptr, inType reflect.Type) reflect.Value {
409 // Calculate base address of stack area (after float and int registers)
410 stackBase := unsafe.Add(argsBase, stackSlot*int(ptrSize))
411
412 // Get type's natural alignment
413 align := uintptr(inType.Align())
414 size := inType.Size()
415
416 // Align the offset
417 if *stackByteOffset%align != 0 {
418 *stackByteOffset = (*stackByteOffset + align - 1) &^ (align - 1)
419 }
420
421 // Read value at aligned offset
422 ptr := unsafe.Add(stackBase, *stackByteOffset)
423 *stackByteOffset += size
424
425 return reflect.NewAt(inType, ptr).Elem()
426}
427
428// callbackArgFromSlotBigEndian reads an argument from an 8-byte slot on big-endian architectures.
429// On s390x:
430// - Integer types are right-justified in GPRs: sub-8-byte values are at offset (8 - size)
431// - Float32 in FPRs is left-justified: stored in upper 32 bits, so at offset 0
432// - Float64 occupies the full 8-byte slot
433func callbackArgFromSlotBigEndian(slotPtr unsafe.Pointer, inType reflect.Type) reflect.Value {
434 size := inType.Size()
435 if size >= 8 {
436 // 8-byte values occupy the entire slot
437 return reflect.NewAt(inType, slotPtr).Elem()
438 }
439 // Float32 is left-justified in FPRs (upper 32 bits), so offset is 0
440 if inType.Kind() == reflect.Float32 {
441 return reflect.NewAt(inType, slotPtr).Elem()
442 }
443 // Integer types are right-justified: offset = 8 - size
444 offset := 8 - size
445 ptr := unsafe.Add(slotPtr, offset)
446 return reflect.NewAt(inType, ptr).Elem()
447}
448
449// callbackasmAddr returns address of runtime.callbackasm
450// function adjusted by i.
451// On x86 and amd64, runtime.callbackasm is a series of CALL instructions,
452// and we want callback to arrive at
453// correspondent call instruction instead of start of
454// runtime.callbackasm.
455// On ARM, runtime.callbackasm is a series of mov and branch instructions.
456// R12 is loaded with the callback index. Each entry is two instructions,
457// hence 8 bytes.
458func callbackasmAddr(i int) uintptr {
459 var entrySize int
460 switch runtime.GOARCH {
461 default:
462 panic("purego: unsupported architecture")
463 case "amd64":
464 // On amd64, each callback entry is just a CALL instruction (5 bytes)
465 entrySize = 5
466 case "386":
467 // On 386, each callback entry is MOVL $imm, CX (5 bytes) + JMP (5 bytes)
468 entrySize = 10
469 case "arm", "arm64", "loong64", "ppc64le", "riscv64":
470 // On ARM, ARM64, Loong64, PPC64LE and RISCV64, each entry is a MOV instruction
471 // followed by a branch instruction
472 entrySize = 8
473 case "s390x":
474 // On S390X, each entry is LGHI (4 bytes) + JG (6 bytes)
475 entrySize = 10
476 }
477 return callbackasmABI0 + uintptr(i*entrySize)
478}