1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: 2022 The Ebitengine Authors
3
4//go:build darwin || freebsd || (linux && (386 || amd64 || arm || arm64 || loong64 || ppc64le || riscv64 || (s390x && (cgo || go1.27)))) || netbsd
5
6package purego
7
8import (
9 "math"
10 "reflect"
11 "runtime"
12 "sync"
13 "unsafe"
14)
15
16var syscallXABI0 uintptr
17
18func syscall_syscallN(fn uintptr, args ...uintptr) (r1, r2, err uintptr) {
19 panic("purego: syscall_syscallN is only supported on windows")
20}
21
22// NewCallback converts a Go function to a function pointer conforming to the C calling convention.
23// This is useful when interoperating with C code requiring callbacks. The argument is expected to be a
24// function with zero or one uintptr-sized result. The function must not have arguments with size larger than the size
25// of uintptr. Only a limited number of callbacks may be created in a single Go process, and any memory allocated
26// for these callbacks is never released. At least 1024 callbacks can always be created. Although this function
27// provides similar functionality to windows.NewCallback it is distinct.
28//
29// Every call to NewCallback creates a new callback even for the same function value, so passing a Go callback to C
30// inside a loop (e.g. a qsort comparator) keeps consuming callbacks and eventually panics once they are exhausted.
31// The same happens when a func value is passed to a C function, as [RegisterFunc] creates a new callback for each
32// call. Create the callback once with NewCallback and reuse the returned pointer instead.
33func NewCallback(fn any) uintptr {
34 ty := reflect.TypeOf(fn)
35 if ty == nil || ty.Kind() != reflect.Func {
36 panic("purego: the type must be a function but was not")
37 }
38 for i := range ty.NumIn() {
39 in := ty.In(i)
40 if !in.AssignableTo(reflect.TypeFor[CDecl]()) {
41 continue
42 }
43 if i != 0 {
44 panic("purego: CDecl must be the first argument")
45 }
46 }
47 return compileCallback(fn)
48}
49
50// maxCb is the maximum number of callbacks
51// only increase this if you have added more to the callbackasm function
52const maxCB = 2000
53
54var cbs struct {
55 lock sync.Mutex
56 numFn int // the number of functions currently in cbs.funcs
57 funcs [maxCB]reflect.Value // the saved callbacks
58}
59
60func compileCallback(fn any) uintptr {
61 val := reflect.ValueOf(fn)
62 if val.Kind() != reflect.Func {
63 panic("purego: the type must be a function but was not")
64 }
65 if val.IsNil() {
66 panic("purego: function must not be nil")
67 }
68 ty := val.Type()
69 for i := range ty.NumIn() {
70 in := ty.In(i)
71 switch in.Kind() {
72 case reflect.Struct:
73 if i == 0 && in.AssignableTo(reflect.TypeFor[CDecl]()) {
74 continue
75 }
76 ensureCallbackStructSupported()
77 checkStructFieldsSupported(in)
78 continue
79 case reflect.Interface, reflect.Func, reflect.Slice,
80 reflect.Chan, reflect.Complex64, reflect.Complex128,
81 reflect.String, reflect.Map, reflect.Invalid:
82 panic("purego: unsupported argument type: " + in.Kind().String())
83 }
84 }
85output:
86 switch {
87 case ty.NumOut() == 1:
88 switch ty.Out(0).Kind() {
89 case reflect.Struct:
90 ensureCallbackStructSupported()
91 checkStructFieldsSupported(ty.Out(0))
92 break output
93 case reflect.Pointer, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
94 reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
95 reflect.Bool, reflect.UnsafePointer:
96 break output
97 }
98 panic("purego: unsupported return type: " + ty.String())
99 case ty.NumOut() > 1:
100 panic("purego: callbacks can only have one return")
101 }
102 cbs.lock.Lock()
103 defer cbs.lock.Unlock()
104 if cbs.numFn >= maxCB {
105 panic("purego: the maximum number of callbacks has been reached")
106 }
107 cbs.funcs[cbs.numFn] = val
108 cbs.numFn++
109 return callbackasmAddr(cbs.numFn - 1)
110}
111
112const ptrSize = unsafe.Sizeof((*int)(nil))
113
114const callbackMaxFrame = 64 * ptrSize
115
116// callbackasm is implemented in zcallback_GOOS_GOARCH.s
117//
118//go:linkname __callbackasm callbackasm
119var __callbackasm byte
120var callbackasmABI0 = uintptr(unsafe.Pointer(&__callbackasm))
121
122// callbackWrap_call allows the calling of the ABIInternal wrapper
123// which is required for runtime.cgocallback without the
124// <ABIInternal> tag which is only allowed in the runtime.
125// This closure is used inside sys_darwin_GOARCH.s
126var callbackWrap_call = callbackWrap
127
128// callbackWrap is called by assembly code which determines which Go function to call.
129// This function takes the arguments and passes them to the Go function and returns the result.
130func callbackWrap(a *callbackArgs) {
131 cbs.lock.Lock()
132 fn := cbs.funcs[a.index]
133 cbs.lock.Unlock()
134 fnType := fn.Type()
135 args := make([]reflect.Value, fnType.NumIn())
136 frame := (*[callbackMaxFrame]uintptr)(a.args)
137 // stackFrame points to stack-passed arguments. On most architectures this is
138 // contiguous with frame (after register args), but on ppc64le it's separate.
139 var stackFrame *[callbackMaxFrame]uintptr
140 var intFrame *[callbackMaxFrame]uintptr
141 if sf := a.stackFrame(); sf != nil {
142 // Only ppc64le uses separate stackArgs pointer due to NOSPLIT constraints
143 stackFrame = (*[callbackMaxFrame]uintptr)(sf)
144 }
145 if intf := a.intFrame(); intf != nil {
146 intFrame = (*[callbackMaxFrame]uintptr)(intf)
147 }
148 // floatsN and intsN track the number of register slots used, not argument count.
149 // This distinction matters on ARM32 where float64 uses 2 slots (32-bit registers).
150 var floatsN int
151 var intsN int
152 // On amd64/loong64/ppc64le/riscv64/s390x, when returning a struct larger than
153 // maxRegAllocStructSize, the caller passes a hidden pointer in the first integer
154 // register. Skip it to avoid misreading it as the first function argument.
155 if (runtime.GOARCH == "amd64" || runtime.GOARCH == "loong64" || runtime.GOARCH == "riscv64" || runtime.GOARCH == "s390x") &&
156 fnType.NumOut() == 1 && fnType.Out(0).Kind() == reflect.Struct &&
157 fnType.Out(0).Size() > maxRegAllocStructSize {
158 intsN = 1
159 }
160 // stackSlot points to the index into frame (or stackFrame) of the current stack element.
161 // When stackFrame is nil, stack begins after float and integer registers in frame.
162 // When stackFrame is not nil (ppc64le), stackSlot indexes into stackFrame starting at 0.
163 stackSlot := numOfIntegerRegisters() + numOfFloatRegisters()
164 if stackFrame != nil {
165 // ppc64le: stackArgs is a separate pointer, indices start at 0
166 stackSlot = 0
167 }
168 // stackByteOffset tracks the byte offset within the stack area for Darwin ARM64
169 // tight packing. On Darwin ARM64, C passes small types packed on the stack.
170 stackByteOffset := uintptr(0)
171 for i := range args {
172 // slots is the number of pointer-sized slots the argument takes
173 inType := fnType.In(i)
174 slots := int((inType.Size() + ptrSize - 1) / ptrSize)
175 switch inType.Kind() {
176 case reflect.Float32, reflect.Float64:
177 if isARMSoftFloat() {
178 // we should restore from integer slot, can skip unnecessary branching here
179 if isARMPaddingNeeded(inType, -1, intsN) {
180 intsN++
181 }
182 if intsN+slots <= numOfIntegerRegisters() {
183 // the integers begin after the floats in frame
184 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[intsN+numOfFloatRegisters()])).Elem()
185 intsN += slots
186 continue
187 }
188 if isARMPaddingNeeded(inType, -1, stackSlot) {
189 stackSlot++
190 }
191 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
192 stackSlot += slots
193 intsN += slots
194 continue
195 }
196
197 if floatsN+slots > numOfFloatRegisters() {
198 if isDarwin && runtime.GOARCH == "arm64" {
199 // Darwin ARM64: read from packed stack with proper alignment
200 args[i] = callbackArgFromStack(a.args, stackSlot, &stackByteOffset, inType)
201 } else if stackFrame != nil {
202 // ppc64le/s390x: stack args are in separate stackFrame
203 switch runtime.GOARCH {
204 case "ppc64le":
205 args[i] = callbackFloatFromDoubleSlot(unsafe.Pointer(&stackFrame[stackSlot]), inType)
206 case "s390x":
207 // s390x big-endian: sub-8-byte values are right-justified
208 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&stackFrame[stackSlot]), inType)
209 default:
210 args[i] = reflect.NewAt(inType, unsafe.Pointer(&stackFrame[stackSlot])).Elem()
211 }
212 stackSlot += slots
213 } else if isARMFloatPaddingNeeded(inType, -1, stackSlot) {
214 stackSlot++
215 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
216 stackSlot += slots
217 } else {
218 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
219 stackSlot += slots
220 }
221 } else {
222 switch runtime.GOARCH {
223 case "ppc64le":
224 args[i] = callbackFloatFromDoubleSlot(unsafe.Pointer(&frame[floatsN]), inType)
225 case "s390x":
226 // s390x big-endian: float32 is right-justified in 8-byte FPR slot
227 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&frame[floatsN]), inType)
228 default:
229 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[floatsN])).Elem()
230 }
231 }
232 floatsN += slots
233 case reflect.Struct:
234 if i == 0 && inType.AssignableTo(reflect.TypeFor[CDecl]()) {
235 args[i] = reflect.Zero(inType)
236 continue
237 }
238 if inType.Size() == 0 {
239 args[i] = reflect.New(inType).Elem()
240 continue
241 }
242 args[i] = getCallbackStruct(inType, a.args, &floatsN, &intsN, &stackSlot, &stackByteOffset)
243 continue
244 default:
245 if isARMPaddingNeeded(inType, -1, intsN) {
246 intsN++
247 }
248 if intsN+slots > numOfIntegerRegisters() {
249 if isDarwin && runtime.GOARCH == "arm64" {
250 // Darwin ARM64: read from packed stack with proper alignment
251 args[i] = callbackArgFromStack(a.args, stackSlot, &stackByteOffset, inType)
252 } else if stackFrame != nil {
253 // ppc64le/s390x: stack args are in separate stackFrame
254 if runtime.GOARCH == "s390x" {
255 // s390x big-endian: sub-8-byte values are right-justified
256 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&stackFrame[stackSlot]), inType)
257 } else {
258 args[i] = reflect.NewAt(inType, unsafe.Pointer(&stackFrame[stackSlot])).Elem()
259 }
260 stackSlot += slots
261 } else if isARMPaddingNeeded(inType, -1, stackSlot) {
262 stackSlot++
263 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
264 stackSlot += slots
265 } else {
266 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
267 stackSlot += slots
268 }
269 } else {
270 if intFrame != nil {
271 args[i] = reflect.NewAt(inType, unsafe.Pointer(&intFrame[intsN])).Elem()
272 } else {
273 // the integers begin after the floats in frame
274 pos := intsN + numOfFloatRegisters()
275 if runtime.GOARCH == "s390x" {
276 // s390x big-endian: sub-8-byte values are right-justified in GPR slot
277 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&frame[pos]), inType)
278 } else {
279 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[pos])).Elem()
280 }
281 }
282 }
283 intsN += slots
284 }
285 }
286 ret := fn.Call(args)
287 if len(ret) > 0 {
288 switch k := ret[0].Kind(); k {
289 case reflect.Uint64:
290 a.setUint64Result(ret[0].Uint())
291 case reflect.Uint, reflect.Uint32, reflect.Uint16, reflect.Uint8, reflect.Uintptr:
292 a.result[0] = uintptr(ret[0].Uint())
293 case reflect.Int64:
294 a.setInt64Result(ret[0].Int())
295 case reflect.Int, reflect.Int32, reflect.Int16, reflect.Int8:
296 a.result[0] = uintptr(ret[0].Int())
297 case reflect.Bool:
298 if ret[0].Bool() {
299 a.result[0] = 1
300 } else {
301 a.result[0] = 0
302 }
303 case reflect.Pointer:
304 a.result[0] = ret[0].Pointer()
305 case reflect.UnsafePointer:
306 a.result[0] = ret[0].Pointer()
307 case reflect.Struct:
308 setStruct(a, ret[0])
309 default:
310 panic("purego: unsupported kind: " + k.String())
311 }
312 }
313}
314
315// callbackArgFromStack reads an argument from the tightly-packed stack area on Darwin ARM64.
316// The C ABI on Darwin ARM64 packs small types on the stack without padding to 8 bytes.
317// This function handles proper alignment and advances stackByteOffset accordingly.
318func callbackArgFromStack(argsBase unsafe.Pointer, stackSlot int, stackByteOffset *uintptr, inType reflect.Type) reflect.Value {
319 // Calculate base address of stack area (after float and int registers)
320 stackBase := unsafe.Add(argsBase, stackSlot*int(ptrSize))
321
322 // Get type's natural alignment
323 align := uintptr(inType.Align())
324 size := inType.Size()
325
326 // Align the offset
327 if *stackByteOffset%align != 0 {
328 *stackByteOffset = (*stackByteOffset + align - 1) &^ (align - 1)
329 }
330
331 // Read value at aligned offset
332 ptr := unsafe.Add(stackBase, *stackByteOffset)
333 *stackByteOffset += size
334
335 return reflect.NewAt(inType, ptr).Elem()
336}
337
338// callbackFloatFromDoubleSlot reads a floating-point callback argument from an
339// 8-byte register or stack slot on ppc64le, where a single-precision value is
340// held in double-precision format.
341func callbackFloatFromDoubleSlot(slotPtr unsafe.Pointer, inType reflect.Type) reflect.Value {
342 if inType.Kind() != reflect.Float32 {
343 return reflect.NewAt(inType, slotPtr).Elem()
344 }
345 v := reflect.New(inType).Elem()
346 v.SetFloat(math.Float64frombits(*(*uint64)(slotPtr)))
347 return v
348}
349
350// callbackArgFromSlotBigEndian reads an argument from an 8-byte slot on big-endian architectures.
351// On s390x:
352// - Integer types are right-justified in GPRs: sub-8-byte values are at offset (8 - size)
353// - Float32 in FPRs is left-justified: stored in upper 32 bits, so at offset 0
354// - Float64 occupies the full 8-byte slot
355func callbackArgFromSlotBigEndian(slotPtr unsafe.Pointer, inType reflect.Type) reflect.Value {
356 size := inType.Size()
357 if size >= 8 {
358 // 8-byte values occupy the entire slot
359 return reflect.NewAt(inType, slotPtr).Elem()
360 }
361 // Float32 is left-justified in FPRs (upper 32 bits), so offset is 0
362 if inType.Kind() == reflect.Float32 {
363 return reflect.NewAt(inType, slotPtr).Elem()
364 }
365 // Integer types are right-justified: offset = 8 - size
366 offset := 8 - size
367 ptr := unsafe.Add(slotPtr, offset)
368 return reflect.NewAt(inType, ptr).Elem()
369}
370
371// callbackasmAddr returns address of runtime.callbackasm
372// function adjusted by i.
373// On x86 and amd64, runtime.callbackasm is a series of CALL instructions,
374// and we want callback to arrive at
375// correspondent call instruction instead of start of
376// runtime.callbackasm.
377// On ARM, runtime.callbackasm is a series of mov and branch instructions.
378// R12 is loaded with the callback index. Each entry is two instructions,
379// hence 8 bytes.
380func callbackasmAddr(i int) uintptr {
381 var entrySize int
382 switch runtime.GOARCH {
383 default:
384 panic("purego: unsupported architecture")
385 case "amd64":
386 // On amd64, each callback entry is just a CALL instruction (5 bytes)
387 entrySize = 5
388 case "386":
389 // On 386, each callback entry is MOVL $imm, CX (5 bytes) + JMP (5 bytes)
390 entrySize = 10
391 case "arm", "arm64", "loong64", "ppc64le", "riscv64":
392 // On ARM, ARM64, Loong64, PPC64LE and RISCV64, each entry is a MOV instruction
393 // followed by a branch instruction
394 entrySize = 8
395 case "s390x":
396 // On S390X, each entry is LGHI (4 bytes) + JG (6 bytes)
397 entrySize = 10
398 }
399 return callbackasmABI0 + uintptr(i*entrySize)
400}