1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: 2022 The Ebitengine Authors
3
4//go:build darwin || freebsd || (linux && (386 || amd64 || arm || arm64 || loong64 || ppc64le || riscv64 || (s390x && (cgo || go1.27)))) || netbsd
5
6package purego
7
8import (
9 "math"
10 "reflect"
11 "runtime"
12 "sync"
13 "unsafe"
14)
15
16var syscallXABI0 uintptr
17
18func syscall_syscallN(fn uintptr, args ...uintptr) (r1, r2, err uintptr) {
19 panic("purego: syscall_syscallN is only supported on windows")
20}
21
22// NewCallback converts a Go function to a function pointer conforming to the C calling convention.
23// This is useful when interoperating with C code requiring callbacks. The argument is expected to be a
24// function with zero or one uintptr-sized result. The function must not have arguments with size larger than the size
25// of uintptr. Only a limited number of callbacks may be created in a single Go process, and any memory allocated
26// for these callbacks is never released. At least 2000 callbacks can always be created. Although this function
27// provides similar functionality to windows.NewCallback it is distinct.
28func NewCallback(fn any) uintptr {
29 ty := reflect.TypeOf(fn)
30 for i := range ty.NumIn() {
31 in := ty.In(i)
32 if !in.AssignableTo(reflect.TypeFor[CDecl]()) {
33 continue
34 }
35 if i != 0 {
36 panic("purego: CDecl must be the first argument")
37 }
38 }
39 return compileCallback(fn)
40}
41
42// maxCb is the maximum number of callbacks
43// only increase this if you have added more to the callbackasm function
44const maxCB = 2000
45
46var cbs struct {
47 lock sync.Mutex
48 numFn int // the number of functions currently in cbs.funcs
49 funcs [maxCB]reflect.Value // the saved callbacks
50}
51
52func compileCallback(fn any) uintptr {
53 val := reflect.ValueOf(fn)
54 if val.Kind() != reflect.Func {
55 panic("purego: the type must be a function but was not")
56 }
57 if val.IsNil() {
58 panic("purego: function must not be nil")
59 }
60 ty := val.Type()
61 for i := range ty.NumIn() {
62 in := ty.In(i)
63 switch in.Kind() {
64 case reflect.Struct:
65 if i == 0 && in.AssignableTo(reflect.TypeFor[CDecl]()) {
66 continue
67 }
68 ensureCallbackStructSupported()
69 checkStructFieldsSupported(in)
70 continue
71 case reflect.Interface, reflect.Func, reflect.Slice,
72 reflect.Chan, reflect.Complex64, reflect.Complex128,
73 reflect.String, reflect.Map, reflect.Invalid:
74 panic("purego: unsupported argument type: " + in.Kind().String())
75 }
76 }
77output:
78 switch {
79 case ty.NumOut() == 1:
80 switch ty.Out(0).Kind() {
81 case reflect.Struct:
82 ensureCallbackStructSupported()
83 checkStructFieldsSupported(ty.Out(0))
84 break output
85 case reflect.Pointer, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
86 reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
87 reflect.Bool, reflect.UnsafePointer:
88 break output
89 }
90 panic("purego: unsupported return type: " + ty.String())
91 case ty.NumOut() > 1:
92 panic("purego: callbacks can only have one return")
93 }
94 cbs.lock.Lock()
95 defer cbs.lock.Unlock()
96 if cbs.numFn >= maxCB {
97 panic("purego: the maximum number of callbacks has been reached")
98 }
99 cbs.funcs[cbs.numFn] = val
100 cbs.numFn++
101 return callbackasmAddr(cbs.numFn - 1)
102}
103
104const ptrSize = unsafe.Sizeof((*int)(nil))
105
106const callbackMaxFrame = 64 * ptrSize
107
108// callbackasm is implemented in zcallback_GOOS_GOARCH.s
109//
110//go:linkname __callbackasm callbackasm
111var __callbackasm byte
112var callbackasmABI0 = uintptr(unsafe.Pointer(&__callbackasm))
113
114// callbackWrap_call allows the calling of the ABIInternal wrapper
115// which is required for runtime.cgocallback without the
116// <ABIInternal> tag which is only allowed in the runtime.
117// This closure is used inside sys_darwin_GOARCH.s
118var callbackWrap_call = callbackWrap
119
120// callbackWrap is called by assembly code which determines which Go function to call.
121// This function takes the arguments and passes them to the Go function and returns the result.
122func callbackWrap(a *callbackArgs) {
123 cbs.lock.Lock()
124 fn := cbs.funcs[a.index]
125 cbs.lock.Unlock()
126 fnType := fn.Type()
127 args := make([]reflect.Value, fnType.NumIn())
128 frame := (*[callbackMaxFrame]uintptr)(a.args)
129 // stackFrame points to stack-passed arguments. On most architectures this is
130 // contiguous with frame (after register args), but on ppc64le it's separate.
131 var stackFrame *[callbackMaxFrame]uintptr
132 var intFrame *[callbackMaxFrame]uintptr
133 if sf := a.stackFrame(); sf != nil {
134 // Only ppc64le uses separate stackArgs pointer due to NOSPLIT constraints
135 stackFrame = (*[callbackMaxFrame]uintptr)(sf)
136 }
137 if intf := a.intFrame(); intf != nil {
138 intFrame = (*[callbackMaxFrame]uintptr)(intf)
139 }
140 // floatsN and intsN track the number of register slots used, not argument count.
141 // This distinction matters on ARM32 where float64 uses 2 slots (32-bit registers).
142 var floatsN int
143 var intsN int
144 // On amd64/loong64/ppc64le/riscv64/s390x, when returning a struct larger than
145 // maxRegAllocStructSize, the caller passes a hidden pointer in the first integer
146 // register. Skip it to avoid misreading it as the first function argument.
147 if (runtime.GOARCH == "amd64" || runtime.GOARCH == "loong64" || runtime.GOARCH == "riscv64" || runtime.GOARCH == "s390x") &&
148 fnType.NumOut() == 1 && fnType.Out(0).Kind() == reflect.Struct &&
149 fnType.Out(0).Size() > maxRegAllocStructSize {
150 intsN = 1
151 }
152 // stackSlot points to the index into frame (or stackFrame) of the current stack element.
153 // When stackFrame is nil, stack begins after float and integer registers in frame.
154 // When stackFrame is not nil (ppc64le), stackSlot indexes into stackFrame starting at 0.
155 stackSlot := numOfIntegerRegisters() + numOfFloatRegisters()
156 if stackFrame != nil {
157 // ppc64le: stackArgs is a separate pointer, indices start at 0
158 stackSlot = 0
159 }
160 // stackByteOffset tracks the byte offset within the stack area for Darwin ARM64
161 // tight packing. On Darwin ARM64, C passes small types packed on the stack.
162 stackByteOffset := uintptr(0)
163 for i := range args {
164 // slots is the number of pointer-sized slots the argument takes
165 inType := fnType.In(i)
166 slots := int((inType.Size() + ptrSize - 1) / ptrSize)
167 switch inType.Kind() {
168 case reflect.Float32, reflect.Float64:
169 if isARMSoftFloat() {
170 // we should restore from integer slot, can skip unnecessary branching here
171 if isARMPaddingNeeded(inType, -1, intsN) {
172 intsN++
173 }
174 if intsN+slots <= numOfIntegerRegisters() {
175 // the integers begin after the floats in frame
176 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[intsN+numOfFloatRegisters()])).Elem()
177 intsN += slots
178 continue
179 }
180 if isARMPaddingNeeded(inType, -1, stackSlot) {
181 stackSlot++
182 }
183 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
184 stackSlot += slots
185 intsN += slots
186 continue
187 }
188
189 if floatsN+slots > numOfFloatRegisters() {
190 if isDarwin && runtime.GOARCH == "arm64" {
191 // Darwin ARM64: read from packed stack with proper alignment
192 args[i] = callbackArgFromStack(a.args, stackSlot, &stackByteOffset, inType)
193 } else if stackFrame != nil {
194 // ppc64le/s390x: stack args are in separate stackFrame
195 switch runtime.GOARCH {
196 case "ppc64le":
197 args[i] = callbackFloatFromDoubleSlot(unsafe.Pointer(&stackFrame[stackSlot]), inType)
198 case "s390x":
199 // s390x big-endian: sub-8-byte values are right-justified
200 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&stackFrame[stackSlot]), inType)
201 default:
202 args[i] = reflect.NewAt(inType, unsafe.Pointer(&stackFrame[stackSlot])).Elem()
203 }
204 stackSlot += slots
205 } else if isARMFloatPaddingNeeded(inType, -1, stackSlot) {
206 stackSlot++
207 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
208 stackSlot += slots
209 } else {
210 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
211 stackSlot += slots
212 }
213 } else {
214 switch runtime.GOARCH {
215 case "ppc64le":
216 args[i] = callbackFloatFromDoubleSlot(unsafe.Pointer(&frame[floatsN]), inType)
217 case "s390x":
218 // s390x big-endian: float32 is right-justified in 8-byte FPR slot
219 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&frame[floatsN]), inType)
220 default:
221 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[floatsN])).Elem()
222 }
223 }
224 floatsN += slots
225 case reflect.Struct:
226 if i == 0 && inType.AssignableTo(reflect.TypeFor[CDecl]()) {
227 args[i] = reflect.Zero(inType)
228 continue
229 }
230 if inType.Size() == 0 {
231 args[i] = reflect.New(inType).Elem()
232 continue
233 }
234 args[i] = getCallbackStruct(inType, a.args, &floatsN, &intsN, &stackSlot, &stackByteOffset)
235 continue
236 default:
237 if isARMPaddingNeeded(inType, -1, intsN) {
238 intsN++
239 }
240 if intsN+slots > numOfIntegerRegisters() {
241 if isDarwin && runtime.GOARCH == "arm64" {
242 // Darwin ARM64: read from packed stack with proper alignment
243 args[i] = callbackArgFromStack(a.args, stackSlot, &stackByteOffset, inType)
244 } else if stackFrame != nil {
245 // ppc64le/s390x: stack args are in separate stackFrame
246 if runtime.GOARCH == "s390x" {
247 // s390x big-endian: sub-8-byte values are right-justified
248 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&stackFrame[stackSlot]), inType)
249 } else {
250 args[i] = reflect.NewAt(inType, unsafe.Pointer(&stackFrame[stackSlot])).Elem()
251 }
252 stackSlot += slots
253 } else if isARMPaddingNeeded(inType, -1, stackSlot) {
254 stackSlot++
255 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
256 stackSlot += slots
257 } else {
258 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
259 stackSlot += slots
260 }
261 } else {
262 if intFrame != nil {
263 args[i] = reflect.NewAt(inType, unsafe.Pointer(&intFrame[intsN])).Elem()
264 } else {
265 // the integers begin after the floats in frame
266 pos := intsN + numOfFloatRegisters()
267 if runtime.GOARCH == "s390x" {
268 // s390x big-endian: sub-8-byte values are right-justified in GPR slot
269 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&frame[pos]), inType)
270 } else {
271 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[pos])).Elem()
272 }
273 }
274 }
275 intsN += slots
276 }
277 }
278 ret := fn.Call(args)
279 if len(ret) > 0 {
280 switch k := ret[0].Kind(); k {
281 case reflect.Uint64:
282 a.setUint64Result(ret[0].Uint())
283 case reflect.Uint, reflect.Uint32, reflect.Uint16, reflect.Uint8, reflect.Uintptr:
284 a.result[0] = uintptr(ret[0].Uint())
285 case reflect.Int64:
286 a.setInt64Result(ret[0].Int())
287 case reflect.Int, reflect.Int32, reflect.Int16, reflect.Int8:
288 a.result[0] = uintptr(ret[0].Int())
289 case reflect.Bool:
290 if ret[0].Bool() {
291 a.result[0] = 1
292 } else {
293 a.result[0] = 0
294 }
295 case reflect.Pointer:
296 a.result[0] = ret[0].Pointer()
297 case reflect.UnsafePointer:
298 a.result[0] = ret[0].Pointer()
299 case reflect.Struct:
300 setStruct(a, ret[0])
301 default:
302 panic("purego: unsupported kind: " + k.String())
303 }
304 }
305}
306
307// callbackArgFromStack reads an argument from the tightly-packed stack area on Darwin ARM64.
308// The C ABI on Darwin ARM64 packs small types on the stack without padding to 8 bytes.
309// This function handles proper alignment and advances stackByteOffset accordingly.
310func callbackArgFromStack(argsBase unsafe.Pointer, stackSlot int, stackByteOffset *uintptr, inType reflect.Type) reflect.Value {
311 // Calculate base address of stack area (after float and int registers)
312 stackBase := unsafe.Add(argsBase, stackSlot*int(ptrSize))
313
314 // Get type's natural alignment
315 align := uintptr(inType.Align())
316 size := inType.Size()
317
318 // Align the offset
319 if *stackByteOffset%align != 0 {
320 *stackByteOffset = (*stackByteOffset + align - 1) &^ (align - 1)
321 }
322
323 // Read value at aligned offset
324 ptr := unsafe.Add(stackBase, *stackByteOffset)
325 *stackByteOffset += size
326
327 return reflect.NewAt(inType, ptr).Elem()
328}
329
330// callbackFloatFromDoubleSlot reads a floating-point callback argument from an
331// 8-byte register or stack slot on ppc64le, where a single-precision value is
332// held in double-precision format.
333func callbackFloatFromDoubleSlot(slotPtr unsafe.Pointer, inType reflect.Type) reflect.Value {
334 if inType.Kind() != reflect.Float32 {
335 return reflect.NewAt(inType, slotPtr).Elem()
336 }
337 v := reflect.New(inType).Elem()
338 v.SetFloat(math.Float64frombits(*(*uint64)(slotPtr)))
339 return v
340}
341
342// callbackArgFromSlotBigEndian reads an argument from an 8-byte slot on big-endian architectures.
343// On s390x:
344// - Integer types are right-justified in GPRs: sub-8-byte values are at offset (8 - size)
345// - Float32 in FPRs is left-justified: stored in upper 32 bits, so at offset 0
346// - Float64 occupies the full 8-byte slot
347func callbackArgFromSlotBigEndian(slotPtr unsafe.Pointer, inType reflect.Type) reflect.Value {
348 size := inType.Size()
349 if size >= 8 {
350 // 8-byte values occupy the entire slot
351 return reflect.NewAt(inType, slotPtr).Elem()
352 }
353 // Float32 is left-justified in FPRs (upper 32 bits), so offset is 0
354 if inType.Kind() == reflect.Float32 {
355 return reflect.NewAt(inType, slotPtr).Elem()
356 }
357 // Integer types are right-justified: offset = 8 - size
358 offset := 8 - size
359 ptr := unsafe.Add(slotPtr, offset)
360 return reflect.NewAt(inType, ptr).Elem()
361}
362
363// callbackasmAddr returns address of runtime.callbackasm
364// function adjusted by i.
365// On x86 and amd64, runtime.callbackasm is a series of CALL instructions,
366// and we want callback to arrive at
367// correspondent call instruction instead of start of
368// runtime.callbackasm.
369// On ARM, runtime.callbackasm is a series of mov and branch instructions.
370// R12 is loaded with the callback index. Each entry is two instructions,
371// hence 8 bytes.
372func callbackasmAddr(i int) uintptr {
373 var entrySize int
374 switch runtime.GOARCH {
375 default:
376 panic("purego: unsupported architecture")
377 case "amd64":
378 // On amd64, each callback entry is just a CALL instruction (5 bytes)
379 entrySize = 5
380 case "386":
381 // On 386, each callback entry is MOVL $imm, CX (5 bytes) + JMP (5 bytes)
382 entrySize = 10
383 case "arm", "arm64", "loong64", "ppc64le", "riscv64":
384 // On ARM, ARM64, Loong64, PPC64LE and RISCV64, each entry is a MOV instruction
385 // followed by a branch instruction
386 entrySize = 8
387 case "s390x":
388 // On S390X, each entry is LGHI (4 bytes) + JG (6 bytes)
389 entrySize = 10
390 }
391 return callbackasmABI0 + uintptr(i*entrySize)
392}