1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: 2022 The Ebitengine Authors
3
4//go:build darwin || freebsd || (linux && (386 || amd64 || arm || arm64 || loong64 || ppc64le || riscv64 || (s390x && (cgo || go1.27)))) || netbsd
5
6package purego
7
8import (
9 "reflect"
10 "runtime"
11 "sync"
12 "unsafe"
13)
14
15var syscallXABI0 uintptr
16
17func syscall_syscallN(fn uintptr, args ...uintptr) (r1, r2, err uintptr) {
18 panic("purego: syscall_syscallN is only supported on windows")
19}
20
21// NewCallback converts a Go function to a function pointer conforming to the C calling convention.
22// This is useful when interoperating with C code requiring callbacks. The argument is expected to be a
23// function with zero or one uintptr-sized result. The function must not have arguments with size larger than the size
24// of uintptr. Only a limited number of callbacks may be created in a single Go process, and any memory allocated
25// for these callbacks is never released. At least 2000 callbacks can always be created. Although this function
26// provides similar functionality to windows.NewCallback it is distinct.
27func NewCallback(fn any) uintptr {
28 ty := reflect.TypeOf(fn)
29 for i := 0; i < ty.NumIn(); i++ {
30 in := ty.In(i)
31 if !in.AssignableTo(reflect.TypeFor[CDecl]()) {
32 continue
33 }
34 if i != 0 {
35 panic("purego: CDecl must be the first argument")
36 }
37 }
38 return compileCallback(fn)
39}
40
41// maxCb is the maximum number of callbacks
42// only increase this if you have added more to the callbackasm function
43const maxCB = 2000
44
45var cbs struct {
46 lock sync.Mutex
47 numFn int // the number of functions currently in cbs.funcs
48 funcs [maxCB]reflect.Value // the saved callbacks
49}
50
51func compileCallback(fn any) uintptr {
52 val := reflect.ValueOf(fn)
53 if val.Kind() != reflect.Func {
54 panic("purego: the type must be a function but was not")
55 }
56 if val.IsNil() {
57 panic("purego: function must not be nil")
58 }
59 ty := val.Type()
60 for i := 0; i < ty.NumIn(); i++ {
61 in := ty.In(i)
62 switch in.Kind() {
63 case reflect.Struct:
64 if i == 0 && in.AssignableTo(reflect.TypeFor[CDecl]()) {
65 continue
66 }
67 ensureStructSupported()
68 checkStructFieldsSupported(in)
69 continue
70 case reflect.Interface, reflect.Func, reflect.Slice,
71 reflect.Chan, reflect.Complex64, reflect.Complex128,
72 reflect.String, reflect.Map, reflect.Invalid:
73 panic("purego: unsupported argument type: " + in.Kind().String())
74 }
75 }
76output:
77 switch {
78 case ty.NumOut() == 1:
79 switch ty.Out(0).Kind() {
80 case reflect.Pointer, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
81 reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
82 reflect.Bool, reflect.UnsafePointer, reflect.Struct:
83 break output
84 }
85 panic("purego: unsupported return type: " + ty.String())
86 case ty.NumOut() > 1:
87 panic("purego: callbacks can only have one return")
88 }
89 cbs.lock.Lock()
90 defer cbs.lock.Unlock()
91 if cbs.numFn >= maxCB {
92 panic("purego: the maximum number of callbacks has been reached")
93 }
94 cbs.funcs[cbs.numFn] = val
95 cbs.numFn++
96 return callbackasmAddr(cbs.numFn - 1)
97}
98
99const ptrSize = unsafe.Sizeof((*int)(nil))
100
101const callbackMaxFrame = 64 * ptrSize
102
103// callbackasm is implemented in zcallback_GOOS_GOARCH.s
104//
105//go:linkname __callbackasm callbackasm
106var __callbackasm byte
107var callbackasmABI0 = uintptr(unsafe.Pointer(&__callbackasm))
108
109// callbackWrap_call allows the calling of the ABIInternal wrapper
110// which is required for runtime.cgocallback without the
111// <ABIInternal> tag which is only allowed in the runtime.
112// This closure is used inside sys_darwin_GOARCH.s
113var callbackWrap_call = callbackWrap
114
115// callbackWrap is called by assembly code which determines which Go function to call.
116// This function takes the arguments and passes them to the Go function and returns the result.
117func callbackWrap(a *callbackArgs) {
118 cbs.lock.Lock()
119 fn := cbs.funcs[a.index]
120 cbs.lock.Unlock()
121 fnType := fn.Type()
122 args := make([]reflect.Value, fnType.NumIn())
123 frame := (*[callbackMaxFrame]uintptr)(a.args)
124 // stackFrame points to stack-passed arguments. On most architectures this is
125 // contiguous with frame (after register args), but on ppc64le it's separate.
126 var stackFrame *[callbackMaxFrame]uintptr
127 var intFrame *[callbackMaxFrame]uintptr
128 if sf := a.stackFrame(); sf != nil {
129 // Only ppc64le uses separate stackArgs pointer due to NOSPLIT constraints
130 stackFrame = (*[callbackMaxFrame]uintptr)(sf)
131 }
132 if intf := a.intFrame(); intf != nil {
133 intFrame = (*[callbackMaxFrame]uintptr)(intf)
134 }
135 // floatsN and intsN track the number of register slots used, not argument count.
136 // This distinction matters on ARM32 where float64 uses 2 slots (32-bit registers).
137 var floatsN int
138 var intsN int
139 // On amd64/loong64/ppc64le/riscv64/s390x, when returning a struct larger than
140 // maxRegAllocStructSize, the caller passes a hidden pointer in the first integer
141 // register. Skip it to avoid misreading it as the first function argument.
142 if (runtime.GOARCH == "amd64" || runtime.GOARCH == "loong64" || runtime.GOARCH == "riscv64" || runtime.GOARCH == "s390x") &&
143 fnType.NumOut() == 1 && fnType.Out(0).Kind() == reflect.Struct &&
144 fnType.Out(0).Size() > maxRegAllocStructSize {
145 intsN = 1
146 }
147 // stackSlot points to the index into frame (or stackFrame) of the current stack element.
148 // When stackFrame is nil, stack begins after float and integer registers in frame.
149 // When stackFrame is not nil (ppc64le), stackSlot indexes into stackFrame starting at 0.
150 stackSlot := numOfIntegerRegisters() + numOfFloatRegisters()
151 if stackFrame != nil {
152 // ppc64le: stackArgs is a separate pointer, indices start at 0
153 stackSlot = 0
154 }
155 // stackByteOffset tracks the byte offset within the stack area for Darwin ARM64
156 // tight packing. On Darwin ARM64, C passes small types packed on the stack.
157 stackByteOffset := uintptr(0)
158 for i := range args {
159 // slots is the number of pointer-sized slots the argument takes
160 var slots int
161 inType := fnType.In(i)
162 switch inType.Kind() {
163 case reflect.Float32, reflect.Float64:
164 slots = int((fnType.In(i).Size() + ptrSize - 1) / ptrSize)
165 if floatsN+slots > numOfFloatRegisters() {
166 if isDarwin && runtime.GOARCH == "arm64" {
167 // Darwin ARM64: read from packed stack with proper alignment
168 args[i] = callbackArgFromStack(a.args, stackSlot, &stackByteOffset, inType)
169 } else if stackFrame != nil {
170 // ppc64le/s390x: stack args are in separate stackFrame
171 if runtime.GOARCH == "s390x" {
172 // s390x big-endian: sub-8-byte values are right-justified
173 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&stackFrame[stackSlot]), inType)
174 } else {
175 args[i] = reflect.NewAt(inType, unsafe.Pointer(&stackFrame[stackSlot])).Elem()
176 }
177 stackSlot += slots
178 } else {
179 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
180 stackSlot += slots
181 }
182 } else {
183 if runtime.GOARCH == "s390x" {
184 // s390x big-endian: float32 is right-justified in 8-byte FPR slot
185 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&frame[floatsN]), inType)
186 } else {
187 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[floatsN])).Elem()
188 }
189 }
190 floatsN += slots
191 case reflect.Struct:
192 if i == 0 && inType.AssignableTo(reflect.TypeFor[CDecl]()) {
193 args[i] = reflect.Zero(inType)
194 continue
195 }
196 if inType.Size() == 0 {
197 args[i] = reflect.New(inType).Elem()
198 continue
199 }
200 args[i] = getCallbackStruct(inType, a.args, &floatsN, &intsN, &stackSlot, &stackByteOffset)
201 continue
202 default:
203 slots = int((inType.Size() + ptrSize - 1) / ptrSize)
204 if intsN+slots > numOfIntegerRegisters() {
205 if isDarwin && runtime.GOARCH == "arm64" {
206 // Darwin ARM64: read from packed stack with proper alignment
207 args[i] = callbackArgFromStack(a.args, stackSlot, &stackByteOffset, inType)
208 } else if stackFrame != nil {
209 // ppc64le/s390x: stack args are in separate stackFrame
210 if runtime.GOARCH == "s390x" {
211 // s390x big-endian: sub-8-byte values are right-justified
212 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&stackFrame[stackSlot]), inType)
213 } else {
214 args[i] = reflect.NewAt(inType, unsafe.Pointer(&stackFrame[stackSlot])).Elem()
215 }
216 stackSlot += slots
217 } else {
218 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
219 stackSlot += slots
220 }
221 } else {
222 if intFrame != nil {
223 args[i] = reflect.NewAt(inType, unsafe.Pointer(&intFrame[intsN])).Elem()
224 } else {
225 // the integers begin after the floats in frame
226 pos := intsN + numOfFloatRegisters()
227 if runtime.GOARCH == "s390x" {
228 // s390x big-endian: sub-8-byte values are right-justified in GPR slot
229 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&frame[pos]), inType)
230 } else {
231 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[pos])).Elem()
232 }
233 }
234 }
235 intsN += slots
236 }
237 }
238 ret := fn.Call(args)
239 if len(ret) > 0 {
240 switch k := ret[0].Kind(); k {
241 case reflect.Uint, reflect.Uint64, reflect.Uint32, reflect.Uint16, reflect.Uint8, reflect.Uintptr:
242 a.result[0] = uintptr(ret[0].Uint())
243 case reflect.Int, reflect.Int64, reflect.Int32, reflect.Int16, reflect.Int8:
244 a.result[0] = uintptr(ret[0].Int())
245 case reflect.Bool:
246 if ret[0].Bool() {
247 a.result[0] = 1
248 } else {
249 a.result[0] = 0
250 }
251 case reflect.Pointer:
252 a.result[0] = ret[0].Pointer()
253 case reflect.UnsafePointer:
254 a.result[0] = ret[0].Pointer()
255 case reflect.Struct:
256 setStruct(a, ret[0])
257 default:
258 panic("purego: unsupported kind: " + k.String())
259 }
260 }
261}
262
263// callbackArgFromStack reads an argument from the tightly-packed stack area on Darwin ARM64.
264// The C ABI on Darwin ARM64 packs small types on the stack without padding to 8 bytes.
265// This function handles proper alignment and advances stackByteOffset accordingly.
266func callbackArgFromStack(argsBase unsafe.Pointer, stackSlot int, stackByteOffset *uintptr, inType reflect.Type) reflect.Value {
267 // Calculate base address of stack area (after float and int registers)
268 stackBase := unsafe.Add(argsBase, stackSlot*int(ptrSize))
269
270 // Get type's natural alignment
271 align := uintptr(inType.Align())
272 size := inType.Size()
273
274 // Align the offset
275 if *stackByteOffset%align != 0 {
276 *stackByteOffset = (*stackByteOffset + align - 1) &^ (align - 1)
277 }
278
279 // Read value at aligned offset
280 ptr := unsafe.Add(stackBase, *stackByteOffset)
281 *stackByteOffset += size
282
283 return reflect.NewAt(inType, ptr).Elem()
284}
285
286// callbackArgFromSlotBigEndian reads an argument from an 8-byte slot on big-endian architectures.
287// On s390x:
288// - Integer types are right-justified in GPRs: sub-8-byte values are at offset (8 - size)
289// - Float32 in FPRs is left-justified: stored in upper 32 bits, so at offset 0
290// - Float64 occupies the full 8-byte slot
291func callbackArgFromSlotBigEndian(slotPtr unsafe.Pointer, inType reflect.Type) reflect.Value {
292 size := inType.Size()
293 if size >= 8 {
294 // 8-byte values occupy the entire slot
295 return reflect.NewAt(inType, slotPtr).Elem()
296 }
297 // Float32 is left-justified in FPRs (upper 32 bits), so offset is 0
298 if inType.Kind() == reflect.Float32 {
299 return reflect.NewAt(inType, slotPtr).Elem()
300 }
301 // Integer types are right-justified: offset = 8 - size
302 offset := 8 - size
303 ptr := unsafe.Add(slotPtr, offset)
304 return reflect.NewAt(inType, ptr).Elem()
305}
306
307// callbackasmAddr returns address of runtime.callbackasm
308// function adjusted by i.
309// On x86 and amd64, runtime.callbackasm is a series of CALL instructions,
310// and we want callback to arrive at
311// correspondent call instruction instead of start of
312// runtime.callbackasm.
313// On ARM, runtime.callbackasm is a series of mov and branch instructions.
314// R12 is loaded with the callback index. Each entry is two instructions,
315// hence 8 bytes.
316func callbackasmAddr(i int) uintptr {
317 var entrySize int
318 switch runtime.GOARCH {
319 default:
320 panic("purego: unsupported architecture")
321 case "amd64":
322 // On amd64, each callback entry is just a CALL instruction (5 bytes)
323 entrySize = 5
324 case "386":
325 // On 386, each callback entry is MOVL $imm, CX (5 bytes) + JMP (5 bytes)
326 entrySize = 10
327 case "arm", "arm64", "loong64", "ppc64le", "riscv64":
328 // On ARM, ARM64, Loong64, PPC64LE and RISCV64, each entry is a MOV instruction
329 // followed by a branch instruction
330 entrySize = 8
331 case "s390x":
332 // On S390X, each entry is LGHI (4 bytes) + JG (6 bytes)
333 entrySize = 10
334 }
335 return callbackasmABI0 + uintptr(i*entrySize)
336}