1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: 2022 The Ebitengine Authors
3
4// TODO: remove s390x cgo dependency once golang/go#77449 is resolved
5//go:build darwin || freebsd || (linux && (386 || amd64 || arm || arm64 || loong64 || ppc64le || riscv64 || (cgo && s390x))) || netbsd
6
7package purego
8
9import (
10 "reflect"
11 "runtime"
12 "sync"
13 "unsafe"
14)
15
16var syscall15XABI0 uintptr
17
18func syscall_syscall15X(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15 uintptr) (r1, r2, err uintptr) {
19 args := thePool.Get().(*syscall15Args)
20 defer thePool.Put(args)
21
22 *args = syscall15Args{
23 fn: fn,
24 a1: a1, a2: a2, a3: a3, a4: a4, a5: a5, a6: a6, a7: a7, a8: a8,
25 a9: a9, a10: a10, a11: a11, a12: a12, a13: a13, a14: a14, a15: a15,
26 f1: a1, f2: a2, f3: a3, f4: a4, f5: a5, f6: a6, f7: a7, f8: a8,
27 }
28
29 runtime_cgocall(syscall15XABI0, unsafe.Pointer(args))
30 return args.a1, args.a2, args.a3
31}
32
33// NewCallback converts a Go function to a function pointer conforming to the C calling convention.
34// This is useful when interoperating with C code requiring callbacks. The argument is expected to be a
35// function with zero or one uintptr-sized result. The function must not have arguments with size larger than the size
36// of uintptr. Only a limited number of callbacks may be created in a single Go process, and any memory allocated
37// for these callbacks is never released. At least 2000 callbacks can always be created. Although this function
38// provides similar functionality to windows.NewCallback it is distinct.
39func NewCallback(fn any) uintptr {
40 ty := reflect.TypeOf(fn)
41 for i := 0; i < ty.NumIn(); i++ {
42 in := ty.In(i)
43 if !in.AssignableTo(reflect.TypeOf(CDecl{})) {
44 continue
45 }
46 if i != 0 {
47 panic("purego: CDecl must be the first argument")
48 }
49 }
50 return compileCallback(fn)
51}
52
53// maxCb is the maximum number of callbacks
54// only increase this if you have added more to the callbackasm function
55const maxCB = 2000
56
57var cbs struct {
58 lock sync.Mutex
59 numFn int // the number of functions currently in cbs.funcs
60 funcs [maxCB]reflect.Value // the saved callbacks
61}
62
63func compileCallback(fn any) uintptr {
64 val := reflect.ValueOf(fn)
65 if val.Kind() != reflect.Func {
66 panic("purego: the type must be a function but was not")
67 }
68 if val.IsNil() {
69 panic("purego: function must not be nil")
70 }
71 ty := val.Type()
72 for i := 0; i < ty.NumIn(); i++ {
73 in := ty.In(i)
74 switch in.Kind() {
75 case reflect.Struct:
76 if i == 0 && in.AssignableTo(reflect.TypeOf(CDecl{})) {
77 continue
78 }
79 ensureStructSupported()
80 checkStructFieldsSupported(in)
81 continue
82 case reflect.Interface, reflect.Func, reflect.Slice,
83 reflect.Chan, reflect.Complex64, reflect.Complex128,
84 reflect.String, reflect.Map, reflect.Invalid:
85 panic("purego: unsupported argument type: " + in.Kind().String())
86 }
87 }
88output:
89 switch {
90 case ty.NumOut() == 1:
91 switch ty.Out(0).Kind() {
92 case reflect.Pointer, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
93 reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
94 reflect.Bool, reflect.UnsafePointer, reflect.Struct:
95 break output
96 }
97 panic("purego: unsupported return type: " + ty.String())
98 case ty.NumOut() > 1:
99 panic("purego: callbacks can only have one return")
100 }
101 cbs.lock.Lock()
102 defer cbs.lock.Unlock()
103 if cbs.numFn >= maxCB {
104 panic("purego: the maximum number of callbacks has been reached")
105 }
106 cbs.funcs[cbs.numFn] = val
107 cbs.numFn++
108 return callbackasmAddr(cbs.numFn - 1)
109}
110
111const ptrSize = unsafe.Sizeof((*int)(nil))
112
113const callbackMaxFrame = 64 * ptrSize
114
115// callbackasm is implemented in zcallback_GOOS_GOARCH.s
116//
117//go:linkname __callbackasm callbackasm
118var __callbackasm byte
119var callbackasmABI0 = uintptr(unsafe.Pointer(&__callbackasm))
120
121// callbackWrap_call allows the calling of the ABIInternal wrapper
122// which is required for runtime.cgocallback without the
123// <ABIInternal> tag which is only allowed in the runtime.
124// This closure is used inside sys_darwin_GOARCH.s
125var callbackWrap_call = callbackWrap
126
127// callbackWrap is called by assembly code which determines which Go function to call.
128// This function takes the arguments and passes them to the Go function and returns the result.
129func callbackWrap(a *callbackArgs) {
130 cbs.lock.Lock()
131 fn := cbs.funcs[a.index]
132 cbs.lock.Unlock()
133 fnType := fn.Type()
134 args := make([]reflect.Value, fnType.NumIn())
135 frame := (*[callbackMaxFrame]uintptr)(a.args)
136 // stackFrame points to stack-passed arguments. On most architectures this is
137 // contiguous with frame (after register args), but on ppc64le it's separate.
138 var stackFrame *[callbackMaxFrame]uintptr
139 if sf := a.stackFrame(); sf != nil {
140 // Only ppc64le uses separate stackArgs pointer due to NOSPLIT constraints
141 stackFrame = (*[callbackMaxFrame]uintptr)(sf)
142 }
143 // floatsN and intsN track the number of register slots used, not argument count.
144 // This distinction matters on ARM32 where float64 uses 2 slots (32-bit registers).
145 var floatsN int
146 var intsN int
147 // On amd64/loong64/ppc64le/riscv64/s390x, when returning a struct larger than
148 // maxRegAllocStructSize, the caller passes a hidden pointer in the first integer
149 // register. Skip it to avoid misreading it as the first function argument.
150 if (runtime.GOARCH == "amd64" || runtime.GOARCH == "loong64" || runtime.GOARCH == "ppc64le" || runtime.GOARCH == "riscv64" || runtime.GOARCH == "s390x") &&
151 fnType.NumOut() == 1 && fnType.Out(0).Kind() == reflect.Struct &&
152 fnType.Out(0).Size() > maxRegAllocStructSize {
153 intsN = 1
154 }
155 // stackSlot points to the index into frame (or stackFrame) of the current stack element.
156 // When stackFrame is nil, stack begins after float and integer registers in frame.
157 // When stackFrame is not nil (ppc64le), stackSlot indexes into stackFrame starting at 0.
158 stackSlot := numOfIntegerRegisters() + numOfFloatRegisters()
159 if stackFrame != nil {
160 // ppc64le: stackArgs is a separate pointer, indices start at 0
161 stackSlot = 0
162 }
163 // stackByteOffset tracks the byte offset within the stack area for Darwin ARM64
164 // tight packing. On Darwin ARM64, C passes small types packed on the stack.
165 stackByteOffset := uintptr(0)
166 for i := range args {
167 // slots is the number of pointer-sized slots the argument takes
168 var slots int
169 inType := fnType.In(i)
170 switch inType.Kind() {
171 case reflect.Float32, reflect.Float64:
172 slots = int((fnType.In(i).Size() + ptrSize - 1) / ptrSize)
173 if floatsN+slots > numOfFloatRegisters() {
174 if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
175 // Darwin ARM64: read from packed stack with proper alignment
176 args[i] = callbackArgFromStack(a.args, stackSlot, &stackByteOffset, inType)
177 } else if stackFrame != nil {
178 // ppc64le/s390x: stack args are in separate stackFrame
179 if runtime.GOARCH == "s390x" {
180 // s390x big-endian: sub-8-byte values are right-justified
181 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&stackFrame[stackSlot]), inType)
182 } else {
183 args[i] = reflect.NewAt(inType, unsafe.Pointer(&stackFrame[stackSlot])).Elem()
184 }
185 stackSlot += slots
186 } else {
187 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
188 stackSlot += slots
189 }
190 } else {
191 if runtime.GOARCH == "s390x" {
192 // s390x big-endian: float32 is right-justified in 8-byte FPR slot
193 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&frame[floatsN]), inType)
194 } else {
195 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[floatsN])).Elem()
196 }
197 }
198 floatsN += slots
199 case reflect.Struct:
200 if i == 0 && inType.AssignableTo(reflect.TypeOf(CDecl{})) {
201 args[i] = reflect.Zero(inType)
202 continue
203 }
204 if inType.Size() == 0 {
205 args[i] = reflect.New(inType).Elem()
206 continue
207 }
208 args[i] = getCallbackStruct(inType, a.args, &floatsN, &intsN, &stackSlot, &stackByteOffset)
209 continue
210 default:
211 slots = int((inType.Size() + ptrSize - 1) / ptrSize)
212 if intsN+slots > numOfIntegerRegisters() {
213 if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
214 // Darwin ARM64: read from packed stack with proper alignment
215 args[i] = callbackArgFromStack(a.args, stackSlot, &stackByteOffset, inType)
216 } else if stackFrame != nil {
217 // ppc64le/s390x: stack args are in separate stackFrame
218 if runtime.GOARCH == "s390x" {
219 // s390x big-endian: sub-8-byte values are right-justified
220 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&stackFrame[stackSlot]), inType)
221 } else {
222 args[i] = reflect.NewAt(inType, unsafe.Pointer(&stackFrame[stackSlot])).Elem()
223 }
224 stackSlot += slots
225 } else {
226 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
227 stackSlot += slots
228 }
229 } else {
230 // the integers begin after the floats in frame
231 pos := intsN + numOfFloatRegisters()
232 if runtime.GOARCH == "s390x" {
233 // s390x big-endian: sub-8-byte values are right-justified in GPR slot
234 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&frame[pos]), inType)
235 } else {
236 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[pos])).Elem()
237 }
238 }
239 intsN += slots
240 }
241 }
242 ret := fn.Call(args)
243 if len(ret) > 0 {
244 switch k := ret[0].Kind(); k {
245 case reflect.Uint, reflect.Uint64, reflect.Uint32, reflect.Uint16, reflect.Uint8, reflect.Uintptr:
246 a.result[0] = uintptr(ret[0].Uint())
247 case reflect.Int, reflect.Int64, reflect.Int32, reflect.Int16, reflect.Int8:
248 a.result[0] = uintptr(ret[0].Int())
249 case reflect.Bool:
250 if ret[0].Bool() {
251 a.result[0] = 1
252 } else {
253 a.result[0] = 0
254 }
255 case reflect.Pointer:
256 a.result[0] = ret[0].Pointer()
257 case reflect.UnsafePointer:
258 a.result[0] = ret[0].Pointer()
259 case reflect.Struct:
260 setStruct(a, ret[0])
261 default:
262 panic("purego: unsupported kind: " + k.String())
263 }
264 }
265}
266
267// callbackArgFromStack reads an argument from the tightly-packed stack area on Darwin ARM64.
268// The C ABI on Darwin ARM64 packs small types on the stack without padding to 8 bytes.
269// This function handles proper alignment and advances stackByteOffset accordingly.
270func callbackArgFromStack(argsBase unsafe.Pointer, stackSlot int, stackByteOffset *uintptr, inType reflect.Type) reflect.Value {
271 // Calculate base address of stack area (after float and int registers)
272 stackBase := unsafe.Add(argsBase, stackSlot*int(ptrSize))
273
274 // Get type's natural alignment
275 align := uintptr(inType.Align())
276 size := inType.Size()
277
278 // Align the offset
279 if *stackByteOffset%align != 0 {
280 *stackByteOffset = (*stackByteOffset + align - 1) &^ (align - 1)
281 }
282
283 // Read value at aligned offset
284 ptr := unsafe.Add(stackBase, *stackByteOffset)
285 *stackByteOffset += size
286
287 return reflect.NewAt(inType, ptr).Elem()
288}
289
290// callbackArgFromSlotBigEndian reads an argument from an 8-byte slot on big-endian architectures.
291// On s390x:
292// - Integer types are right-justified in GPRs: sub-8-byte values are at offset (8 - size)
293// - Float32 in FPRs is left-justified: stored in upper 32 bits, so at offset 0
294// - Float64 occupies the full 8-byte slot
295func callbackArgFromSlotBigEndian(slotPtr unsafe.Pointer, inType reflect.Type) reflect.Value {
296 size := inType.Size()
297 if size >= 8 {
298 // 8-byte values occupy the entire slot
299 return reflect.NewAt(inType, slotPtr).Elem()
300 }
301 // Float32 is left-justified in FPRs (upper 32 bits), so offset is 0
302 if inType.Kind() == reflect.Float32 {
303 return reflect.NewAt(inType, slotPtr).Elem()
304 }
305 // Integer types are right-justified: offset = 8 - size
306 offset := 8 - size
307 ptr := unsafe.Add(slotPtr, offset)
308 return reflect.NewAt(inType, ptr).Elem()
309}
310
311// callbackasmAddr returns address of runtime.callbackasm
312// function adjusted by i.
313// On x86 and amd64, runtime.callbackasm is a series of CALL instructions,
314// and we want callback to arrive at
315// correspondent call instruction instead of start of
316// runtime.callbackasm.
317// On ARM, runtime.callbackasm is a series of mov and branch instructions.
318// R12 is loaded with the callback index. Each entry is two instructions,
319// hence 8 bytes.
320func callbackasmAddr(i int) uintptr {
321 var entrySize int
322 switch runtime.GOARCH {
323 default:
324 panic("purego: unsupported architecture")
325 case "amd64":
326 // On amd64, each callback entry is just a CALL instruction (5 bytes)
327 entrySize = 5
328 case "386":
329 // On 386, each callback entry is MOVL $imm, CX (5 bytes) + JMP (5 bytes)
330 entrySize = 10
331 case "arm", "arm64", "loong64", "ppc64le", "riscv64":
332 // On ARM, ARM64, Loong64, PPC64LE and RISCV64, each entry is a MOV instruction
333 // followed by a branch instruction
334 entrySize = 8
335 case "s390x":
336 // On S390X, each entry is LGHI (4 bytes) + JG (6 bytes)
337 entrySize = 10
338 }
339 return callbackasmABI0 + uintptr(i*entrySize)
340}