1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: 2022 The Ebitengine Authors
3
4// TODO: remove s390x cgo dependency once golang/go#77449 is resolved
5//go:build darwin || freebsd || (linux && (386 || amd64 || arm || arm64 || loong64 || ppc64le || riscv64 || (cgo && s390x))) || netbsd
6
7package purego
8
9import (
10 "reflect"
11 "runtime"
12 "sync"
13 "unsafe"
14
15 "github.com/bnema/purego/internal/strings"
16)
17
18var syscall15XABI0 uintptr
19
20func syscall_syscall15X(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15 uintptr) (r1, r2, err uintptr) {
21 args := thePool.Get().(*syscall15Args)
22 defer thePool.Put(args)
23
24 *args = syscall15Args{
25 fn: fn,
26 a1: a1, a2: a2, a3: a3, a4: a4, a5: a5, a6: a6, a7: a7, a8: a8,
27 a9: a9, a10: a10, a11: a11, a12: a12, a13: a13, a14: a14, a15: a15,
28 f1: a1, f2: a2, f3: a3, f4: a4, f5: a5, f6: a6, f7: a7, f8: a8,
29 }
30
31 runtime_cgocall(syscall15XABI0, unsafe.Pointer(args))
32 return args.a1, args.a2, args.a3
33}
34
35// NewCallback converts a Go function to a function pointer conforming to the C calling convention.
36// This is useful when interoperating with C code requiring callbacks. The argument is expected to be a
37// function with zero or one uintptr-sized result. The function must not have arguments with size larger than the size
38// of uintptr. Only a limited number of callbacks may be created in a single Go process, and any memory allocated
39// for these callbacks is never released. At least 2000 callbacks can always be created. Although this function
40// provides similar functionality to windows.NewCallback it is distinct.
41func NewCallback(fn any) uintptr {
42 ty := reflect.TypeOf(fn)
43 for i := 0; i < ty.NumIn(); i++ {
44 in := ty.In(i)
45 if !in.AssignableTo(reflect.TypeOf(CDecl{})) {
46 continue
47 }
48 if i != 0 {
49 panic("purego: CDecl must be the first argument")
50 }
51 }
52 return compileCallback(fn)
53}
54
55// maxCb is the maximum number of callbacks
56// only increase this if you have added more to the callbackasm function
57const maxCB = 2000
58
59var cbs struct {
60 lock sync.RWMutex
61 numFn int // the number of functions currently in cbs.funcs
62 funcs [maxCB]reflect.Value // the saved callbacks
63 argPools [maxCB]*sync.Pool // pre-allocated argument buffers per callback
64}
65
66func compileCallback(fn any) uintptr {
67 val := reflect.ValueOf(fn)
68 if val.Kind() != reflect.Func {
69 panic("purego: the type must be a function but was not")
70 }
71 if val.IsNil() {
72 panic("purego: function must not be nil")
73 }
74 ty := val.Type()
75 for i := 0; i < ty.NumIn(); i++ {
76 in := ty.In(i)
77 switch in.Kind() {
78 case reflect.Struct:
79 if i == 0 && in.AssignableTo(reflect.TypeOf(CDecl{})) {
80 continue
81 }
82 ensureStructSupported()
83 checkStructFieldsSupported(in)
84 continue
85 case reflect.Interface, reflect.Func, reflect.Slice,
86 reflect.Chan, reflect.Complex64, reflect.Complex128,
87 reflect.Map, reflect.Invalid:
88 panic("purego: unsupported argument type: " + in.Kind().String())
89 }
90 }
91output:
92 switch {
93 case ty.NumOut() == 1:
94 switch ty.Out(0).Kind() {
95 case reflect.Pointer, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
96 reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
97 reflect.Bool, reflect.UnsafePointer, reflect.Struct:
98 break output
99 }
100 panic("purego: unsupported return type: " + ty.String())
101 case ty.NumOut() > 1:
102 panic("purego: callbacks can only have one return")
103 }
104 cbs.lock.Lock()
105 defer cbs.lock.Unlock()
106 if cbs.numFn >= maxCB {
107 panic("purego: the maximum number of callbacks has been reached")
108 }
109 index := cbs.numFn
110 cbs.funcs[index] = val
111 numIn := ty.NumIn()
112 cbs.argPools[index] = &sync.Pool{
113 New: func() any {
114 return make([]reflect.Value, numIn)
115 },
116 }
117 cbs.numFn++
118 return callbackasmAddr(index)
119}
120
121const ptrSize = unsafe.Sizeof((*int)(nil))
122
123const callbackMaxFrame = 64 * ptrSize
124
125// callbackasm is implemented in zcallback_GOOS_GOARCH.s
126//
127//go:linkname __callbackasm callbackasm
128var __callbackasm byte
129var callbackasmABI0 = uintptr(unsafe.Pointer(&__callbackasm))
130
131// callbackWrap_call allows the calling of the ABIInternal wrapper
132// which is required for runtime.cgocallback without the
133// <ABIInternal> tag which is only allowed in the runtime.
134// This closure is used inside sys_darwin_GOARCH.s
135var callbackWrap_call = callbackWrap
136
137// callbackWrap is called by assembly code which determines which Go function to call.
138// This function takes the arguments and passes them to the Go function and returns the result.
139func callbackWrap(a *callbackArgs) {
140 cbs.lock.RLock()
141 fn := cbs.funcs[a.index]
142 pool := cbs.argPools[a.index]
143 cbs.lock.RUnlock()
144 fnType := fn.Type()
145 args := pool.Get().([]reflect.Value)
146 defer func() {
147 for i := range args {
148 args[i] = reflect.Value{}
149 }
150 pool.Put(args)
151 }()
152 frame := (*[callbackMaxFrame]uintptr)(a.args)
153 // stackFrame points to stack-passed arguments. On most architectures this is
154 // contiguous with frame (after register args), but on ppc64le it's separate.
155 var stackFrame *[callbackMaxFrame]uintptr
156 if sf := a.stackFrame(); sf != nil {
157 // Only ppc64le uses separate stackArgs pointer due to NOSPLIT constraints
158 stackFrame = (*[callbackMaxFrame]uintptr)(sf)
159 }
160 // floatsN and intsN track the number of register slots used, not argument count.
161 // This distinction matters on ARM32 where float64 uses 2 slots (32-bit registers).
162 var floatsN int
163 var intsN int
164 // On amd64/loong64/ppc64le/riscv64/s390x, when returning a struct larger than
165 // maxRegAllocStructSize, the caller passes a hidden pointer in the first integer
166 // register. Skip it to avoid misreading it as the first function argument.
167 if (runtime.GOARCH == "amd64" || runtime.GOARCH == "loong64" || runtime.GOARCH == "ppc64le" || runtime.GOARCH == "riscv64" || runtime.GOARCH == "s390x") &&
168 fnType.NumOut() == 1 && fnType.Out(0).Kind() == reflect.Struct &&
169 fnType.Out(0).Size() > maxRegAllocStructSize {
170 intsN = 1
171 }
172 // stackSlot points to the index into frame (or stackFrame) of the current stack element.
173 // When stackFrame is nil, stack begins after float and integer registers in frame.
174 // When stackFrame is not nil (ppc64le), stackSlot indexes into stackFrame starting at 0.
175 stackSlot := numOfIntegerRegisters() + numOfFloatRegisters()
176 if stackFrame != nil {
177 // ppc64le: stackArgs is a separate pointer, indices start at 0
178 stackSlot = 0
179 }
180 // stackByteOffset tracks the byte offset within the stack area for Darwin ARM64
181 // tight packing. On Darwin ARM64, C passes small types packed on the stack.
182 stackByteOffset := uintptr(0)
183 for i := range args {
184 // slots is the number of pointer-sized slots the argument takes
185 var slots int
186 inType := fnType.In(i)
187 switch inType.Kind() {
188 case reflect.Float32, reflect.Float64:
189 slots = int((fnType.In(i).Size() + ptrSize - 1) / ptrSize)
190 if floatsN+slots > numOfFloatRegisters() {
191 if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
192 // Darwin ARM64: read from packed stack with proper alignment
193 args[i] = callbackArgFromStack(a.args, stackSlot, &stackByteOffset, inType)
194 } else if stackFrame != nil {
195 // ppc64le/s390x: stack args are in separate stackFrame
196 if runtime.GOARCH == "s390x" {
197 // s390x big-endian: sub-8-byte values are right-justified
198 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&stackFrame[stackSlot]), inType)
199 } else {
200 args[i] = reflect.NewAt(inType, unsafe.Pointer(&stackFrame[stackSlot])).Elem()
201 }
202 stackSlot += slots
203 } else {
204 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
205 stackSlot += slots
206 }
207 } else {
208 if runtime.GOARCH == "s390x" {
209 // s390x big-endian: float32 is right-justified in 8-byte FPR slot
210 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&frame[floatsN]), inType)
211 } else {
212 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[floatsN])).Elem()
213 }
214 }
215 floatsN += slots
216 case reflect.String:
217 slots = 1
218 var ptr uintptr
219 if intsN+slots > numOfIntegerRegisters() {
220 if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
221 ptr = uintptr(callbackArgFromStack(a.args, stackSlot, &stackByteOffset, reflect.TypeOf(uintptr(0))).Uint())
222 } else if stackFrame != nil {
223 if runtime.GOARCH == "s390x" {
224 ptr = uintptr(callbackArgFromSlotBigEndian(unsafe.Pointer(&stackFrame[stackSlot]), reflect.TypeOf(uintptr(0))).Uint())
225 } else {
226 ptr = *(*uintptr)(unsafe.Pointer(&stackFrame[stackSlot]))
227 }
228 stackSlot += slots
229 } else {
230 ptr = frame[stackSlot]
231 stackSlot += slots
232 }
233 } else {
234 pos := intsN + numOfFloatRegisters()
235 ptr = frame[pos]
236 }
237 intsN += slots
238 args[i] = reflect.ValueOf(strings.GoString(ptr))
239 continue
240 case reflect.Struct:
241 if i == 0 && inType.AssignableTo(reflect.TypeOf(CDecl{})) {
242 args[i] = reflect.Zero(inType)
243 continue
244 }
245 if inType.Size() == 0 {
246 args[i] = reflect.New(inType).Elem()
247 continue
248 }
249 args[i] = getCallbackStruct(inType, a.args, &floatsN, &intsN, &stackSlot, &stackByteOffset)
250 continue
251 default:
252 slots = int((inType.Size() + ptrSize - 1) / ptrSize)
253 if intsN+slots > numOfIntegerRegisters() {
254 if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
255 // Darwin ARM64: read from packed stack with proper alignment
256 args[i] = callbackArgFromStack(a.args, stackSlot, &stackByteOffset, inType)
257 } else if stackFrame != nil {
258 // ppc64le/s390x: stack args are in separate stackFrame
259 if runtime.GOARCH == "s390x" {
260 // s390x big-endian: sub-8-byte values are right-justified
261 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&stackFrame[stackSlot]), inType)
262 } else {
263 args[i] = reflect.NewAt(inType, unsafe.Pointer(&stackFrame[stackSlot])).Elem()
264 }
265 stackSlot += slots
266 } else {
267 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
268 stackSlot += slots
269 }
270 } else {
271 // the integers begin after the floats in frame
272 pos := intsN + numOfFloatRegisters()
273 if runtime.GOARCH == "s390x" {
274 // s390x big-endian: sub-8-byte values are right-justified in GPR slot
275 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&frame[pos]), inType)
276 } else {
277 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[pos])).Elem()
278 }
279 }
280 intsN += slots
281 }
282 }
283 ret := fn.Call(args)
284 if len(ret) > 0 {
285 switch k := ret[0].Kind(); k {
286 case reflect.Uint, reflect.Uint64, reflect.Uint32, reflect.Uint16, reflect.Uint8, reflect.Uintptr:
287 a.result[0] = uintptr(ret[0].Uint())
288 case reflect.Int, reflect.Int64, reflect.Int32, reflect.Int16, reflect.Int8:
289 a.result[0] = uintptr(ret[0].Int())
290 case reflect.Bool:
291 if ret[0].Bool() {
292 a.result[0] = 1
293 } else {
294 a.result[0] = 0
295 }
296 case reflect.Pointer:
297 a.result[0] = ret[0].Pointer()
298 case reflect.UnsafePointer:
299 a.result[0] = ret[0].Pointer()
300 case reflect.Struct:
301 setStruct(a, ret[0])
302 default:
303 panic("purego: unsupported kind: " + k.String())
304 }
305 }
306}
307
308// callbackArgFromStack reads an argument from the tightly-packed stack area on Darwin ARM64.
309// The C ABI on Darwin ARM64 packs small types on the stack without padding to 8 bytes.
310// This function handles proper alignment and advances stackByteOffset accordingly.
311func callbackArgFromStack(argsBase unsafe.Pointer, stackSlot int, stackByteOffset *uintptr, inType reflect.Type) reflect.Value {
312 // Calculate base address of stack area (after float and int registers)
313 stackBase := unsafe.Add(argsBase, stackSlot*int(ptrSize))
314
315 // Get type's natural alignment
316 align := uintptr(inType.Align())
317 size := inType.Size()
318
319 // Align the offset
320 if *stackByteOffset%align != 0 {
321 *stackByteOffset = (*stackByteOffset + align - 1) &^ (align - 1)
322 }
323
324 // Read value at aligned offset
325 ptr := unsafe.Add(stackBase, *stackByteOffset)
326 *stackByteOffset += size
327
328 return reflect.NewAt(inType, ptr).Elem()
329}
330
331// callbackArgFromSlotBigEndian reads an argument from an 8-byte slot on big-endian architectures.
332// On s390x:
333// - Integer types are right-justified in GPRs: sub-8-byte values are at offset (8 - size)
334// - Float32 in FPRs is left-justified: stored in upper 32 bits, so at offset 0
335// - Float64 occupies the full 8-byte slot
336func callbackArgFromSlotBigEndian(slotPtr unsafe.Pointer, inType reflect.Type) reflect.Value {
337 size := inType.Size()
338 if size >= 8 {
339 // 8-byte values occupy the entire slot
340 return reflect.NewAt(inType, slotPtr).Elem()
341 }
342 // Float32 is left-justified in FPRs (upper 32 bits), so offset is 0
343 if inType.Kind() == reflect.Float32 {
344 return reflect.NewAt(inType, slotPtr).Elem()
345 }
346 // Integer types are right-justified: offset = 8 - size
347 offset := 8 - size
348 ptr := unsafe.Add(slotPtr, offset)
349 return reflect.NewAt(inType, ptr).Elem()
350}
351
352// callbackasmAddr returns address of runtime.callbackasm
353// function adjusted by i.
354// On x86 and amd64, runtime.callbackasm is a series of CALL instructions,
355// and we want callback to arrive at
356// correspondent call instruction instead of start of
357// runtime.callbackasm.
358// On ARM, runtime.callbackasm is a series of mov and branch instructions.
359// R12 is loaded with the callback index. Each entry is two instructions,
360// hence 8 bytes.
361func callbackasmAddr(i int) uintptr {
362 var entrySize int
363 switch runtime.GOARCH {
364 default:
365 panic("purego: unsupported architecture")
366 case "amd64":
367 // On amd64, each callback entry is just a CALL instruction (5 bytes)
368 entrySize = 5
369 case "386":
370 // On 386, each callback entry is MOVL $imm, CX (5 bytes) + JMP (5 bytes)
371 entrySize = 10
372 case "arm", "arm64", "loong64", "ppc64le", "riscv64":
373 // On ARM, ARM64, Loong64, PPC64LE and RISCV64, each entry is a MOV instruction
374 // followed by a branch instruction
375 entrySize = 8
376 case "s390x":
377 // On S390X, each entry is LGHI (4 bytes) + JG (6 bytes)
378 entrySize = 10
379 }
380 return callbackasmABI0 + uintptr(i*entrySize)
381}