1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: 2022 The Ebitengine Authors
3
4// TODO: remove s390x cgo dependency once golang/go#77449 is resolved
5//go:build darwin || freebsd || (linux && (386 || amd64 || arm || arm64 || loong64 || ppc64le || riscv64 || (cgo && s390x))) || netbsd
6
7package purego
8
9import (
10 "reflect"
11 "runtime"
12 "sync"
13 "unsafe"
14)
15
16var syscall15XABI0 uintptr
17
18func syscall_syscall15X(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15 uintptr) (r1, r2, err uintptr) {
19 args := &syscall15Args{
20 fn: fn,
21 a1: a1, a2: a2, a3: a3, a4: a4, a5: a5, a6: a6, a7: a7, a8: a8,
22 a9: a9, a10: a10, a11: a11, a12: a12, a13: a13, a14: a14, a15: a15,
23 f1: a1, f2: a2, f3: a3, f4: a4, f5: a5, f6: a6, f7: a7, f8: a8,
24 }
25
26 runtime_cgocall(syscall15XABI0, unsafe.Pointer(args))
27 return args.a1, args.a2, args.a3
28}
29
30// NewCallback converts a Go function to a function pointer conforming to the C calling convention.
31// This is useful when interoperating with C code requiring callbacks. The argument is expected to be a
32// function with zero or one uintptr-sized result. The function must not have arguments with size larger than the size
33// of uintptr. Only a limited number of callbacks may be created in a single Go process, and any memory allocated
34// for these callbacks is never released. At least 2000 callbacks can always be created. Although this function
35// provides similar functionality to windows.NewCallback it is distinct.
36func NewCallback(fn any) uintptr {
37 ty := reflect.TypeOf(fn)
38 for i := 0; i < ty.NumIn(); i++ {
39 in := ty.In(i)
40 if !in.AssignableTo(reflect.TypeOf(CDecl{})) {
41 continue
42 }
43 if i != 0 {
44 panic("purego: CDecl must be the first argument")
45 }
46 }
47 return compileCallback(fn)
48}
49
50// maxCb is the maximum number of callbacks
51// only increase this if you have added more to the callbackasm function
52const maxCB = 2000
53
54var cbs struct {
55 lock sync.Mutex
56 numFn int // the number of functions currently in cbs.funcs
57 funcs [maxCB]reflect.Value // the saved callbacks
58}
59
60func compileCallback(fn any) uintptr {
61 val := reflect.ValueOf(fn)
62 if val.Kind() != reflect.Func {
63 panic("purego: the type must be a function but was not")
64 }
65 if val.IsNil() {
66 panic("purego: function must not be nil")
67 }
68 ty := val.Type()
69 for i := 0; i < ty.NumIn(); i++ {
70 in := ty.In(i)
71 switch in.Kind() {
72 case reflect.Struct:
73 if i == 0 && in.AssignableTo(reflect.TypeOf(CDecl{})) {
74 continue
75 }
76 fallthrough
77 case reflect.Interface, reflect.Func, reflect.Slice,
78 reflect.Chan, reflect.Complex64, reflect.Complex128,
79 reflect.String, reflect.Map, reflect.Invalid:
80 panic("purego: unsupported argument type: " + in.Kind().String())
81 }
82 }
83output:
84 switch {
85 case ty.NumOut() == 1:
86 switch ty.Out(0).Kind() {
87 case reflect.Pointer, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
88 reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
89 reflect.Bool, reflect.UnsafePointer:
90 break output
91 }
92 panic("purego: unsupported return type: " + ty.String())
93 case ty.NumOut() > 1:
94 panic("purego: callbacks can only have one return")
95 }
96 cbs.lock.Lock()
97 defer cbs.lock.Unlock()
98 if cbs.numFn >= maxCB {
99 panic("purego: the maximum number of callbacks has been reached")
100 }
101 cbs.funcs[cbs.numFn] = val
102 cbs.numFn++
103 return callbackasmAddr(cbs.numFn - 1)
104}
105
106const ptrSize = unsafe.Sizeof((*int)(nil))
107
108const callbackMaxFrame = 64 * ptrSize
109
110// callbackasm is implemented in zcallback_GOOS_GOARCH.s
111//
112//go:linkname __callbackasm callbackasm
113var __callbackasm byte
114var callbackasmABI0 = uintptr(unsafe.Pointer(&__callbackasm))
115
116// callbackWrap_call allows the calling of the ABIInternal wrapper
117// which is required for runtime.cgocallback without the
118// <ABIInternal> tag which is only allowed in the runtime.
119// This closure is used inside sys_darwin_GOARCH.s
120var callbackWrap_call = callbackWrap
121
122// callbackWrap is called by assembly code which determines which Go function to call.
123// This function takes the arguments and passes them to the Go function and returns the result.
124func callbackWrap(a *callbackArgs) {
125 cbs.lock.Lock()
126 fn := cbs.funcs[a.index]
127 cbs.lock.Unlock()
128 fnType := fn.Type()
129 args := make([]reflect.Value, fnType.NumIn())
130 frame := (*[callbackMaxFrame]uintptr)(a.args)
131 // stackFrame points to stack-passed arguments. On most architectures this is
132 // contiguous with frame (after register args), but on ppc64le it's separate.
133 var stackFrame *[callbackMaxFrame]uintptr
134 if sf := a.stackFrame(); sf != nil {
135 // Only ppc64le uses separate stackArgs pointer due to NOSPLIT constraints
136 stackFrame = (*[callbackMaxFrame]uintptr)(sf)
137 }
138 // floatsN and intsN track the number of register slots used, not argument count.
139 // This distinction matters on ARM32 where float64 uses 2 slots (32-bit registers).
140 var floatsN int
141 var intsN int
142 // stackSlot points to the index into frame (or stackFrame) of the current stack element.
143 // When stackFrame is nil, stack begins after float and integer registers in frame.
144 // When stackFrame is not nil (ppc64le), stackSlot indexes into stackFrame starting at 0.
145 stackSlot := numOfIntegerRegisters() + numOfFloatRegisters()
146 if stackFrame != nil {
147 // ppc64le: stackArgs is a separate pointer, indices start at 0
148 stackSlot = 0
149 }
150 // stackByteOffset tracks the byte offset within the stack area for Darwin ARM64
151 // tight packing. On Darwin ARM64, C passes small types packed on the stack.
152 stackByteOffset := uintptr(0)
153 for i := range args {
154 // slots is the number of pointer-sized slots the argument takes
155 var slots int
156 inType := fnType.In(i)
157 switch inType.Kind() {
158 case reflect.Float32, reflect.Float64:
159 slots = int((fnType.In(i).Size() + ptrSize - 1) / ptrSize)
160 if floatsN+slots > numOfFloatRegisters() {
161 if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
162 // Darwin ARM64: read from packed stack with proper alignment
163 args[i] = callbackArgFromStack(a.args, stackSlot, &stackByteOffset, inType)
164 } else if stackFrame != nil {
165 // ppc64le/s390x: stack args are in separate stackFrame
166 if runtime.GOARCH == "s390x" {
167 // s390x big-endian: sub-8-byte values are right-justified
168 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&stackFrame[stackSlot]), inType)
169 } else {
170 args[i] = reflect.NewAt(inType, unsafe.Pointer(&stackFrame[stackSlot])).Elem()
171 }
172 stackSlot += slots
173 } else {
174 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
175 stackSlot += slots
176 }
177 } else {
178 if runtime.GOARCH == "s390x" {
179 // s390x big-endian: float32 is right-justified in 8-byte FPR slot
180 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&frame[floatsN]), inType)
181 } else {
182 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[floatsN])).Elem()
183 }
184 }
185 floatsN += slots
186 case reflect.Struct:
187 // This is the CDecl field
188 args[i] = reflect.Zero(inType)
189 default:
190 slots = int((inType.Size() + ptrSize - 1) / ptrSize)
191 if intsN+slots > numOfIntegerRegisters() {
192 if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
193 // Darwin ARM64: read from packed stack with proper alignment
194 args[i] = callbackArgFromStack(a.args, stackSlot, &stackByteOffset, inType)
195 } else if stackFrame != nil {
196 // ppc64le/s390x: stack args are in separate stackFrame
197 if runtime.GOARCH == "s390x" {
198 // s390x big-endian: sub-8-byte values are right-justified
199 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&stackFrame[stackSlot]), inType)
200 } else {
201 args[i] = reflect.NewAt(inType, unsafe.Pointer(&stackFrame[stackSlot])).Elem()
202 }
203 stackSlot += slots
204 } else {
205 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[stackSlot])).Elem()
206 stackSlot += slots
207 }
208 } else {
209 // the integers begin after the floats in frame
210 pos := intsN + numOfFloatRegisters()
211 if runtime.GOARCH == "s390x" {
212 // s390x big-endian: sub-8-byte values are right-justified in GPR slot
213 args[i] = callbackArgFromSlotBigEndian(unsafe.Pointer(&frame[pos]), inType)
214 } else {
215 args[i] = reflect.NewAt(inType, unsafe.Pointer(&frame[pos])).Elem()
216 }
217 }
218 intsN += slots
219 }
220 }
221 ret := fn.Call(args)
222 if len(ret) > 0 {
223 switch k := ret[0].Kind(); k {
224 case reflect.Uint, reflect.Uint64, reflect.Uint32, reflect.Uint16, reflect.Uint8, reflect.Uintptr:
225 a.result = uintptr(ret[0].Uint())
226 case reflect.Int, reflect.Int64, reflect.Int32, reflect.Int16, reflect.Int8:
227 a.result = uintptr(ret[0].Int())
228 case reflect.Bool:
229 if ret[0].Bool() {
230 a.result = 1
231 } else {
232 a.result = 0
233 }
234 case reflect.Pointer:
235 a.result = ret[0].Pointer()
236 case reflect.UnsafePointer:
237 a.result = ret[0].Pointer()
238 default:
239 panic("purego: unsupported kind: " + k.String())
240 }
241 }
242}
243
244// callbackArgFromStack reads an argument from the tightly-packed stack area on Darwin ARM64.
245// The C ABI on Darwin ARM64 packs small types on the stack without padding to 8 bytes.
246// This function handles proper alignment and advances stackByteOffset accordingly.
247func callbackArgFromStack(argsBase unsafe.Pointer, stackSlot int, stackByteOffset *uintptr, inType reflect.Type) reflect.Value {
248 // Calculate base address of stack area (after float and int registers)
249 stackBase := unsafe.Add(argsBase, stackSlot*int(ptrSize))
250
251 // Get type's natural alignment
252 align := uintptr(inType.Align())
253 size := inType.Size()
254
255 // Align the offset
256 if *stackByteOffset%align != 0 {
257 *stackByteOffset = (*stackByteOffset + align - 1) &^ (align - 1)
258 }
259
260 // Read value at aligned offset
261 ptr := unsafe.Add(stackBase, *stackByteOffset)
262 *stackByteOffset += size
263
264 return reflect.NewAt(inType, ptr).Elem()
265}
266
267// callbackArgFromSlotBigEndian reads an argument from an 8-byte slot on big-endian architectures.
268// On s390x:
269// - Integer types are right-justified in GPRs: sub-8-byte values are at offset (8 - size)
270// - Float32 in FPRs is left-justified: stored in upper 32 bits, so at offset 0
271// - Float64 occupies the full 8-byte slot
272func callbackArgFromSlotBigEndian(slotPtr unsafe.Pointer, inType reflect.Type) reflect.Value {
273 size := inType.Size()
274 if size >= 8 {
275 // 8-byte values occupy the entire slot
276 return reflect.NewAt(inType, slotPtr).Elem()
277 }
278 // Float32 is left-justified in FPRs (upper 32 bits), so offset is 0
279 if inType.Kind() == reflect.Float32 {
280 return reflect.NewAt(inType, slotPtr).Elem()
281 }
282 // Integer types are right-justified: offset = 8 - size
283 offset := 8 - size
284 ptr := unsafe.Add(slotPtr, offset)
285 return reflect.NewAt(inType, ptr).Elem()
286}
287
288// callbackasmAddr returns address of runtime.callbackasm
289// function adjusted by i.
290// On x86 and amd64, runtime.callbackasm is a series of CALL instructions,
291// and we want callback to arrive at
292// correspondent call instruction instead of start of
293// runtime.callbackasm.
294// On ARM, runtime.callbackasm is a series of mov and branch instructions.
295// R12 is loaded with the callback index. Each entry is two instructions,
296// hence 8 bytes.
297func callbackasmAddr(i int) uintptr {
298 var entrySize int
299 switch runtime.GOARCH {
300 default:
301 panic("purego: unsupported architecture")
302 case "amd64":
303 // On amd64, each callback entry is just a CALL instruction (5 bytes)
304 entrySize = 5
305 case "386":
306 // On 386, each callback entry is MOVL $imm, CX (5 bytes) + JMP (5 bytes)
307 entrySize = 10
308 case "arm", "arm64", "loong64", "ppc64le", "riscv64":
309 // On ARM, ARM64, Loong64, PPC64LE and RISCV64, each entry is a MOV instruction
310 // followed by a branch instruction
311 entrySize = 8
312 case "s390x":
313 // On S390X, each entry is LGHI (4 bytes) + JG (6 bytes)
314 entrySize = 10
315 }
316 return callbackasmABI0 + uintptr(i*entrySize)
317}