1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: 2022 The Ebitengine Authors
3
4//go:build darwin || freebsd || linux || netbsd || windows
5
6package purego
7
8import (
9 "fmt"
10 "math"
11 "reflect"
12 "runtime"
13 "sync"
14 "unsafe"
15
16 "github.com/ebitengine/purego/internal/strings"
17)
18
19const (
20 align8ByteMask = 7 // Mask for 8-byte alignment: (val + 7) &^ 7
21 align8ByteSize = 8 // 8-byte alignment boundary
22)
23
24var thePool = sync.Pool{New: func() any {
25 return new(syscallArgs)
26}}
27
28// RegisterLibFunc is a wrapper around RegisterFunc that uses the C function returned from Dlsym(handle, name).
29// It panics if it can't find the name symbol.
30func RegisterLibFunc(fptr any, handle uintptr, name string) {
31 sym, err := loadSymbol(handle, name)
32 if err != nil {
33 panic(err)
34 }
35 RegisterFunc(fptr, sym)
36}
37
38// RegisterFunc takes a pointer to a Go function representing the calling convention of the C function.
39// fptr will be set to a function that when called will call the C function given by cfn with the
40// parameters passed in the correct registers and stack.
41//
42// A panic is produced if the type is not a function pointer or if the function returns more than 1 value.
43//
44// These conversions describe how a Go type in the fptr will be used to call
45// the C function. It is important to note that there is no way to verify that fptr
46// matches the C function. This also holds true for struct types where the padding
47// needs to be ensured to match that of C; RegisterFunc does not verify this.
48//
49// # Type Conversions (Go <=> C)
50//
51// string <=> char*
52// bool <=> _Bool
53// uintptr <=> uintptr_t
54// uint <=> uint32_t or uint64_t
55// uint8 <=> uint8_t
56// uint16 <=> uint16_t
57// uint32 <=> uint32_t
58// uint64 <=> uint64_t
59// int <=> int32_t or int64_t
60// int8 <=> int8_t
61// int16 <=> int16_t
62// int32 <=> int32_t
63// int64 <=> int64_t
64// float32 <=> float
65// float64 <=> double
66// struct <=> struct (android, darwin, ios, linux, and windows on amd64/arm64)
67// func <=> C function
68// unsafe.Pointer, *T <=> void*
69// []T => void*
70//
71// There is a special case when the last argument of fptr is a variadic interface (or []interface}
72// it will be expanded into a call to the C function as if it had the arguments in that slice.
73// This means that using arg ...any is like a cast to the function with the arguments inside arg.
74// This is not the same as C variadic.
75//
76// # Memory
77//
78// In general it is not possible for purego to guarantee the lifetimes of objects returned or received from
79// calling functions using RegisterFunc. For arguments to a C function it is important that the C function doesn't
80// hold onto a reference to Go memory. This is the same as the [Cgo rules].
81//
82// However, there are some special cases. When passing a string as an argument if the string does not end in a null
83// terminated byte (\x00) then the string will be copied into memory maintained by purego. The memory is only valid for
84// that specific call. Therefore, if the C code keeps a reference to that string it may become invalid at some
85// undefined time. However, if the string does already contain a null-terminated byte then no copy is done.
86// It is then the responsibility of the caller to ensure the string stays alive as long as it's needed in C memory.
87// This can be done using runtime.KeepAlive or allocating the string in C memory using malloc. When a C function
88// returns a null-terminated pointer to char a Go string can be used. Purego will allocate a new string in Go memory
89// and copy the data over. This string will be garbage collected whenever Go decides it's no longer referenced.
90// This C created string will not be freed by purego. If the pointer to char is not null-terminated or must continue
91// to point to C memory (because it's a buffer for example) then use a pointer to byte and then convert that to a slice
92// using unsafe.Slice. Doing this means that it becomes the responsibility of the caller to care about the lifetime
93// of the pointer
94//
95// # Structs
96//
97// Purego can handle the most common structs that have fields of builtin types like int8, uint16, float32, etc. However,
98// it does not support aligning fields properly. It is therefore the responsibility of the caller to ensure
99// that all padding is added to the Go struct to match the C one. See `BoolStructFn` in struct_test.go for an example.
100//
101// On Apple ARM64 platforms (macOS and iOS), purego handles proper alignment of struct arguments
102// when passing them on the stack, following the C ABI's byte-level packing rules.
103//
104// On Windows, struct arguments and returns are supported on amd64 and arm64 when calling C functions.
105// Passing or returning structs in callbacks created with [NewCallback] is not supported on Windows.
106//
107// # Example
108//
109// All functions below call this C function:
110//
111// char *foo(char *str);
112//
113// // Let purego convert types
114// var foo func(s string) string
115// goString := foo("copied")
116// // Go will garbage collect this string
117//
118// // Manually, handle allocations
119// var foo2 func(b string) *byte
120// mustFree := foo2("not copied\x00")
121// defer free(mustFree)
122//
123// [Cgo rules]: https://pkg.go.dev/cmd/cgo#hdr-Go_references_to_C
124func RegisterFunc(fptr any, cfn uintptr) {
125 const is32bit = unsafe.Sizeof(uintptr(0)) == 4
126 fn := reflect.ValueOf(fptr).Elem()
127 ty := fn.Type()
128 if ty.Kind() != reflect.Func {
129 panic("purego: fptr must be a function pointer")
130 }
131 if ty.NumOut() > 1 {
132 panic("purego: function can only return zero or one values")
133 }
134 if cfn == 0 {
135 panic("purego: cfn is nil")
136 }
137 if ty.NumOut() == 1 && (ty.Out(0).Kind() == reflect.Float32 || ty.Out(0).Kind() == reflect.Float64) &&
138 runtime.GOARCH != "arm" && runtime.GOARCH != "arm64" && runtime.GOARCH != "386" && runtime.GOARCH != "amd64" && runtime.GOARCH != "loong64" && runtime.GOARCH != "ppc64le" && runtime.GOARCH != "riscv64" && runtime.GOARCH != "s390x" {
139 panic("purego: float returns are not supported")
140 }
141 {
142 // this code checks how many registers and stack this function will use
143 // to avoid crashing with too many arguments
144 var ints int
145 var floats int
146 floatArgRegs := numOfFloatRegisters()
147 var stack int
148 for i := 0; i < ty.NumIn(); i++ {
149 arg := ty.In(i)
150 switch arg.Kind() {
151 case reflect.Func:
152 // This only does preliminary testing to ensure the CDecl argument
153 // is the first argument. Full testing is done when the callback is actually
154 // created in NewCallback.
155 for j := 0; j < arg.NumIn(); j++ {
156 in := arg.In(j)
157 if !in.AssignableTo(reflect.TypeFor[CDecl]()) {
158 continue
159 }
160 if j != 0 {
161 panic("purego: CDecl must be the first argument")
162 }
163 }
164 case reflect.String, reflect.Uintptr, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64,
165 reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Pointer, reflect.UnsafePointer,
166 reflect.Slice, reflect.Bool:
167 if ints < numOfIntegerRegisters() {
168 ints++
169 } else {
170 stack++
171 }
172 case reflect.Float32, reflect.Float64:
173 if floats < floatArgRegs {
174 floats++
175 } else {
176 stack++
177 }
178 case reflect.Struct:
179 ensureStructSupported()
180 if arg.Size() == 0 && runtime.GOOS != "windows" {
181 // On Windows an empty struct still consumes one argument slot.
182 continue
183 }
184 addInt := func(u uintptr) {
185 ints++
186 }
187 addFloat := func(u uintptr) {
188 floats++
189 }
190 addStack := func(u uintptr) {
191 stack++
192 }
193 _ = addStruct(reflect.New(arg).Elem(), &ints, &floats, &stack, addInt, addFloat, addStack, nil)
194 default:
195 panic("purego: unsupported kind " + arg.Kind().String())
196 }
197 }
198 if ty.NumOut() == 1 && ty.Out(0).Kind() == reflect.Struct {
199 ensureStructSupported()
200 outType := ty.Out(0)
201 checkStructFieldsSupported(outType)
202 if runtime.GOARCH == "amd64" && amd64StructReturnInMemory(outType.Size()) {
203 // on amd64 a struct returned in memory is allocated by the caller
204 // and its pointer is passed as a hidden first argument.
205 ints++
206 }
207 }
208
209 argsLimit := maxArgs
210 sizeOfStack := argsLimit - numOfIntegerRegisters()
211 if runtime.GOOS == "windows" {
212 if ints+floats+stack > argsLimit {
213 panic("purego: too many stack arguments")
214 }
215 } else if isDarwin && runtime.GOARCH == "arm64" {
216 // On Darwin ARM64, use byte-based validation since arguments pack efficiently.
217 // See https://developer.apple.com/documentation/xcode/writing-arm64-code-for-apple-platforms
218 stackBytes := estimateStackBytes(ty)
219 maxStackBytes := sizeOfStack * 8
220 if stackBytes > maxStackBytes {
221 panic("purego: too many stack arguments")
222 }
223 } else {
224 if stack > sizeOfStack {
225 panic("purego: too many stack arguments")
226 }
227 }
228 }
229
230 v := reflect.MakeFunc(ty, func(args []reflect.Value) (results []reflect.Value) {
231 var sysargs [maxArgs]uintptr
232 // Use maxArgs instead of numOfFloatRegisters() to keep this code path allocation-free,
233 // since numOfFloatRegisters() is a function call, not a constant.
234 // maxArgs is always greater than or equal to numOfFloatRegisters() so this is safe.
235 var floats [maxArgs]uintptr
236 floatArgRegs := numOfFloatRegisters()
237 var numInts int
238 var numFloats int
239 var numStack int
240 var addStack, addInt, addFloat func(x uintptr)
241 if runtime.GOARCH == "arm64" || runtime.GOOS != "windows" {
242 // Windows arm64 uses the same calling convention as macOS and Linux
243 addStack = func(x uintptr) {
244 sysargs[numOfIntegerRegisters()+numStack] = x
245 numStack++
246 }
247 addInt = func(x uintptr) {
248 if numInts >= numOfIntegerRegisters() {
249 addStack(x)
250 } else {
251 sysargs[numInts] = x
252 numInts++
253 }
254 }
255 addFloat = func(x uintptr) {
256 if numFloats < floatArgRegs {
257 floats[numFloats] = x
258 numFloats++
259 } else {
260 addStack(x)
261 }
262 }
263 } else {
264 // On Windows amd64 the arguments are passed in the numbered registered.
265 // So the first int is in the first integer register and the first float
266 // is in the second floating register if there is already a first int.
267 // This is in contrast to how macOS and Linux pass arguments which
268 // tries to use as many registers as possible in the calling convention.
269 addStack = func(x uintptr) {
270 if numStack >= maxArgs {
271 panic("purego: too many stack arguments")
272 }
273 sysargs[numStack] = x
274 numStack++
275 }
276 addInt = addStack
277 addFloat = addStack
278 }
279
280 var keepAlive []any
281 defer func() {
282 runtime.KeepAlive(keepAlive)
283 runtime.KeepAlive(args)
284 }()
285
286 var arm64_r8 uintptr
287 if ty.NumOut() == 1 && ty.Out(0).Kind() == reflect.Struct {
288 outType := ty.Out(0)
289 amd64InMemory := runtime.GOARCH == "amd64" && amd64StructReturnInMemory(outType.Size())
290 otherInMemory := (runtime.GOARCH == "loong64" || runtime.GOARCH == "ppc64le" || runtime.GOARCH == "riscv64" || runtime.GOARCH == "s390x") && outType.Size() > maxRegAllocStructSize
291 if amd64InMemory || otherInMemory {
292 val := reflect.New(outType)
293 keepAlive = append(keepAlive, val)
294 addInt(val.Pointer())
295 } else if runtime.GOARCH == "arm64" && outType.Size() > maxRegAllocStructSize {
296 isAllFloats, numFields := isAllSameFloat(outType)
297 if !isAllFloats || numFields > 4 {
298 val := reflect.New(outType)
299 keepAlive = append(keepAlive, val)
300 arm64_r8 = val.Pointer()
301 }
302 }
303 }
304 for i, v := range args {
305 if variadic, ok := reflect.TypeAssert[[]any](args[i]); ok {
306 if i != len(args)-1 {
307 panic("purego: can only expand last parameter")
308 }
309 for _, x := range variadic {
310 keepAlive = addValue(reflect.ValueOf(x), keepAlive, addInt, addFloat, addStack, &numInts, &numFloats, &numStack)
311 }
312 continue
313 }
314 // Check if we need to start Darwin ARM64 C-style stack packing
315 if runtime.GOARCH == "arm64" && isDarwin && shouldBundleStackArgs(v, numInts, numFloats) {
316 // Collect and separate remaining args into register vs stack
317 stackArgs, newKeepAlive := collectStackArgs(args, i, numInts, numFloats,
318 keepAlive, addInt, addFloat, addStack, &numInts, &numFloats, &numStack)
319 keepAlive = newKeepAlive
320
321 // Bundle stack arguments with C-style packing
322 bundleStackArgs(stackArgs, addStack)
323 break
324 }
325 keepAlive = addValue(v, keepAlive, addInt, addFloat, addStack, &numInts, &numFloats, &numStack)
326 }
327
328 var syscall *syscallArgs
329 if runtime.GOOS == "windows" && runtime.GOARCH != "arm64" {
330 // Windows amd64, 386, and arm use syscall.SyscallN.
331 syscall = thePool.Get().(*syscallArgs)
332 syscall.a1, syscall.a2, _ = syscall_syscallN(cfn, sysargs[:numStack]...)
333 syscall.f1 = syscall.a2 // on amd64 a2 stores the float return. On 32bit platforms floats aren't support
334 } else {
335 syscall = syscall_SyscallN(cfn, sysargs[:], floats[:], arm64_r8)
336 }
337 defer thePool.Put(syscall)
338 if ty.NumOut() == 0 {
339 return nil
340 }
341 outType := ty.Out(0)
342 v := reflect.New(outType).Elem()
343 switch outType.Kind() {
344 case reflect.Uintptr, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
345 v.SetUint(uint64(syscall.a1))
346 case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
347 v.SetInt(int64(syscall.a1))
348 case reflect.Bool:
349 v.SetBool(byte(syscall.a1) != 0)
350 case reflect.UnsafePointer:
351 // We take the address and then dereference it to trick go vet from creating a possible miss-use of unsafe.Pointer
352 v.SetPointer(*(*unsafe.Pointer)(unsafe.Pointer(&syscall.a1)))
353 case reflect.Pointer:
354 // Copy syscall.a1 into a local variable to prevent v
355 // from holding a pointer to the pooled syscallArgs field.
356 a1 := syscall.a1
357 v = reflect.NewAt(outType, unsafe.Pointer(&a1)).Elem()
358 case reflect.Func:
359 // wrap this C function in a nicely typed Go function
360 v = reflect.New(outType)
361 RegisterFunc(v.Interface(), syscall.a1)
362 case reflect.String:
363 v.SetString(strings.GoString(syscall.a1))
364 case reflect.Float32:
365 // NOTE: syscall.r2 is only the floating return value on 64bit platforms.
366 // On 32bit platforms syscall.r2 is the upper part of a 64bit return.
367 // On 386, x87 FPU returns floats as float64 in ST(0), so we read as float64 and convert.
368 // On PPC64LE, C ABI converts float32 to double in FPR, so we read as float64.
369 // On S390X (big-endian), float32 is in upper 32 bits of the 64-bit FP register.
370 switch runtime.GOARCH {
371 case "386":
372 v.SetFloat(math.Float64frombits(uint64(syscall.f1) | (uint64(syscall.f2) << 32)))
373 case "ppc64le":
374 v.SetFloat(math.Float64frombits(uint64(syscall.f1)))
375 case "s390x":
376 // S390X is big-endian: float32 in upper 32 bits of 64-bit register
377 v.SetFloat(float64(math.Float32frombits(uint32(syscall.f1 >> 32))))
378 default:
379 v.SetFloat(float64(math.Float32frombits(uint32(syscall.f1))))
380 }
381 case reflect.Float64:
382 // NOTE: syscall.r2 is only the floating return value on 64bit platforms.
383 // On 32bit platforms syscall.r2 is the upper part of a 64bit return.
384 if is32bit {
385 v.SetFloat(math.Float64frombits(uint64(syscall.f1) | (uint64(syscall.f2) << 32)))
386 } else {
387 v.SetFloat(math.Float64frombits(uint64(syscall.f1)))
388 }
389 case reflect.Struct:
390 v = getStruct(outType, *syscall)
391 default:
392 panic("purego: unsupported return kind: " + outType.Kind().String())
393 }
394 if len(args) > 0 {
395 // reuse args slice instead of allocating one when possible
396 args[0] = v
397 return args[:1]
398 } else {
399 return []reflect.Value{v}
400 }
401 })
402 fn.Set(v)
403}
404
405func addValue(v reflect.Value, keepAlive []any, addInt func(x uintptr), addFloat func(x uintptr), addStack func(x uintptr), numInts *int, numFloats *int, numStack *int) []any {
406 const is32bit = unsafe.Sizeof(uintptr(0)) == 4
407 switch v.Kind() {
408 case reflect.String:
409 ptr := strings.CString(v.String())
410 keepAlive = append(keepAlive, ptr)
411 addInt(uintptr(unsafe.Pointer(ptr)))
412 case reflect.Uintptr, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
413 addInt(uintptr(v.Uint()))
414 case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
415 addInt(uintptr(v.Int()))
416 case reflect.Pointer, reflect.UnsafePointer, reflect.Slice:
417 // There is no need to keepAlive this pointer separately because it is kept alive in the args variable
418 addInt(v.Pointer())
419 case reflect.Func:
420 addInt(NewCallback(v.Interface()))
421 case reflect.Bool:
422 if v.Bool() {
423 addInt(1)
424 } else {
425 addInt(0)
426 }
427 case reflect.Float32:
428 // On S390X big-endian, float32 goes in upper 32 bits of 64-bit FP register
429 if runtime.GOARCH == "s390x" {
430 addFloat(uintptr(math.Float32bits(float32(v.Float()))) << 32)
431 } else {
432 addFloat(uintptr(math.Float32bits(float32(v.Float()))))
433 }
434 case reflect.Float64:
435 if is32bit {
436 bits := math.Float64bits(v.Float())
437 addFloat(uintptr(bits))
438 addFloat(uintptr(bits >> 32))
439 } else {
440 addFloat(uintptr(math.Float64bits(v.Float())))
441 }
442 case reflect.Struct:
443 keepAlive = addStruct(v, numInts, numFloats, numStack, addInt, addFloat, addStack, keepAlive)
444 default:
445 panic("purego: unsupported kind: " + v.Kind().String())
446 }
447 return keepAlive
448}
449
450// maxRegAllocStructSize is the biggest a struct can be while still fitting in registers.
451// if it is bigger than this than enough space must be allocated on the heap and then passed into
452// the function as the first parameter on amd64 or in R8 on arm64.
453//
454// If you change this make sure to update it in objc_runtime_darwin.go
455const maxRegAllocStructSize = 16
456
457func isAllSameFloat(ty reflect.Type) (allFloats bool, numFields int) {
458 allFloats = true
459 root := ty.Field(0).Type
460 for root.Kind() == reflect.Struct {
461 root = root.Field(0).Type
462 }
463 first := root.Kind()
464 if first != reflect.Float32 && first != reflect.Float64 {
465 allFloats = false
466 }
467 for i := 0; i < ty.NumField(); i++ {
468 f := ty.Field(i).Type
469 if f.Kind() == reflect.Struct {
470 var structNumFields int
471 allFloats, structNumFields = isAllSameFloat(f)
472 numFields += structNumFields
473 continue
474 }
475 numFields++
476 if f.Kind() != first {
477 allFloats = false
478 }
479 }
480 return allFloats, numFields
481}
482
483func checkStructFieldsSupported(ty reflect.Type) {
484 for i := 0; i < ty.NumField(); i++ {
485 f := ty.Field(i).Type
486 if f.Kind() == reflect.Array {
487 f = f.Elem()
488 } else if f.Kind() == reflect.Struct {
489 checkStructFieldsSupported(f)
490 continue
491 }
492 switch f.Kind() {
493 case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
494 reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64,
495 reflect.Uintptr, reflect.Pointer, reflect.UnsafePointer, reflect.Float64, reflect.Float32,
496 reflect.Bool:
497 default:
498 panic(fmt.Sprintf("purego: struct field type %s is not supported", f))
499 }
500 }
501}
502
503func ensureStructSupported() {
504 if runtime.GOARCH != "amd64" && runtime.GOARCH != "arm64" {
505 panic("purego: struct arguments/returns are only supported on amd64 and arm64")
506 }
507 switch runtime.GOOS {
508 case "android", "darwin", "ios", "linux", "windows":
509 default:
510 panic("purego: struct arguments/returns are only supported on android, darwin, ios, linux, and windows")
511 }
512}
513
514// isDarwin is true on platforms that use Apple's calling convention.
515// iOS (GOOS=ios) shares it with macOS (GOOS=darwin).
516const isDarwin = runtime.GOOS == "darwin" || runtime.GOOS == "ios"
517
518// amd64StructReturnInMemory reports whether a struct return value of the given
519// size is returned through a caller-allocated hidden pointer (true) rather than
520// in registers (false). It must only be consulted on amd64.
521func amd64StructReturnInMemory(size uintptr) bool {
522 if size == 0 {
523 return false
524 }
525 if runtime.GOOS == "windows" {
526 // The Win64 ABI returns aggregates of exactly 1, 2, 4, or 8 bytes in
527 // RAX. Every other size is returned through a caller-allocated hidden
528 // pointer that the callee also returns in RAX.
529 switch size {
530 case 1, 2, 4, 8:
531 return false
532 default:
533 return true
534 }
535 }
536 // The System V ABI returns aggregates of up to two eightbytes in registers.
537 return size > maxRegAllocStructSize
538}
539
540func roundUpTo8(val uintptr) uintptr {
541 return (val + align8ByteMask) &^ align8ByteMask
542}
543
544func numOfFloatRegisters() int {
545 switch runtime.GOARCH {
546 case "amd64", "arm64", "loong64", "ppc64le", "riscv64":
547 return 8
548 case "s390x":
549 return 4
550 case "arm":
551 return 16
552 case "386":
553 // i386 SysV ABI passes all arguments on the stack, including floats
554 return 0
555 default:
556 // since this platform isn't supported and can therefore only access
557 // integer registers it is safest to return 8
558 return 8
559 }
560}
561
562func numOfIntegerRegisters() int {
563 switch runtime.GOARCH {
564 case "arm64", "loong64", "ppc64le", "riscv64":
565 return 8
566 case "amd64":
567 return 6
568 case "s390x":
569 // S390X uses R2-R6 for integer arguments
570 return 5
571 case "arm":
572 return 4
573 case "386":
574 // i386 SysV ABI passes all arguments on the stack
575 return 0
576 default:
577 // since this platform isn't supported and can therefore only access
578 // integer registers it is fine to return the maxArgs
579 return maxArgs
580 }
581}
582
583// estimateStackBytes estimates stack bytes needed for Darwin ARM64 validation.
584// This is a conservative estimate used only for early error detection.
585func estimateStackBytes(ty reflect.Type) int {
586 var numInts, numFloats int
587 var stackBytes int
588
589 for i := 0; i < ty.NumIn(); i++ {
590 arg := ty.In(i)
591 size := int(arg.Size())
592
593 // Check if this goes to register or stack
594 usesInt := arg.Kind() != reflect.Float32 && arg.Kind() != reflect.Float64
595 if usesInt && numInts < numOfIntegerRegisters() {
596 numInts++
597 } else if !usesInt && numFloats < numOfFloatRegisters() {
598 numFloats++
599 } else {
600 // Goes to stack - accumulate total bytes
601 stackBytes += size
602 }
603 }
604 // Round total to 8-byte boundary
605 if stackBytes > 0 && stackBytes%align8ByteSize != 0 {
606 stackBytes = int(roundUpTo8(uintptr(stackBytes)))
607 }
608 return stackBytes
609}