1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: 2024 The Ebitengine Authors
3
4package purego
5
6import (
7 "math"
8 "reflect"
9 "runtime"
10 "unsafe"
11)
12
13// structReturnInMemory reports whether a struct return value of the given size
14// is returned through a caller-allocated hidden pointer passed as the first
15// integer argument (true) rather than in registers (false).
16func structReturnInMemory(outType reflect.Type) bool {
17 size := outType.Size()
18 if size == 0 {
19 return false
20 }
21 if runtime.GOOS == "windows" {
22 // The Win64 ABI returns aggregates of exactly 1, 2, 4, or 8 bytes in
23 // RAX. Every other size is returned through a caller-allocated hidden
24 // pointer that the callee also returns in RAX.
25 switch size {
26 case 1, 2, 4, 8:
27 return false
28 default:
29 return true
30 }
31 }
32 // The System V ABI returns aggregates of up to two eightbytes in registers.
33 return size > maxRegAllocStructSize
34}
35
36func getStruct(outType reflect.Type, syscall syscallArgs) (v reflect.Value) {
37 outSize := outType.Size()
38 if runtime.GOOS == "windows" {
39 switch {
40 case outSize == 0:
41 return reflect.New(outType).Elem()
42 case structReturnInMemory(outType):
43 // Returned through the caller-allocated hidden pointer, which the
44 // callee also returns in RAX.
45 return reflect.NewAt(outType, *(*unsafe.Pointer)(unsafe.Pointer(&syscall.a1))).Elem()
46 default:
47 // 1, 2, 4, or 8 byte aggregates are returned in RAX.
48 return reflect.NewAt(outType, unsafe.Pointer(&struct{ a uintptr }{syscall.a1})).Elem()
49 }
50 }
51 switch {
52 case outSize == 0:
53 return reflect.New(outType).Elem()
54 case outSize <= 8:
55 if isAllFloats(outType) {
56 // 2 float32s or 1 float64s are return in the float register
57 return reflect.NewAt(outType, unsafe.Pointer(&struct{ a uintptr }{syscall.f1})).Elem()
58 }
59 // up to 8 bytes is returned in RAX
60 return reflect.NewAt(outType, unsafe.Pointer(&struct{ a uintptr }{syscall.a1})).Elem()
61 case outSize <= 16:
62 r1, r2 := syscall.a1, syscall.a2
63 if isAllFloats(outType) {
64 r1 = syscall.f1
65 r2 = syscall.f2
66 } else {
67 // check first 8 bytes if it's floats
68 hasFirstFloat := false
69 f1 := outType.Field(0).Type
70 if f1.Kind() == reflect.Float64 || f1.Kind() == reflect.Float32 && outType.Field(1).Type.Kind() == reflect.Float32 {
71 r1 = syscall.f1
72 hasFirstFloat = true
73 }
74
75 // find index of the field that starts the second 8 bytes
76 var i int
77 for i = 0; i < outType.NumField(); i++ {
78 if outType.Field(i).Offset == 8 {
79 break
80 }
81 }
82
83 // check last 8 bytes if they are floats
84 f1 = outType.Field(i).Type
85 if f1.Kind() == reflect.Float64 || f1.Kind() == reflect.Float32 && i+1 == outType.NumField() {
86 r2 = syscall.f1
87 } else if hasFirstFloat {
88 // if the first field was a float then that means the second integer field
89 // comes from the first integer register
90 r2 = syscall.a1
91 }
92 }
93 return reflect.NewAt(outType, unsafe.Pointer(&struct{ a, b uintptr }{r1, r2})).Elem()
94 default:
95 // create struct from the Go pointer created above
96 // weird pointer dereference to circumvent go vet
97 return reflect.NewAt(outType, *(*unsafe.Pointer)(unsafe.Pointer(&syscall.a1))).Elem()
98 }
99}
100
101func isAllFloats(ty reflect.Type) bool {
102 for i := 0; i < ty.NumField(); i++ {
103 f := ty.Field(i)
104 switch f.Type.Kind() {
105 case reflect.Float64, reflect.Float32:
106 default:
107 return false
108 }
109 }
110 return true
111}
112
113// https://refspecs.linuxbase.org/elf/x86_64-abi-0.99.pdf
114// https://gitlab.com/x86-psABIs/x86-64-ABI
115// Class determines where the 8 byte value goes.
116// Higher value classes win over lower value classes
117const (
118 _NO_CLASS = 0b0000
119 _SSE = 0b0001
120 _X87 = 0b0011 // long double not used in Go
121 _INTEGER = 0b0111
122 _MEMORY = 0b1111
123)
124
125func addStruct(v reflect.Value, numInts, numFloats, numStack *int, addInt, addFloat, addStack func(uintptr), keepAlive []any) []any {
126 if runtime.GOOS == "windows" {
127 // Win64 still passes an empty struct as an argument slot, so this must
128 // run before the zero-size early return used by the System V path.
129 return addStructWindows(v, addInt, keepAlive)
130 }
131
132 if v.Type().Size() == 0 {
133 return keepAlive
134 }
135
136 // if greater than 64 bytes place on stack
137 if v.Type().Size() > 8*8 {
138 placeStack(v, addStack)
139 return keepAlive
140 }
141 var (
142 savedNumFloats = *numFloats
143 savedNumInts = *numInts
144 savedNumStack = *numStack
145 )
146 placeOnStack := postMerger(v.Type()) || !tryPlaceRegister(v, addFloat, addInt)
147 if placeOnStack {
148 // reset any values placed in registers
149 *numFloats = savedNumFloats
150 *numInts = savedNumInts
151 *numStack = savedNumStack
152 placeStack(v, addStack)
153 }
154 return keepAlive
155}
156
157// addStructWindows passes a struct argument under the Win64 ABI. Aggregates of
158// exactly 1, 2, 4, or 8 bytes are passed by value in a single integer slot; all
159// other sizes are passed as a pointer to a caller-allocated copy. Empty structs
160// fall in the latter group: unlike the System V ABI, Win64 still consumes an
161// argument slot for them.
162func addStructWindows(v reflect.Value, addInt func(uintptr), keepAlive []any) []any {
163 switch v.Type().Size() {
164 case 1, 2, 4, 8:
165 var val uintptr
166 reflect.NewAt(v.Type(), unsafe.Pointer(&val)).Elem().Set(v)
167 addInt(val)
168 default:
169 ptrStruct := reflect.New(v.Type())
170 ptrStruct.Elem().Set(v)
171 ptr := ptrStruct.Elem().Addr().UnsafePointer()
172 keepAlive = append(keepAlive, ptr)
173 addInt(uintptr(ptr))
174 }
175 return keepAlive
176}
177
178func postMerger(t reflect.Type) (passInMemory bool) {
179 // (c) If the size of the aggregate exceeds two eightbytes and the first eight- byte isn’t SSE or any other
180 // eightbyte isn’t SSEUP, the whole argument is passed in memory.
181 if t.Kind() != reflect.Struct {
182 return false
183 }
184 if t.Size() <= 2*8 {
185 return false
186 }
187 return true // Go does not have an SSE/SSEUP type so this is always true
188}
189
190func tryPlaceRegister(v reflect.Value, addFloat func(uintptr), addInt func(uintptr)) (ok bool) {
191 ok = true
192 var val uint64
193 var shift byte // # of bits to shift
194 var flushed bool
195 class := _NO_CLASS
196 flushIfNeeded := func() {
197 if flushed {
198 return
199 }
200 flushed = true
201 if class == _SSE {
202 addFloat(uintptr(val))
203 } else {
204 addInt(uintptr(val))
205 }
206 val = 0
207 shift = 0
208 class = _NO_CLASS
209 }
210 var place func(v reflect.Value)
211 place = func(v reflect.Value) {
212 var numFields int
213 if v.Kind() == reflect.Struct {
214 numFields = v.Type().NumField()
215 } else {
216 numFields = v.Type().Len()
217 }
218
219 for i := 0; i < numFields; i++ {
220 flushed = false
221 var f reflect.Value
222 if v.Kind() == reflect.Struct {
223 f = v.Field(i)
224 } else {
225 f = v.Index(i)
226 }
227 switch f.Kind() {
228 case reflect.Struct:
229 place(f)
230 case reflect.Bool:
231 if f.Bool() {
232 val |= 1 << shift
233 }
234 shift += 8
235 class |= _INTEGER
236 case reflect.Pointer, reflect.UnsafePointer:
237 val = uint64(f.Pointer())
238 shift = 64
239 class = _INTEGER
240 case reflect.Int8:
241 val |= uint64(f.Int()&0xFF) << shift
242 shift += 8
243 class |= _INTEGER
244 case reflect.Int16:
245 val |= uint64(f.Int()&0xFFFF) << shift
246 shift += 16
247 class |= _INTEGER
248 case reflect.Int32:
249 val |= uint64(f.Int()&0xFFFF_FFFF) << shift
250 shift += 32
251 class |= _INTEGER
252 case reflect.Int64, reflect.Int:
253 val = uint64(f.Int())
254 shift = 64
255 class = _INTEGER
256 case reflect.Uint8:
257 val |= f.Uint() << shift
258 shift += 8
259 class |= _INTEGER
260 case reflect.Uint16:
261 val |= f.Uint() << shift
262 shift += 16
263 class |= _INTEGER
264 case reflect.Uint32:
265 val |= f.Uint() << shift
266 shift += 32
267 class |= _INTEGER
268 case reflect.Uint64, reflect.Uint, reflect.Uintptr:
269 val = f.Uint()
270 shift = 64
271 class = _INTEGER
272 case reflect.Float32:
273 val |= uint64(math.Float32bits(float32(f.Float()))) << shift
274 shift += 32
275 class |= _SSE
276 case reflect.Float64:
277 if v.Type().Size() > 16 {
278 ok = false
279 return
280 }
281 val = uint64(math.Float64bits(f.Float()))
282 shift = 64
283 class = _SSE
284 case reflect.Array:
285 place(f)
286 default:
287 panic("purego: unsupported kind " + f.Kind().String())
288 }
289
290 if shift == 64 {
291 flushIfNeeded()
292 } else if shift > 64 {
293 // Should never happen, but may if we forget to reset shift after flush (or forget to flush),
294 // better fall apart here, than corrupt arguments.
295 panic("purego: tryPlaceRegisters shift > 64")
296 }
297 }
298 }
299
300 place(v)
301 flushIfNeeded()
302 return ok
303}
304
305func placeStack(v reflect.Value, addStack func(uintptr)) {
306 // Copy the struct as a contiguous block of memory in eightbyte (8-byte)
307 // chunks. The x86-64 ABI requires structs passed on the stack to be
308 // laid out exactly as in memory, including padding and field packing
309 // within eightbytes. Decomposing field-by-field would place each field
310 // as a separate stack slot, breaking structs with mixed-type fields
311 // that share an eightbyte (e.g. int32 + float32).
312 if !v.CanAddr() {
313 tmp := reflect.New(v.Type()).Elem()
314 tmp.Set(v)
315 v = tmp
316 }
317 ptr := v.Addr().UnsafePointer()
318 size := v.Type().Size()
319 for off := uintptr(0); off < size; off += 8 {
320 chunk := *(*uintptr)(unsafe.Add(ptr, off))
321 addStack(chunk)
322 }
323}
324
325func placeRegisters(v reflect.Value, addFloat func(uintptr), addInt func(uintptr)) {
326 panic("purego: placeRegisters not implemented on amd64")
327}
328
329// shouldBundleStackArgs always returns false on non-Darwin platforms
330// since C-style stack argument bundling is only needed on Darwin ARM64.
331func shouldBundleStackArgs(v reflect.Value, numInts, numFloats int) bool {
332 return false
333}
334
335// structFitsInRegisters is not used on amd64.
336func structFitsInRegisters(val reflect.Value, tempNumInts, tempNumFloats int) (bool, int, int) {
337 panic("purego: structFitsInRegisters should not be called on amd64")
338}
339
340// collectStackArgs is not used on amd64.
341func collectStackArgs(args []reflect.Value, startIdx int, numInts, numFloats int,
342 keepAlive []any, addInt, addFloat, addStack func(uintptr),
343 pNumInts, pNumFloats, pNumStack *int) ([]reflect.Value, []any) {
344 panic("purego: collectStackArgs should not be called on amd64")
345}
346
347// bundleStackArgs is not used on amd64.
348func bundleStackArgs(stackArgs []reflect.Value, addStack func(uintptr)) {
349 panic("purego: bundleStackArgs should not be called on amd64")
350}
351
352// getCallbackStruct reads a struct argument from the callback frame on amd64.
353// It mirrors the SysV AMD64 ABI rules used by addStruct for the Go→C path.
354//
355// getCallbackStruct is only used on Unix. On Windows, callbacks are handled by
356// the runtime's own callback mechanism, so this function is compiled but unused.
357//
358// SysV AMD64 struct argument passing rules:
359// - Struct > 16 bytes (postMerger): passed on the stack as raw bytes
360// - Struct ≤ 16 bytes: classify each eightbyte (INTEGER or SSE),
361// read from the appropriate register class
362// - If not enough registers for all eightbytes: entire struct goes on the stack
363func getCallbackStruct(inType reflect.Type, frame unsafe.Pointer, floatsN *int, intsN *int, stackSlot *int, stackByteOffset *uintptr) reflect.Value {
364 switch runtime.GOOS {
365 case "android", "darwin", "freebsd", "ios", "linux", "netbsd":
366 default:
367 panic("purego: getCallbackStruct is not supported on " + runtime.GOOS)
368 }
369
370 f := (*[callbackMaxFrame]uintptr)(frame)
371 size := inType.Size()
372
373 // Structs > 16 bytes are passed on the stack as raw bytes (SysV ABI MEMORY class).
374 if postMerger(inType) {
375 numSlots := int((size + 7) / 8)
376 v := reflect.NewAt(inType, unsafe.Pointer(&f[*stackSlot])).Elem()
377 *stackSlot += numSlots
378 return v
379 }
380
381 // Struct ≤ 16 bytes: classify each eightbyte and read from the appropriate register.
382 numEightbytes := int((size + 7) / 8)
383
384 // Count how many integer and SSE registers this struct needs.
385 var needInts, needFloats int
386 for i := 0; i < numEightbytes; i++ {
387 class := classifyEightbyte(inType, uintptr(i)*8, uintptr(i)*8+8)
388 if class == _SSE {
389 needFloats++
390 } else {
391 needInts++
392 }
393 }
394
395 // If not enough registers for all eightbytes, the entire struct goes on the stack.
396 if *intsN+needInts > numOfIntegerRegisters() || *floatsN+needFloats > numOfFloatRegisters() {
397 v := reflect.NewAt(inType, unsafe.Pointer(&f[*stackSlot])).Elem()
398 *stackSlot += numEightbytes
399 return v
400 }
401
402 // Read each eightbyte from its appropriate register class.
403 var r1, r2 uintptr
404 for i := 0; i < numEightbytes; i++ {
405 class := classifyEightbyte(inType, uintptr(i)*8, uintptr(i)*8+8)
406 if class == _SSE {
407 if i == 0 {
408 r1 = f[*floatsN]
409 } else {
410 r2 = f[*floatsN]
411 }
412 *floatsN++
413 } else {
414 if i == 0 {
415 r1 = f[numOfFloatRegisters()+*intsN]
416 } else {
417 r2 = f[numOfFloatRegisters()+*intsN]
418 }
419 *intsN++
420 }
421 }
422
423 if numEightbytes == 1 {
424 return reflect.NewAt(inType, unsafe.Pointer(&struct{ a uintptr }{r1})).Elem()
425 }
426 return reflect.NewAt(inType, unsafe.Pointer(&struct{ a, b uintptr }{r1, r2})).Elem()
427}
428
429func setStruct(a *callbackArgs, ret reflect.Value) {
430 outSize := ret.Type().Size()
431 switch {
432 case outSize == 0:
433 return
434 case outSize <= 16:
435 // Copy the struct's raw bytes (including padding) into a buffer.
436 var buf [2]uintptr
437 reflect.NewAt(ret.Type(), unsafe.Pointer(&buf[0])).Elem().Set(ret)
438 // Classify each eightbyte by the SysV ABI rules (§3.2.3, rule 4d
439 // of https://refspecs.linuxbase.org/elf/x86_64-abi-0.99.pdf).
440 // INTEGER wins over SSE for mixed eightbytes. Place INTEGER eightbytes in
441 // result[0]/result[1] (AX/DX) and SSE eightbytes in result[2]/result[3]
442 // (XMM0/XMM1).
443 // Assign each eightbyte to the next available register of the
444 // appropriate class. The ABI counts integer (AX, DX) and SSE
445 // (XMM0, XMM1) return registers independently.
446 var numInts int
447 var numFloats int
448 for i := 0; i < 2 && uintptr(i)*8 < outSize; i++ {
449 class := classifyEightbyte(ret.Type(), uintptr(i)*8, uintptr(i)*8+8)
450 if class == _SSE {
451 switch numFloats {
452 case 0:
453 a.result[2] = buf[i]
454 case 1:
455 a.result[3] = buf[i]
456 }
457 numFloats++
458 } else {
459 switch numInts {
460 case 0:
461 a.result[0] = buf[i]
462 case 1:
463 a.result[1] = buf[i]
464 }
465 numInts++
466 }
467 }
468 default:
469 // Structs > 16 bytes are returned by hidden pointer.
470 // a.result[0] contains the pointer passed by the caller in RDI.
471 // Write the struct through this pointer.
472 reflect.NewAt(ret.Type(), *(*unsafe.Pointer)(unsafe.Pointer(&a.result[0]))).Elem().Set(ret)
473 }
474}
475
476// classifyEightbyte returns the SysV ABI class for the byte range [start, end)
477// within a type, by examining all scalar fields that overlap that range.
478func classifyEightbyte(t reflect.Type, start, end uintptr) int {
479 return doClassifyEightbyte(t, 0, start, end)
480}
481
482func doClassifyEightbyte(t reflect.Type, base, start, end uintptr) int {
483 switch t.Kind() {
484 case reflect.Struct:
485 class := _NO_CLASS
486 for i := 0; i < t.NumField(); i++ {
487 f := t.Field(i)
488 class |= doClassifyEightbyte(f.Type, base+f.Offset, start, end)
489 }
490 return class
491 case reflect.Array:
492 class := _NO_CLASS
493 elemSize := t.Elem().Size()
494 for i := 0; i < t.Len(); i++ {
495 class |= doClassifyEightbyte(t.Elem(), base+uintptr(i)*elemSize, start, end)
496 }
497 return class
498 default:
499 fStart := base
500 fEnd := base + t.Size()
501 if fStart >= end || fEnd <= start {
502 return _NO_CLASS
503 }
504 switch t.Kind() {
505 case reflect.Float32, reflect.Float64:
506 return _SSE
507 default:
508 return _INTEGER
509 }
510 }
511}