1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: 2024 The Ebitengine Authors
3
4package purego
5
6import (
7 "math"
8 "reflect"
9 "runtime"
10 "unsafe"
11)
12
13func getStruct(outType reflect.Type, syscall syscallArgs) (v reflect.Value) {
14 outSize := outType.Size()
15 if runtime.GOOS == "windows" {
16 switch {
17 case outSize == 0:
18 return reflect.New(outType).Elem()
19 case amd64StructReturnInMemory(outSize):
20 // Returned through the caller-allocated hidden pointer, which the
21 // callee also returns in RAX.
22 return reflect.NewAt(outType, *(*unsafe.Pointer)(unsafe.Pointer(&syscall.a1))).Elem()
23 default:
24 // 1, 2, 4, or 8 byte aggregates are returned in RAX.
25 return reflect.NewAt(outType, unsafe.Pointer(&struct{ a uintptr }{syscall.a1})).Elem()
26 }
27 }
28 switch {
29 case outSize == 0:
30 return reflect.New(outType).Elem()
31 case outSize <= 8:
32 if isAllFloats(outType) {
33 // 2 float32s or 1 float64s are return in the float register
34 return reflect.NewAt(outType, unsafe.Pointer(&struct{ a uintptr }{syscall.f1})).Elem()
35 }
36 // up to 8 bytes is returned in RAX
37 return reflect.NewAt(outType, unsafe.Pointer(&struct{ a uintptr }{syscall.a1})).Elem()
38 case outSize <= 16:
39 r1, r2 := syscall.a1, syscall.a2
40 if isAllFloats(outType) {
41 r1 = syscall.f1
42 r2 = syscall.f2
43 } else {
44 // check first 8 bytes if it's floats
45 hasFirstFloat := false
46 f1 := outType.Field(0).Type
47 if f1.Kind() == reflect.Float64 || f1.Kind() == reflect.Float32 && outType.Field(1).Type.Kind() == reflect.Float32 {
48 r1 = syscall.f1
49 hasFirstFloat = true
50 }
51
52 // find index of the field that starts the second 8 bytes
53 var i int
54 for i = 0; i < outType.NumField(); i++ {
55 if outType.Field(i).Offset == 8 {
56 break
57 }
58 }
59
60 // check last 8 bytes if they are floats
61 f1 = outType.Field(i).Type
62 if f1.Kind() == reflect.Float64 || f1.Kind() == reflect.Float32 && i+1 == outType.NumField() {
63 r2 = syscall.f1
64 } else if hasFirstFloat {
65 // if the first field was a float then that means the second integer field
66 // comes from the first integer register
67 r2 = syscall.a1
68 }
69 }
70 return reflect.NewAt(outType, unsafe.Pointer(&struct{ a, b uintptr }{r1, r2})).Elem()
71 default:
72 // create struct from the Go pointer created above
73 // weird pointer dereference to circumvent go vet
74 return reflect.NewAt(outType, *(*unsafe.Pointer)(unsafe.Pointer(&syscall.a1))).Elem()
75 }
76}
77
78func isAllFloats(ty reflect.Type) bool {
79 for i := 0; i < ty.NumField(); i++ {
80 f := ty.Field(i)
81 switch f.Type.Kind() {
82 case reflect.Float64, reflect.Float32:
83 default:
84 return false
85 }
86 }
87 return true
88}
89
90// https://refspecs.linuxbase.org/elf/x86_64-abi-0.99.pdf
91// https://gitlab.com/x86-psABIs/x86-64-ABI
92// Class determines where the 8 byte value goes.
93// Higher value classes win over lower value classes
94const (
95 _NO_CLASS = 0b0000
96 _SSE = 0b0001
97 _X87 = 0b0011 // long double not used in Go
98 _INTEGER = 0b0111
99 _MEMORY = 0b1111
100)
101
102func addStruct(v reflect.Value, numInts, numFloats, numStack *int, addInt, addFloat, addStack func(uintptr), keepAlive []any) []any {
103 if runtime.GOOS == "windows" {
104 // Win64 still passes an empty struct as an argument slot, so this must
105 // run before the zero-size early return used by the System V path.
106 return addStructWindows(v, addInt, keepAlive)
107 }
108
109 if v.Type().Size() == 0 {
110 return keepAlive
111 }
112
113 // if greater than 64 bytes place on stack
114 if v.Type().Size() > 8*8 {
115 placeStack(v, addStack)
116 return keepAlive
117 }
118 var (
119 savedNumFloats = *numFloats
120 savedNumInts = *numInts
121 savedNumStack = *numStack
122 )
123 placeOnStack := postMerger(v.Type()) || !tryPlaceRegister(v, addFloat, addInt)
124 if placeOnStack {
125 // reset any values placed in registers
126 *numFloats = savedNumFloats
127 *numInts = savedNumInts
128 *numStack = savedNumStack
129 placeStack(v, addStack)
130 }
131 return keepAlive
132}
133
134// addStructWindows passes a struct argument under the Win64 ABI. Aggregates of
135// exactly 1, 2, 4, or 8 bytes are passed by value in a single integer slot; all
136// other sizes are passed as a pointer to a caller-allocated copy. Empty structs
137// fall in the latter group: unlike the System V ABI, Win64 still consumes an
138// argument slot for them.
139func addStructWindows(v reflect.Value, addInt func(uintptr), keepAlive []any) []any {
140 switch v.Type().Size() {
141 case 1, 2, 4, 8:
142 var val uintptr
143 reflect.NewAt(v.Type(), unsafe.Pointer(&val)).Elem().Set(v)
144 addInt(val)
145 default:
146 ptrStruct := reflect.New(v.Type())
147 ptrStruct.Elem().Set(v)
148 ptr := ptrStruct.Elem().Addr().UnsafePointer()
149 keepAlive = append(keepAlive, ptr)
150 addInt(uintptr(ptr))
151 }
152 return keepAlive
153}
154
155func postMerger(t reflect.Type) (passInMemory bool) {
156 // (c) If the size of the aggregate exceeds two eightbytes and the first eight- byte isn’t SSE or any other
157 // eightbyte isn’t SSEUP, the whole argument is passed in memory.
158 if t.Kind() != reflect.Struct {
159 return false
160 }
161 if t.Size() <= 2*8 {
162 return false
163 }
164 return true // Go does not have an SSE/SSEUP type so this is always true
165}
166
167func tryPlaceRegister(v reflect.Value, addFloat func(uintptr), addInt func(uintptr)) (ok bool) {
168 ok = true
169 var val uint64
170 var shift byte // # of bits to shift
171 var flushed bool
172 class := _NO_CLASS
173 flushIfNeeded := func() {
174 if flushed {
175 return
176 }
177 flushed = true
178 if class == _SSE {
179 addFloat(uintptr(val))
180 } else {
181 addInt(uintptr(val))
182 }
183 val = 0
184 shift = 0
185 class = _NO_CLASS
186 }
187 var place func(v reflect.Value)
188 place = func(v reflect.Value) {
189 var numFields int
190 if v.Kind() == reflect.Struct {
191 numFields = v.Type().NumField()
192 } else {
193 numFields = v.Type().Len()
194 }
195
196 for i := 0; i < numFields; i++ {
197 flushed = false
198 var f reflect.Value
199 if v.Kind() == reflect.Struct {
200 f = v.Field(i)
201 } else {
202 f = v.Index(i)
203 }
204 switch f.Kind() {
205 case reflect.Struct:
206 place(f)
207 case reflect.Bool:
208 if f.Bool() {
209 val |= 1 << shift
210 }
211 shift += 8
212 class |= _INTEGER
213 case reflect.Pointer, reflect.UnsafePointer:
214 val = uint64(f.Pointer())
215 shift = 64
216 class = _INTEGER
217 case reflect.Int8:
218 val |= uint64(f.Int()&0xFF) << shift
219 shift += 8
220 class |= _INTEGER
221 case reflect.Int16:
222 val |= uint64(f.Int()&0xFFFF) << shift
223 shift += 16
224 class |= _INTEGER
225 case reflect.Int32:
226 val |= uint64(f.Int()&0xFFFF_FFFF) << shift
227 shift += 32
228 class |= _INTEGER
229 case reflect.Int64, reflect.Int:
230 val = uint64(f.Int())
231 shift = 64
232 class = _INTEGER
233 case reflect.Uint8:
234 val |= f.Uint() << shift
235 shift += 8
236 class |= _INTEGER
237 case reflect.Uint16:
238 val |= f.Uint() << shift
239 shift += 16
240 class |= _INTEGER
241 case reflect.Uint32:
242 val |= f.Uint() << shift
243 shift += 32
244 class |= _INTEGER
245 case reflect.Uint64, reflect.Uint, reflect.Uintptr:
246 val = f.Uint()
247 shift = 64
248 class = _INTEGER
249 case reflect.Float32:
250 val |= uint64(math.Float32bits(float32(f.Float()))) << shift
251 shift += 32
252 class |= _SSE
253 case reflect.Float64:
254 if v.Type().Size() > 16 {
255 ok = false
256 return
257 }
258 val = uint64(math.Float64bits(f.Float()))
259 shift = 64
260 class = _SSE
261 case reflect.Array:
262 place(f)
263 default:
264 panic("purego: unsupported kind " + f.Kind().String())
265 }
266
267 if shift == 64 {
268 flushIfNeeded()
269 } else if shift > 64 {
270 // Should never happen, but may if we forget to reset shift after flush (or forget to flush),
271 // better fall apart here, than corrupt arguments.
272 panic("purego: tryPlaceRegisters shift > 64")
273 }
274 }
275 }
276
277 place(v)
278 flushIfNeeded()
279 return ok
280}
281
282func placeStack(v reflect.Value, addStack func(uintptr)) {
283 // Copy the struct as a contiguous block of memory in eightbyte (8-byte)
284 // chunks. The x86-64 ABI requires structs passed on the stack to be
285 // laid out exactly as in memory, including padding and field packing
286 // within eightbytes. Decomposing field-by-field would place each field
287 // as a separate stack slot, breaking structs with mixed-type fields
288 // that share an eightbyte (e.g. int32 + float32).
289 if !v.CanAddr() {
290 tmp := reflect.New(v.Type()).Elem()
291 tmp.Set(v)
292 v = tmp
293 }
294 ptr := v.Addr().UnsafePointer()
295 size := v.Type().Size()
296 for off := uintptr(0); off < size; off += 8 {
297 chunk := *(*uintptr)(unsafe.Add(ptr, off))
298 addStack(chunk)
299 }
300}
301
302func placeRegisters(v reflect.Value, addFloat func(uintptr), addInt func(uintptr)) {
303 panic("purego: placeRegisters not implemented on amd64")
304}
305
306// shouldBundleStackArgs always returns false on non-Darwin platforms
307// since C-style stack argument bundling is only needed on Darwin ARM64.
308func shouldBundleStackArgs(v reflect.Value, numInts, numFloats int) bool {
309 return false
310}
311
312// structFitsInRegisters is not used on amd64.
313func structFitsInRegisters(val reflect.Value, tempNumInts, tempNumFloats int) (bool, int, int) {
314 panic("purego: structFitsInRegisters should not be called on amd64")
315}
316
317// collectStackArgs is not used on amd64.
318func collectStackArgs(args []reflect.Value, startIdx int, numInts, numFloats int,
319 keepAlive []any, addInt, addFloat, addStack func(uintptr),
320 pNumInts, pNumFloats, pNumStack *int) ([]reflect.Value, []any) {
321 panic("purego: collectStackArgs should not be called on amd64")
322}
323
324// bundleStackArgs is not used on amd64.
325func bundleStackArgs(stackArgs []reflect.Value, addStack func(uintptr)) {
326 panic("purego: bundleStackArgs should not be called on amd64")
327}
328
329// getCallbackStruct reads a struct argument from the callback frame on amd64.
330// It mirrors the SysV AMD64 ABI rules used by addStruct for the Go→C path.
331//
332// getCallbackStruct is only used on Unix. On Windows, callbacks are handled by
333// the runtime's own callback mechanism, so this function is compiled but unused.
334//
335// SysV AMD64 struct argument passing rules:
336// - Struct > 16 bytes (postMerger): passed on the stack as raw bytes
337// - Struct ≤ 16 bytes: classify each eightbyte (INTEGER or SSE),
338// read from the appropriate register class
339// - If not enough registers for all eightbytes: entire struct goes on the stack
340func getCallbackStruct(inType reflect.Type, frame unsafe.Pointer, floatsN *int, intsN *int, stackSlot *int, stackByteOffset *uintptr) reflect.Value {
341 switch runtime.GOOS {
342 case "android", "darwin", "freebsd", "ios", "linux", "netbsd":
343 default:
344 panic("purego: getCallbackStruct is not supported on " + runtime.GOOS)
345 }
346
347 f := (*[callbackMaxFrame]uintptr)(frame)
348 size := inType.Size()
349
350 // Structs > 16 bytes are passed on the stack as raw bytes (SysV ABI MEMORY class).
351 if postMerger(inType) {
352 numSlots := int((size + 7) / 8)
353 v := reflect.NewAt(inType, unsafe.Pointer(&f[*stackSlot])).Elem()
354 *stackSlot += numSlots
355 return v
356 }
357
358 // Struct ≤ 16 bytes: classify each eightbyte and read from the appropriate register.
359 numEightbytes := int((size + 7) / 8)
360
361 // Count how many integer and SSE registers this struct needs.
362 var needInts, needFloats int
363 for i := 0; i < numEightbytes; i++ {
364 class := classifyEightbyte(inType, uintptr(i)*8, uintptr(i)*8+8)
365 if class == _SSE {
366 needFloats++
367 } else {
368 needInts++
369 }
370 }
371
372 // If not enough registers for all eightbytes, the entire struct goes on the stack.
373 if *intsN+needInts > numOfIntegerRegisters() || *floatsN+needFloats > numOfFloatRegisters() {
374 v := reflect.NewAt(inType, unsafe.Pointer(&f[*stackSlot])).Elem()
375 *stackSlot += numEightbytes
376 return v
377 }
378
379 // Read each eightbyte from its appropriate register class.
380 var r1, r2 uintptr
381 for i := 0; i < numEightbytes; i++ {
382 class := classifyEightbyte(inType, uintptr(i)*8, uintptr(i)*8+8)
383 if class == _SSE {
384 if i == 0 {
385 r1 = f[*floatsN]
386 } else {
387 r2 = f[*floatsN]
388 }
389 *floatsN++
390 } else {
391 if i == 0 {
392 r1 = f[numOfFloatRegisters()+*intsN]
393 } else {
394 r2 = f[numOfFloatRegisters()+*intsN]
395 }
396 *intsN++
397 }
398 }
399
400 if numEightbytes == 1 {
401 return reflect.NewAt(inType, unsafe.Pointer(&struct{ a uintptr }{r1})).Elem()
402 }
403 return reflect.NewAt(inType, unsafe.Pointer(&struct{ a, b uintptr }{r1, r2})).Elem()
404}
405
406func setStruct(a *callbackArgs, ret reflect.Value) {
407 outSize := ret.Type().Size()
408 switch {
409 case outSize == 0:
410 return
411 case outSize <= 16:
412 // Copy the struct's raw bytes (including padding) into a buffer.
413 var buf [2]uintptr
414 reflect.NewAt(ret.Type(), unsafe.Pointer(&buf[0])).Elem().Set(ret)
415 // Classify each eightbyte by the SysV ABI rules (§3.2.3, rule 4d
416 // of https://refspecs.linuxbase.org/elf/x86_64-abi-0.99.pdf).
417 // INTEGER wins over SSE for mixed eightbytes. Place INTEGER eightbytes in
418 // result[0]/result[1] (AX/DX) and SSE eightbytes in result[2]/result[3]
419 // (XMM0/XMM1).
420 // Assign each eightbyte to the next available register of the
421 // appropriate class. The ABI counts integer (AX, DX) and SSE
422 // (XMM0, XMM1) return registers independently.
423 var numInts int
424 var numFloats int
425 for i := 0; i < 2 && uintptr(i)*8 < outSize; i++ {
426 class := classifyEightbyte(ret.Type(), uintptr(i)*8, uintptr(i)*8+8)
427 if class == _SSE {
428 switch numFloats {
429 case 0:
430 a.result[2] = buf[i]
431 case 1:
432 a.result[3] = buf[i]
433 }
434 numFloats++
435 } else {
436 switch numInts {
437 case 0:
438 a.result[0] = buf[i]
439 case 1:
440 a.result[1] = buf[i]
441 }
442 numInts++
443 }
444 }
445 default:
446 // Structs > 16 bytes are returned by hidden pointer.
447 // a.result[0] contains the pointer passed by the caller in RDI.
448 // Write the struct through this pointer.
449 reflect.NewAt(ret.Type(), *(*unsafe.Pointer)(unsafe.Pointer(&a.result[0]))).Elem().Set(ret)
450 }
451}
452
453// classifyEightbyte returns the SysV ABI class for the byte range [start, end)
454// within a type, by examining all scalar fields that overlap that range.
455func classifyEightbyte(t reflect.Type, start, end uintptr) int {
456 return doClassifyEightbyte(t, 0, start, end)
457}
458
459func doClassifyEightbyte(t reflect.Type, base, start, end uintptr) int {
460 switch t.Kind() {
461 case reflect.Struct:
462 class := _NO_CLASS
463 for i := 0; i < t.NumField(); i++ {
464 f := t.Field(i)
465 class |= doClassifyEightbyte(f.Type, base+f.Offset, start, end)
466 }
467 return class
468 case reflect.Array:
469 class := _NO_CLASS
470 elemSize := t.Elem().Size()
471 for i := 0; i < t.Len(); i++ {
472 class |= doClassifyEightbyte(t.Elem(), base+uintptr(i)*elemSize, start, end)
473 }
474 return class
475 default:
476 fStart := base
477 fEnd := base + t.Size()
478 if fStart >= end || fEnd <= start {
479 return _NO_CLASS
480 }
481 switch t.Kind() {
482 case reflect.Float32, reflect.Float64:
483 return _SSE
484 default:
485 return _INTEGER
486 }
487 }
488}