1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: 2024 The Ebitengine Authors
3
4package purego
5
6import (
7 "math"
8 "reflect"
9 "runtime"
10 "unsafe"
11)
12
13// structReturnInMemory reports whether a struct return value of the given size
14// is returned through a caller-allocated hidden pointer passed as the first
15// integer argument (true) rather than in registers (false).
16func structReturnInMemory(outType reflect.Type) bool {
17 size := outType.Size()
18 if size == 0 {
19 return false
20 }
21 if runtime.GOOS == "windows" {
22 // The Win64 ABI returns aggregates of exactly 1, 2, 4, or 8 bytes in
23 // RAX. Every other size is returned through a caller-allocated hidden
24 // pointer that the callee also returns in RAX.
25 switch size {
26 case 1, 2, 4, 8:
27 return false
28 default:
29 return true
30 }
31 }
32 // The System V ABI returns aggregates of up to two eightbytes in registers.
33 return size > maxRegAllocStructSize
34}
35
36func getStruct(outType reflect.Type, syscall syscallArgs) (v reflect.Value) {
37 outSize := outType.Size()
38 if runtime.GOOS == "windows" {
39 switch {
40 case outSize == 0:
41 return reflect.New(outType).Elem()
42 case structReturnInMemory(outType):
43 // Returned through the caller-allocated hidden pointer, which the
44 // callee also returns in RAX.
45 return reflect.NewAt(outType, *(*unsafe.Pointer)(unsafe.Pointer(&syscall.a1))).Elem()
46 default:
47 // 1, 2, 4, or 8 byte aggregates are returned in RAX.
48 return reflect.NewAt(outType, unsafe.Pointer(&struct{ a uintptr }{syscall.a1})).Elem()
49 }
50 }
51 switch {
52 case outSize == 0:
53 return reflect.New(outType).Elem()
54 case outSize <= 8:
55 if isAllFloats(outType) {
56 // 2 float32s or 1 float64s are return in the float register
57 return reflect.NewAt(outType, unsafe.Pointer(&struct{ a uintptr }{syscall.f1})).Elem()
58 }
59 // up to 8 bytes is returned in RAX
60 return reflect.NewAt(outType, unsafe.Pointer(&struct{ a uintptr }{syscall.a1})).Elem()
61 case outSize <= 16:
62 r1, r2 := syscall.a1, syscall.a2
63 if isAllFloats(outType) {
64 r1 = syscall.f1
65 r2 = syscall.f2
66 } else {
67 // check first 8 bytes if it's floats
68 hasFirstFloat := false
69 numFields := numABIFields(outType)
70 f1 := abiField(outType, 0).Type
71 if f1.Kind() == reflect.Float64 || f1.Kind() == reflect.Float32 && abiField(outType, 1).Type.Kind() == reflect.Float32 {
72 r1 = syscall.f1
73 hasFirstFloat = true
74 }
75
76 // find index of the field that starts the second 8 bytes
77 var i int
78 for i = 0; i < numFields; i++ {
79 if abiField(outType, i).Offset == 8 {
80 break
81 }
82 }
83
84 // check last 8 bytes if they are floats
85 f1 = abiField(outType, i).Type
86 if f1.Kind() == reflect.Float64 || f1.Kind() == reflect.Float32 && i+1 == numFields {
87 r2 = syscall.f1
88 } else if hasFirstFloat {
89 // if the first field was a float then that means the second integer field
90 // comes from the first integer register
91 r2 = syscall.a1
92 }
93 }
94 return reflect.NewAt(outType, unsafe.Pointer(&struct{ a, b uintptr }{r1, r2})).Elem()
95 default:
96 // create struct from the Go pointer created above
97 // weird pointer dereference to circumvent go vet
98 return reflect.NewAt(outType, *(*unsafe.Pointer)(unsafe.Pointer(&syscall.a1))).Elem()
99 }
100}
101
102func isAllFloats(ty reflect.Type) bool {
103 for i := range ty.NumField() {
104 f := ty.Field(i)
105 if !isABIField(f) {
106 continue
107 }
108 switch f.Type.Kind() {
109 case reflect.Float64, reflect.Float32:
110 default:
111 return false
112 }
113 }
114 return true
115}
116
117// https://refspecs.linuxbase.org/elf/x86_64-abi-0.99.pdf
118// https://gitlab.com/x86-psABIs/x86-64-ABI
119// Class determines where the 8 byte value goes.
120// Higher value classes win over lower value classes
121const (
122 _NO_CLASS = 0b0000
123 _SSE = 0b0001
124 _X87 = 0b0011 // long double not used in Go
125 _INTEGER = 0b0111
126 _MEMORY = 0b1111
127)
128
129func addStruct(v reflect.Value, numInts, numFloats, numStack *int, addInt, addFloat, addStack func(uintptr), keepAlive []any) []any {
130 if runtime.GOOS == "windows" {
131 // Win64 still passes an empty struct as an argument slot, so this must
132 // run before the zero-size early return used by the System V path.
133 return addStructWindows(v, addInt, keepAlive)
134 }
135
136 if v.Type().Size() == 0 {
137 return keepAlive
138 }
139
140 // if greater than 64 bytes place on stack
141 if v.Type().Size() > 8*8 {
142 placeStack(v, addStack)
143 return keepAlive
144 }
145 var (
146 savedNumFloats = *numFloats
147 savedNumInts = *numInts
148 savedNumStack = *numStack
149 )
150 placeOnStack := postMerger(v.Type()) || !tryPlaceRegister(v, addFloat, addInt)
151 if placeOnStack {
152 // reset any values placed in registers
153 *numFloats = savedNumFloats
154 *numInts = savedNumInts
155 *numStack = savedNumStack
156 placeStack(v, addStack)
157 }
158 return keepAlive
159}
160
161// addStructWindows passes a struct argument under the Win64 ABI. Aggregates of
162// exactly 1, 2, 4, or 8 bytes are passed by value in a single integer slot; all
163// other sizes are passed as a pointer to a caller-allocated copy. Empty structs
164// fall in the latter group: unlike the System V ABI, Win64 still consumes an
165// argument slot for them.
166func addStructWindows(v reflect.Value, addInt func(uintptr), keepAlive []any) []any {
167 switch v.Type().Size() {
168 case 1, 2, 4, 8:
169 var val uintptr
170 reflect.NewAt(v.Type(), unsafe.Pointer(&val)).Elem().Set(v)
171 addInt(val)
172 default:
173 ptrStruct := reflect.New(v.Type())
174 ptrStruct.Elem().Set(v)
175 ptr := ptrStruct.Elem().Addr().UnsafePointer()
176 keepAlive = append(keepAlive, ptr)
177 addInt(uintptr(ptr))
178 }
179 return keepAlive
180}
181
182func postMerger(t reflect.Type) (passInMemory bool) {
183 // (c) If the size of the aggregate exceeds two eightbytes and the first eight- byte isn’t SSE or any other
184 // eightbyte isn’t SSEUP, the whole argument is passed in memory.
185 if t.Kind() != reflect.Struct {
186 return false
187 }
188 if t.Size() <= 2*8 {
189 return false
190 }
191 return true // Go does not have an SSE/SSEUP type so this is always true
192}
193
194func tryPlaceRegister(v reflect.Value, addFloat func(uintptr), addInt func(uintptr)) (ok bool) {
195 ok = true
196 var val uint64
197 var shift byte // # of bits to shift
198 var flushed bool
199 class := _NO_CLASS
200 flushIfNeeded := func() {
201 if flushed {
202 return
203 }
204 flushed = true
205 if class == _SSE {
206 addFloat(uintptr(val))
207 } else {
208 addInt(uintptr(val))
209 }
210 val = 0
211 shift = 0
212 class = _NO_CLASS
213 }
214 var place func(v reflect.Value)
215 place = func(v reflect.Value) {
216 var numFields int
217 if v.Kind() == reflect.Struct {
218 numFields = v.Type().NumField()
219 } else {
220 numFields = v.Type().Len()
221 }
222
223 for i := range numFields {
224 if v.Kind() == reflect.Struct && !isABIField(v.Type().Field(i)) {
225 continue
226 }
227 flushed = false
228 var f reflect.Value
229 if v.Kind() == reflect.Struct {
230 f = v.Field(i)
231 } else {
232 f = v.Index(i)
233 }
234 switch f.Kind() {
235 case reflect.Struct:
236 place(f)
237 case reflect.Bool:
238 if f.Bool() {
239 val |= 1 << shift
240 }
241 shift += 8
242 class |= _INTEGER
243 case reflect.Pointer, reflect.UnsafePointer:
244 val = uint64(f.Pointer())
245 shift = 64
246 class = _INTEGER
247 case reflect.Int8:
248 val |= uint64(f.Int()&0xFF) << shift
249 shift += 8
250 class |= _INTEGER
251 case reflect.Int16:
252 val |= uint64(f.Int()&0xFFFF) << shift
253 shift += 16
254 class |= _INTEGER
255 case reflect.Int32:
256 val |= uint64(f.Int()&0xFFFF_FFFF) << shift
257 shift += 32
258 class |= _INTEGER
259 case reflect.Int64, reflect.Int:
260 val = uint64(f.Int())
261 shift = 64
262 class = _INTEGER
263 case reflect.Uint8:
264 val |= f.Uint() << shift
265 shift += 8
266 class |= _INTEGER
267 case reflect.Uint16:
268 val |= f.Uint() << shift
269 shift += 16
270 class |= _INTEGER
271 case reflect.Uint32:
272 val |= f.Uint() << shift
273 shift += 32
274 class |= _INTEGER
275 case reflect.Uint64, reflect.Uint, reflect.Uintptr:
276 val = f.Uint()
277 shift = 64
278 class = _INTEGER
279 case reflect.Float32:
280 val |= uint64(math.Float32bits(float32(f.Float()))) << shift
281 shift += 32
282 class |= _SSE
283 case reflect.Float64:
284 if v.Type().Size() > 16 {
285 ok = false
286 return
287 }
288 val = uint64(math.Float64bits(f.Float()))
289 shift = 64
290 class = _SSE
291 case reflect.Array:
292 place(f)
293 default:
294 panic("purego: unsupported kind " + f.Kind().String())
295 }
296
297 if shift == 64 {
298 flushIfNeeded()
299 } else if shift > 64 {
300 // Should never happen, but may if we forget to reset shift after flush (or forget to flush),
301 // better fall apart here, than corrupt arguments.
302 panic("purego: tryPlaceRegisters shift > 64")
303 }
304 }
305 }
306
307 place(v)
308 flushIfNeeded()
309 return ok
310}
311
312func placeStack(v reflect.Value, addStack func(uintptr)) {
313 // Copy the struct as a contiguous block of memory in eightbyte (8-byte)
314 // chunks. The x86-64 ABI requires structs passed on the stack to be
315 // laid out exactly as in memory, including padding and field packing
316 // within eightbytes. Decomposing field-by-field would place each field
317 // as a separate stack slot, breaking structs with mixed-type fields
318 // that share an eightbyte (e.g. int32 + float32).
319 if !v.CanAddr() {
320 tmp := reflect.New(v.Type()).Elem()
321 tmp.Set(v)
322 v = tmp
323 }
324 ptr := v.Addr().UnsafePointer()
325 size := v.Type().Size()
326 for off := uintptr(0); off < size; off += 8 {
327 chunk := *(*uintptr)(unsafe.Add(ptr, off))
328 addStack(chunk)
329 }
330}
331
332func placeRegisters(v reflect.Value, addFloat func(uintptr), addInt func(uintptr)) {
333 panic("purego: placeRegisters not implemented on amd64")
334}
335
336// shouldBundleStackArgs always returns false on non-Darwin platforms
337// since C-style stack argument bundling is only needed on Darwin ARM64.
338func shouldBundleStackArgs(v reflect.Value, numInts, numFloats int) bool {
339 return false
340}
341
342// structFitsInRegisters is not used on amd64.
343func structFitsInRegisters(val reflect.Value, tempNumInts, tempNumFloats int) (bool, int, int) {
344 panic("purego: structFitsInRegisters should not be called on amd64")
345}
346
347// collectStackArgs is not used on amd64.
348func collectStackArgs(args []reflect.Value, startIdx int, numInts, numFloats int,
349 keepAlive []any, addInt, addFloat, addStack func(uintptr),
350 pNumInts, pNumFloats, pNumStack *int) ([]reflect.Value, []any) {
351 panic("purego: collectStackArgs should not be called on amd64")
352}
353
354// bundleStackArgs is not used on amd64.
355func bundleStackArgs(stackArgs []reflect.Value, addStack func(uintptr)) {
356 panic("purego: bundleStackArgs should not be called on amd64")
357}
358
359// getCallbackStruct reads a struct argument from the callback frame on amd64.
360// It mirrors the SysV AMD64 ABI rules used by addStruct for the Go→C path.
361//
362// getCallbackStruct is only used on Unix. On Windows, callbacks are handled by
363// the runtime's own callback mechanism, so this function is compiled but unused.
364//
365// SysV AMD64 struct argument passing rules:
366// - Struct > 16 bytes (postMerger): passed on the stack as raw bytes
367// - Struct ≤ 16 bytes: classify each eightbyte (INTEGER or SSE),
368// read from the appropriate register class
369// - If not enough registers for all eightbytes: entire struct goes on the stack
370func getCallbackStruct(inType reflect.Type, frame unsafe.Pointer, floatsN *int, intsN *int, stackSlot *int, stackByteOffset *uintptr) reflect.Value {
371 switch runtime.GOOS {
372 case "android", "darwin", "freebsd", "ios", "linux", "netbsd":
373 default:
374 panic("purego: getCallbackStruct is not supported on " + runtime.GOOS)
375 }
376
377 f := (*[callbackMaxFrame]uintptr)(frame)
378 size := inType.Size()
379
380 // Structs > 16 bytes are passed on the stack as raw bytes (SysV ABI MEMORY class).
381 if postMerger(inType) {
382 numSlots := int((size + 7) / 8)
383 v := reflect.NewAt(inType, unsafe.Pointer(&f[*stackSlot])).Elem()
384 *stackSlot += numSlots
385 return v
386 }
387
388 // Struct ≤ 16 bytes: classify each eightbyte and read from the appropriate register.
389 numEightbytes := int((size + 7) / 8)
390
391 // Count how many integer and SSE registers this struct needs.
392 var needInts, needFloats int
393 for i := range numEightbytes {
394 class := classifyEightbyte(inType, uintptr(i)*8, uintptr(i)*8+8)
395 if class == _SSE {
396 needFloats++
397 } else {
398 needInts++
399 }
400 }
401
402 // If not enough registers for all eightbytes, the entire struct goes on the stack.
403 if *intsN+needInts > numOfIntegerRegisters() || *floatsN+needFloats > numOfFloatRegisters() {
404 v := reflect.NewAt(inType, unsafe.Pointer(&f[*stackSlot])).Elem()
405 *stackSlot += numEightbytes
406 return v
407 }
408
409 // Read each eightbyte from its appropriate register class.
410 var r1, r2 uintptr
411 for i := range numEightbytes {
412 class := classifyEightbyte(inType, uintptr(i)*8, uintptr(i)*8+8)
413 if class == _SSE {
414 if i == 0 {
415 r1 = f[*floatsN]
416 } else {
417 r2 = f[*floatsN]
418 }
419 *floatsN++
420 } else {
421 if i == 0 {
422 r1 = f[numOfFloatRegisters()+*intsN]
423 } else {
424 r2 = f[numOfFloatRegisters()+*intsN]
425 }
426 *intsN++
427 }
428 }
429
430 if numEightbytes == 1 {
431 return reflect.NewAt(inType, unsafe.Pointer(&struct{ a uintptr }{r1})).Elem()
432 }
433 return reflect.NewAt(inType, unsafe.Pointer(&struct{ a, b uintptr }{r1, r2})).Elem()
434}
435
436func setStruct(a *callbackArgs, ret reflect.Value) {
437 outSize := ret.Type().Size()
438 switch {
439 case outSize == 0:
440 return
441 case outSize <= 16:
442 // Copy the struct's raw bytes (including padding) into a buffer.
443 var buf [2]uintptr
444 reflect.NewAt(ret.Type(), unsafe.Pointer(&buf[0])).Elem().Set(ret)
445 // Classify each eightbyte by the SysV ABI rules (§3.2.3, rule 4d
446 // of https://refspecs.linuxbase.org/elf/x86_64-abi-0.99.pdf).
447 // INTEGER wins over SSE for mixed eightbytes. Place INTEGER eightbytes in
448 // result[0]/result[1] (AX/DX) and SSE eightbytes in result[2]/result[3]
449 // (XMM0/XMM1).
450 // Assign each eightbyte to the next available register of the
451 // appropriate class. The ABI counts integer (AX, DX) and SSE
452 // (XMM0, XMM1) return registers independently.
453 var numInts int
454 var numFloats int
455 for i := 0; i < 2 && uintptr(i)*8 < outSize; i++ {
456 class := classifyEightbyte(ret.Type(), uintptr(i)*8, uintptr(i)*8+8)
457 if class == _SSE {
458 switch numFloats {
459 case 0:
460 a.result[2] = buf[i]
461 case 1:
462 a.result[3] = buf[i]
463 }
464 numFloats++
465 } else {
466 switch numInts {
467 case 0:
468 a.result[0] = buf[i]
469 case 1:
470 a.result[1] = buf[i]
471 }
472 numInts++
473 }
474 }
475 default:
476 // Structs > 16 bytes are returned by hidden pointer.
477 // a.result[0] contains the pointer passed by the caller in RDI.
478 // Write the struct through this pointer.
479 reflect.NewAt(ret.Type(), *(*unsafe.Pointer)(unsafe.Pointer(&a.result[0]))).Elem().Set(ret)
480 }
481}
482
483// classifyEightbyte returns the SysV ABI class for the byte range [start, end)
484// within a type, by examining all scalar fields that overlap that range.
485func classifyEightbyte(t reflect.Type, start, end uintptr) int {
486 return doClassifyEightbyte(t, 0, start, end)
487}
488
489func doClassifyEightbyte(t reflect.Type, base, start, end uintptr) int {
490 switch t.Kind() {
491 case reflect.Struct:
492 class := _NO_CLASS
493 for i := range t.NumField() {
494 f := t.Field(i)
495 class |= doClassifyEightbyte(f.Type, base+f.Offset, start, end)
496 }
497 return class
498 case reflect.Array:
499 class := _NO_CLASS
500 elemSize := t.Elem().Size()
501 for i := range t.Len() {
502 class |= doClassifyEightbyte(t.Elem(), base+uintptr(i)*elemSize, start, end)
503 }
504 return class
505 default:
506 fStart := base
507 fEnd := base + t.Size()
508 if fStart >= end || fEnd <= start {
509 return _NO_CLASS
510 }
511 switch t.Kind() {
512 case reflect.Float32, reflect.Float64:
513 return _SSE
514 default:
515 return _INTEGER
516 }
517 }
518}