1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: 2024 The Ebitengine Authors
3
4package purego
5
6import (
7 "math"
8 "reflect"
9 "runtime"
10 "strconv"
11 stdstrings "strings"
12 "unsafe"
13
14 "github.com/ebitengine/purego/internal/strings"
15)
16
17func getStruct(outType reflect.Type, syscall syscall15Args) (v reflect.Value) {
18 outSize := outType.Size()
19 switch {
20 case outSize == 0:
21 return reflect.New(outType).Elem()
22 case outSize <= 8:
23 r1 := syscall.a1
24 if isAllFloats, numFields := isAllSameFloat(outType); isAllFloats {
25 r1 = syscall.f1
26 if numFields == 2 {
27 r1 = syscall.f2<<32 | syscall.f1
28 }
29 }
30 return reflect.NewAt(outType, unsafe.Pointer(&struct{ a uintptr }{r1})).Elem()
31 case outSize <= 16:
32 r1, r2 := syscall.a1, syscall.a2
33 if isAllFloats, numFields := isAllSameFloat(outType); isAllFloats {
34 switch numFields {
35 case 4:
36 r1 = syscall.f2<<32 | syscall.f1
37 r2 = syscall.f4<<32 | syscall.f3
38 case 3:
39 r1 = syscall.f2<<32 | syscall.f1
40 r2 = syscall.f3
41 case 2:
42 r1 = syscall.f1
43 r2 = syscall.f2
44 default:
45 panic("unreachable")
46 }
47 }
48 return reflect.NewAt(outType, unsafe.Pointer(&struct{ a, b uintptr }{r1, r2})).Elem()
49 default:
50 if isAllFloats, numFields := isAllSameFloat(outType); isAllFloats && numFields <= 4 {
51 switch numFields {
52 case 4:
53 return reflect.NewAt(outType, unsafe.Pointer(&struct{ a, b, c, d uintptr }{syscall.f1, syscall.f2, syscall.f3, syscall.f4})).Elem()
54 case 3:
55 return reflect.NewAt(outType, unsafe.Pointer(&struct{ a, b, c uintptr }{syscall.f1, syscall.f2, syscall.f3})).Elem()
56 default:
57 panic("unreachable")
58 }
59 }
60 // create struct from the Go pointer created in arm64_r8
61 // weird pointer dereference to circumvent go vet
62 return reflect.NewAt(outType, *(*unsafe.Pointer)(unsafe.Pointer(&syscall.arm64_r8))).Elem()
63 }
64}
65
66// https://github.com/ARM-software/abi-aa/blob/main/sysvabi64/sysvabi64.rst
67const (
68 _NO_CLASS = 0b00
69 _FLOAT = 0b01
70 _INT = 0b11
71)
72
73func addStruct(v reflect.Value, numInts, numFloats, numStack *int, addInt, addFloat, addStack func(uintptr), keepAlive []any) []any {
74 if v.Type().Size() == 0 {
75 return keepAlive
76 }
77
78 if hva, hfa, size := isHVA(v.Type()), isHFA(v.Type()), v.Type().Size(); hva || hfa || size <= 16 {
79 // if this doesn't fit entirely in registers then
80 // each element goes onto the stack
81 if hfa && *numFloats+v.NumField() > numOfFloatRegisters() {
82 *numFloats = numOfFloatRegisters()
83 } else if hva && *numInts+v.NumField() > numOfIntegerRegisters() {
84 *numInts = numOfIntegerRegisters()
85 }
86
87 placeRegisters(v, addFloat, addInt)
88 } else {
89 keepAlive = placeStack(v, keepAlive, addInt)
90 }
91 return keepAlive // the struct was allocated so don't panic
92}
93
94func placeRegisters(v reflect.Value, addFloat func(uintptr), addInt func(uintptr)) {
95 if runtime.GOOS == "darwin" {
96 placeRegistersDarwin(v, addFloat, addInt)
97 return
98 }
99 placeRegistersArm64(v, addFloat, addInt)
100}
101
102func placeRegistersArm64(v reflect.Value, addFloat func(uintptr), addInt func(uintptr)) {
103 var val uint64
104 var shift byte
105 var flushed bool
106 class := _NO_CLASS
107 var place func(v reflect.Value)
108 place = func(v reflect.Value) {
109 var numFields int
110 if v.Kind() == reflect.Struct {
111 numFields = v.Type().NumField()
112 } else {
113 numFields = v.Type().Len()
114 }
115 for k := 0; k < numFields; k++ {
116 flushed = false
117 var f reflect.Value
118 if v.Kind() == reflect.Struct {
119 f = v.Field(k)
120 } else {
121 f = v.Index(k)
122 }
123 align := byte(f.Type().Align()*8 - 1)
124 shift = (shift + align) &^ align
125 if shift >= 64 {
126 shift = 0
127 flushed = true
128 if class == _FLOAT {
129 addFloat(uintptr(val))
130 } else {
131 addInt(uintptr(val))
132 }
133 val = 0
134 class = _NO_CLASS
135 }
136 switch f.Type().Kind() {
137 case reflect.Struct:
138 place(f)
139 case reflect.Bool:
140 if f.Bool() {
141 val |= 1 << shift
142 }
143 shift += 8
144 class |= _INT
145 case reflect.Uint8:
146 val |= f.Uint() << shift
147 shift += 8
148 class |= _INT
149 case reflect.Uint16:
150 val |= f.Uint() << shift
151 shift += 16
152 class |= _INT
153 case reflect.Uint32:
154 val |= f.Uint() << shift
155 shift += 32
156 class |= _INT
157 case reflect.Uint64, reflect.Uint, reflect.Uintptr:
158 addInt(uintptr(f.Uint()))
159 shift = 0
160 flushed = true
161 class = _NO_CLASS
162 case reflect.Int8:
163 val |= uint64(f.Int()&0xFF) << shift
164 shift += 8
165 class |= _INT
166 case reflect.Int16:
167 val |= uint64(f.Int()&0xFFFF) << shift
168 shift += 16
169 class |= _INT
170 case reflect.Int32:
171 val |= uint64(f.Int()&0xFFFF_FFFF) << shift
172 shift += 32
173 class |= _INT
174 case reflect.Int64, reflect.Int:
175 addInt(uintptr(f.Int()))
176 shift = 0
177 flushed = true
178 class = _NO_CLASS
179 case reflect.Float32:
180 if class == _FLOAT {
181 addFloat(uintptr(val))
182 val = 0
183 shift = 0
184 }
185 val |= uint64(math.Float32bits(float32(f.Float()))) << shift
186 shift += 32
187 class |= _FLOAT
188 case reflect.Float64:
189 addFloat(uintptr(math.Float64bits(float64(f.Float()))))
190 shift = 0
191 flushed = true
192 class = _NO_CLASS
193 case reflect.Ptr, reflect.UnsafePointer:
194 addInt(f.Pointer())
195 shift = 0
196 flushed = true
197 class = _NO_CLASS
198 case reflect.Array:
199 place(f)
200 default:
201 panic("purego: unsupported kind " + f.Kind().String())
202 }
203 }
204 }
205 place(v)
206 if !flushed {
207 if class == _FLOAT {
208 addFloat(uintptr(val))
209 } else {
210 addInt(uintptr(val))
211 }
212 }
213}
214
215func placeStack(v reflect.Value, keepAlive []any, addInt func(uintptr)) []any {
216 // Struct is too big to be placed in registers.
217 // Copy to heap and place the pointer in register
218 ptrStruct := reflect.New(v.Type())
219 ptrStruct.Elem().Set(v)
220 ptr := ptrStruct.Elem().Addr().UnsafePointer()
221 keepAlive = append(keepAlive, ptr)
222 addInt(uintptr(ptr))
223 return keepAlive
224}
225
226// isHFA reports a Homogeneous Floating-point Aggregate (HFA) which is a Fundamental Data Type that is a
227// Floating-Point type and at most four uniquely addressable members (5.9.5.1 in [Arm64 Calling Convention]).
228// This type of struct will be placed more compactly than the individual fields.
229//
230// [Arm64 Calling Convention]: https://github.com/ARM-software/abi-aa/blob/main/sysvabi64/sysvabi64.rst
231func isHFA(t reflect.Type) bool {
232 // round up struct size to nearest 8 see section B.4
233 structSize := roundUpTo8(t.Size())
234 if structSize == 0 || t.NumField() > 4 {
235 return false
236 }
237 first := t.Field(0)
238 switch first.Type.Kind() {
239 case reflect.Float32, reflect.Float64:
240 firstKind := first.Type.Kind()
241 for i := 0; i < t.NumField(); i++ {
242 if t.Field(i).Type.Kind() != firstKind {
243 return false
244 }
245 }
246 return true
247 case reflect.Array:
248 switch first.Type.Elem().Kind() {
249 case reflect.Float32, reflect.Float64:
250 return true
251 default:
252 return false
253 }
254 case reflect.Struct:
255 for i := 0; i < first.Type.NumField(); i++ {
256 if !isHFA(first.Type) {
257 return false
258 }
259 }
260 return true
261 default:
262 return false
263 }
264}
265
266// isHVA reports a Homogeneous Aggregate with a Fundamental Data Type that is a Short-Vector type
267// and at most four uniquely addressable members (5.9.5.2 in [Arm64 Calling Convention]).
268// A short vector is a machine type that is composed of repeated instances of one fundamental integral or
269// floating-point type. It may be 8 or 16 bytes in total size (5.4 in [Arm64 Calling Convention]).
270// This type of struct will be placed more compactly than the individual fields.
271//
272// [Arm64 Calling Convention]: https://github.com/ARM-software/abi-aa/blob/main/sysvabi64/sysvabi64.rst
273func isHVA(t reflect.Type) bool {
274 // round up struct size to nearest 8 see section B.4
275 structSize := roundUpTo8(t.Size())
276 if structSize == 0 || (structSize != 8 && structSize != 16) {
277 return false
278 }
279 first := t.Field(0)
280 switch first.Type.Kind() {
281 case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Int8, reflect.Int16, reflect.Int32:
282 firstKind := first.Type.Kind()
283 for i := 0; i < t.NumField(); i++ {
284 if t.Field(i).Type.Kind() != firstKind {
285 return false
286 }
287 }
288 return true
289 case reflect.Array:
290 switch first.Type.Elem().Kind() {
291 case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Int8, reflect.Int16, reflect.Int32:
292 return true
293 default:
294 return false
295 }
296 default:
297 return false
298 }
299}
300
301// copyStruct8ByteChunks copies struct memory in 8-byte chunks to the provided callback.
302// This is used for Darwin ARM64's byte-level packing of non-HFA/HVA structs.
303func copyStruct8ByteChunks(ptr unsafe.Pointer, size uintptr, addChunk func(uintptr)) {
304 if runtime.GOOS != "darwin" {
305 panic("purego: should only be called on darwin")
306 }
307 for offset := uintptr(0); offset < size; offset += 8 {
308 var chunk uintptr
309 remaining := size - offset
310 if remaining >= 8 {
311 chunk = *(*uintptr)(unsafe.Add(ptr, offset))
312 } else {
313 // Read byte-by-byte to avoid reading beyond allocation
314 for i := uintptr(0); i < remaining; i++ {
315 b := *(*byte)(unsafe.Add(ptr, offset+i))
316 chunk |= uintptr(b) << (i * 8)
317 }
318 }
319 addChunk(chunk)
320 }
321}
322
323// placeRegisters implements Darwin ARM64 calling convention for struct arguments.
324//
325// For HFA/HVA structs, each element must go in a separate register (or stack slot for elements
326// that don't fit in registers). We use placeRegistersArm64 for this.
327//
328// For non-HFA/HVA structs, Darwin uses byte-level packing. We copy the struct memory in
329// 8-byte chunks, which works correctly for both register and stack placement.
330func placeRegistersDarwin(v reflect.Value, addFloat func(uintptr), addInt func(uintptr)) {
331 if runtime.GOOS != "darwin" {
332 panic("purego: placeRegistersDarwin should only be called on darwin")
333 }
334 // Check if this is an HFA/HVA
335 hfa := isHFA(v.Type())
336 hva := isHVA(v.Type())
337
338 // For HFA/HVA structs, use the standard ARM64 logic which places each element separately
339 if hfa || hva {
340 placeRegistersArm64(v, addFloat, addInt)
341 return
342 }
343
344 // For non-HFA/HVA structs, use byte-level copying
345 // If the value is not addressable, create an addressable copy
346 if !v.CanAddr() {
347 addressable := reflect.New(v.Type()).Elem()
348 addressable.Set(v)
349 v = addressable
350 }
351 ptr := unsafe.Pointer(v.Addr().Pointer())
352 size := v.Type().Size()
353 copyStruct8ByteChunks(ptr, size, addInt)
354}
355
356// shouldBundleStackArgs determines if we need to start C-style packing for
357// Darwin ARM64 stack arguments. This happens when registers are exhausted.
358func shouldBundleStackArgs(v reflect.Value, numInts, numFloats int) bool {
359 if runtime.GOOS != "darwin" {
360 return false
361 }
362
363 kind := v.Kind()
364 isFloat := kind == reflect.Float32 || kind == reflect.Float64
365 isInt := !isFloat && kind != reflect.Struct
366 primitiveOnStack :=
367 (isInt && numInts >= numOfIntegerRegisters()) ||
368 (isFloat && numFloats >= numOfFloatRegisters())
369 if primitiveOnStack {
370 return true
371 }
372 if kind != reflect.Struct {
373 return false
374 }
375 hfa := isHFA(v.Type())
376 hva := isHVA(v.Type())
377 size := v.Type().Size()
378 eligible := hfa || hva || size <= 16
379 if !eligible {
380 return false
381 }
382
383 if hfa {
384 need := v.NumField()
385 return numFloats+need > numOfFloatRegisters()
386 }
387
388 if hva {
389 need := v.NumField()
390 return numInts+need > numOfIntegerRegisters()
391 }
392
393 slotsNeeded := int((size + align8ByteMask) / align8ByteSize)
394 return numInts+slotsNeeded > numOfIntegerRegisters()
395}
396
397// structFitsInRegisters determines if a struct can still fit in remaining
398// registers, used during stack argument bundling to decide if a struct
399// should go through normal register allocation or be bundled with stack args.
400func structFitsInRegisters(val reflect.Value, tempNumInts, tempNumFloats int) (bool, int, int) {
401 if runtime.GOOS != "darwin" {
402 panic("purego: structFitsInRegisters should only be called on darwin")
403 }
404 hfa := isHFA(val.Type())
405 hva := isHVA(val.Type())
406 size := val.Type().Size()
407
408 if hfa {
409 // HFA: check if elements fit in float registers
410 if tempNumFloats+val.NumField() <= numOfFloatRegisters() {
411 return true, tempNumInts, tempNumFloats + val.NumField()
412 }
413 } else if hva {
414 // HVA: check if elements fit in int registers
415 if tempNumInts+val.NumField() <= numOfIntegerRegisters() {
416 return true, tempNumInts + val.NumField(), tempNumFloats
417 }
418 } else if size <= 16 {
419 // Non-HFA/HVA small structs use int registers for byte-packing
420 slotsNeeded := int((size + align8ByteMask) / align8ByteSize)
421 if tempNumInts+slotsNeeded <= numOfIntegerRegisters() {
422 return true, tempNumInts + slotsNeeded, tempNumFloats
423 }
424 }
425
426 return false, tempNumInts, tempNumFloats
427}
428
429// collectStackArgs separates remaining arguments into those that fit in registers vs those that go on stack.
430// It returns the stack arguments and processes register arguments through addValue.
431func collectStackArgs(args []reflect.Value, startIdx int, numInts, numFloats int,
432 keepAlive []any, addInt, addFloat, addStack func(uintptr),
433 pNumInts, pNumFloats, pNumStack *int) ([]reflect.Value, []any) {
434 if runtime.GOOS != "darwin" {
435 panic("purego: collectStackArgs should only be called on darwin")
436 }
437
438 var stackArgs []reflect.Value
439 tempNumInts := numInts
440 tempNumFloats := numFloats
441
442 for j, val := range args[startIdx:] {
443 // Determine if this argument goes to register or stack
444 var fitsInRegister bool
445 var newNumInts, newNumFloats int
446
447 if val.Kind() == reflect.Struct {
448 // Check if struct still fits in remaining registers
449 fitsInRegister, newNumInts, newNumFloats = structFitsInRegisters(val, tempNumInts, tempNumFloats)
450 } else {
451 // Primitive argument
452 isFloat := val.Kind() == reflect.Float32 || val.Kind() == reflect.Float64
453 if isFloat {
454 fitsInRegister = tempNumFloats < numOfFloatRegisters()
455 newNumFloats = tempNumFloats + 1
456 newNumInts = tempNumInts
457 } else {
458 fitsInRegister = tempNumInts < numOfIntegerRegisters()
459 newNumInts = tempNumInts + 1
460 newNumFloats = tempNumFloats
461 }
462 }
463
464 if fitsInRegister {
465 // Process through normal register allocation
466 tempNumInts = newNumInts
467 tempNumFloats = newNumFloats
468 keepAlive = addValue(val, keepAlive, addInt, addFloat, addStack, pNumInts, pNumFloats, pNumStack)
469 } else {
470 // Convert strings to C strings before bundling
471 if val.Kind() == reflect.String {
472 ptr := strings.CString(val.String())
473 keepAlive = append(keepAlive, ptr)
474 val = reflect.ValueOf(ptr)
475 args[startIdx+j] = val
476 }
477 stackArgs = append(stackArgs, val)
478 }
479 }
480
481 return stackArgs, keepAlive
482}
483
484const (
485 paddingFieldPrefix = "Pad"
486)
487
488// bundleStackArgs bundles remaining arguments for Darwin ARM64 C-style stack packing.
489// It creates a packed struct with proper alignment and copies it to the stack in 8-byte chunks.
490func bundleStackArgs(stackArgs []reflect.Value, addStack func(uintptr)) {
491 if runtime.GOOS != "darwin" {
492 panic("purego: bundleStackArgs should only be called on darwin")
493 }
494 if len(stackArgs) == 0 {
495 return
496 }
497
498 // Build struct fields with proper C alignment and padding
499 var fields []reflect.StructField
500 currentOffset := uintptr(0)
501 fieldIndex := 0
502
503 for j, val := range stackArgs {
504 valSize := val.Type().Size()
505 valAlign := val.Type().Align()
506
507 // ARM64 requires 8-byte alignment for 8-byte or larger structs
508 if val.Kind() == reflect.Struct && valSize >= 8 {
509 valAlign = 8
510 }
511
512 // Add padding field if needed for alignment
513 if currentOffset%uintptr(valAlign) != 0 {
514 paddingNeeded := uintptr(valAlign) - (currentOffset % uintptr(valAlign))
515 fields = append(fields, reflect.StructField{
516 Name: paddingFieldPrefix + strconv.Itoa(fieldIndex),
517 Type: reflect.ArrayOf(int(paddingNeeded), reflect.TypeOf(byte(0))),
518 })
519 currentOffset += paddingNeeded
520 fieldIndex++
521 }
522
523 fields = append(fields, reflect.StructField{
524 Name: "X" + strconv.Itoa(j),
525 Type: val.Type(),
526 })
527 currentOffset += valSize
528 fieldIndex++
529 }
530
531 // Create and populate the packed struct
532 structType := reflect.StructOf(fields)
533 structInstance := reflect.New(structType).Elem()
534
535 // Set values (skip padding fields)
536 argIndex := 0
537 for j := 0; j < structInstance.NumField(); j++ {
538 fieldName := structType.Field(j).Name
539 if stdstrings.HasPrefix(fieldName, paddingFieldPrefix) {
540 continue
541 }
542 structInstance.Field(j).Set(stackArgs[argIndex])
543 argIndex++
544 }
545
546 ptr := unsafe.Pointer(structInstance.Addr().Pointer())
547 size := structType.Size()
548 copyStruct8ByteChunks(ptr, size, addStack)
549}