1package vulkan
2
3import (
4 "fmt"
5 "image"
6 "math"
7 "os"
8 "runtime"
9 "strconv"
10 "unsafe"
11
12 "github.com/bnema/neferwl/internal/adapters/hdrcursor"
13 "github.com/bnema/neferwl/internal/adapters/hintlabel"
14 "github.com/bnema/neferwl/internal/ports"
15 "github.com/bnema/zerowrap"
16
17 vk "github.com/bnema/purego-vulkan/vulkan"
18)
19
20// Renderer is owned by one goroutine; its methods must not be called concurrently.
21type Renderer struct {
22 width, height int
23 // queuePriority is the global priority the queue got: realtime, high
24 // or default.
25 queuePriority string
26 instance vk.Instance
27 id *vk.InstanceDispatch
28 device vk.Device
29 dd *vk.DeviceDispatch
30 queue vk.Queue
31 // own is the SDR composition image; HDR uses hdrOwn and PQ targets.
32 own target
33 hdrOwn target // linear BT.709 in units of SDR reference white
34 targets []*target
35 current int
36 renderMods []uint64
37 hdrMods []uint64
38 // importMods caches the modifiers a target of a format can be imported
39 // with (probeImportModifiers), by what the answer depends on.
40 importMods map[importKey][]uint64
41 hdrNits float64
42 hdr hdrPass
43 hdrError error
44 // modPlanes is the memory plane count of each probed target modifier
45 // (SDR and HDR): DCC modifiers have more than one.
46 modPlanes map[uint64]uint32
47 // log reports the chosen target modifier; the zero value discards.
48 log zerowrap.Logger
49 // traceDamage logs every frame's redrawn area (--debug=render-damage).
50 traceDamage bool
51 // hdrReadback makes exported HDR targets transfer sources, for GPU tests
52 // and headless --screenshot-raw (SetHDRReadback); never DRM.
53 hdrReadback bool
54 // virtual: the output has no display (headless), so ExportTargets
55 // accepts an empty modifier list for HDR (SetVirtualOutput).
56 virtual bool
57 physical vk.PhysicalDevice
58 // One converted cursor image is kept; changes replace it in bounded memory.
59 cursorCache hdrcursor.Cache
60 // lastHDRTarget is the index in targets of the exported PQ target the
61 // last HDR frame rendered into; -1 when the current targets hold none.
62 lastHDRTarget int
63 // last is the target of the last frame: what captures copy.
64 last *target
65 // captures are the readback slots (capture.go), allocated on the
66 // first capture; capturePass converts HDR frames into captureSDR.
67 captures []*captureSlot
68 capturePass hdrPass
69 captureSDR target
70 sync captureSync
71 pool vk.CommandPool
72 memory vk.PhysicalDeviceMemoryProperties
73 family uint32
74 // Frames in flight (frame.go): frame is the last frame recorded,
75 // submitted the last submitted, completed the last known finished.
76 slots [frameSlots]frameSlot
77 frame, submitted, completed uint64
78 deferred []deferredFree
79 // syncFD: frames export a sync file (else Render waits for them).
80 syncFD bool
81 // dmabuf is what clients may send; imports are their buffers by ID.
82 dmabuf ports.DMABufSupport
83 imports map[uint64]*imported
84 // pools are client wl_shm pools mapped by pool ID.
85 pools map[uint64]*mapping
86 // retired are pool mappings replaced this frame, unmapped once the
87 // frame's copies are done.
88 retired [][]byte
89 // shm are the GPU copies of wl_shm surfaces (shm.go); copied counts
90 // the bytes copied into them, for tests.
91 shm map[shmKey]*shmSurface
92 copied int
93 // labels rasterizes the hint labels (hint_labels.go); the zero value works.
94 labels hintlabel.Raster
95 // shmRects is scratch for shmCopyFor.
96 shmRects []image.Rectangle
97 // marks date the frames for Trim's wall-clock eviction (trim.go).
98 marks []trimMark
99 // redrawn counts the target pixels drawn (TakeRedrawn).
100 redrawn int
101 // fullRedraws counts why whole targets or windows were redrawn; each
102 // TakeRedrawn logs and resets it.
103 fullRedraws fullRedraws
104 // cursors are the exported cursor images (CursorBuffers).
105 cursors [2]*cursorImage
106 // compose draws every frame (compose.go); maxRange bounds one
107 // storage buffer binding.
108 compose composer
109 maxRange int
110 // Scratch belongs to the renderer goroutine; neither slice is published.
111 scratchDraws []draw
112 scratchCovers []image.Rectangle
113 damage damageRegion
114 damageSeen map[ports.WindowID]bool
115 damageDrawn map[ports.WindowID]heldWindow
116 frameDMAs []*imported
117 frameAcquires map[*imported]*os.File
118 frameWaits []vk.Semaphore
119 frameStages []vk.PipelineStageFlags
120 // call holds the structs a per-frame Vulkan call is handed by pointer.
121 call callScratch
122 // capCall holds the structs a capture hands Vulkan by pointer.
123 capCall captureScratch
124}
125
126// priorityAttempts falls back to an ordinary queue when elevated priority is unavailable.
127func priorityAttempts(supported bool) []vk.QueueGlobalPriority {
128 if supported {
129 return []vk.QueueGlobalPriority{vk.QueueGlobalPriorityRealtime, vk.QueueGlobalPriorityHigh, 0}
130 }
131 return []vk.QueueGlobalPriority{0}
132}
133
134func checked(name string, result vk.Result) error {
135 if err := vk.Check(result); err != nil {
136 return fmt.Errorf("%s: %w", name, err)
137 }
138 return nil
139}
140
141func New(width, height int) (r *Renderer, err error) {
142 if width <= 0 || height <= 0 || width > math.MaxUint32 || height > math.MaxUint32 || uint64(width)*uint64(height) > uint64(math.MaxInt/4) {
143 return nil, fmt.Errorf("invalid renderer dimensions %d x %d", width, height)
144 }
145 r = &Renderer{width: width, height: height, lastHDRTarget: -1, imports: map[uint64]*imported{}, pools: map[uint64]*mapping{}, shm: map[shmKey]*shmSurface{}}
146 defer func() {
147 if err != nil {
148 r.Close()
149 r = nil
150 }
151 }()
152 if err = vk.Init(); err != nil {
153 return nil, fmt.Errorf("vk.Init: %w", err)
154 }
155 app := vk.ApplicationInfo{SType: vk.StructureTypeApplicationInfo, ApiVersion: vk.MakeVersion(1, 3, 0)}
156 info := vk.InstanceCreateInfo{SType: vk.StructureTypeInstanceCreateInfo, ApplicationInfo: &app}
157 if err = checked("vkCreateInstance", vk.Global().CreateInstance(&info, nil, &r.instance)); err != nil {
158 return
159 }
160 r.id, err = vk.LoadInstanceDispatch(r.instance)
161 if err != nil {
162 err = fmt.Errorf("LoadInstanceDispatch: %w", err)
163 return
164 }
165 var count uint32
166 if err = checked("vkEnumeratePhysicalDevices(count)", r.id.EnumeratePhysicalDevices(r.instance, &count, nil)); err != nil {
167 return
168 }
169 if count == 0 {
170 err = fmt.Errorf("no Vulkan 1.3 device")
171 return
172 }
173 devices := make([]vk.PhysicalDevice, count)
174 if err = checked("vkEnumeratePhysicalDevices", r.id.EnumeratePhysicalDevices(r.instance, &count, &devices[0])); err != nil {
175 return
176 }
177 index := -1
178 if value, ok := os.LookupEnv("NEFERWL_VK_DEVICE"); ok {
179 index, err = strconv.Atoi(value)
180 if err != nil || index < 0 || index >= int(count) {
181 err = fmt.Errorf("invalid NEFERWL_VK_DEVICE index %q", value)
182 return
183 }
184 }
185 var physical vk.PhysicalDevice
186 var family uint32
187 for i, device := range devices[:count] {
188 if index >= 0 && i != index {
189 continue
190 }
191 var properties vk.PhysicalDeviceProperties
192 r.id.GetPhysicalDeviceProperties(device, &properties)
193 if properties.ApiVersion < vk.MakeVersion(1, 3, 0) {
194 continue
195 }
196 var n uint32
197 r.id.GetPhysicalDeviceQueueFamilyProperties(device, &n, nil)
198 if n == 0 {
199 continue
200 }
201 families := make([]vk.QueueFamilyProperties, n)
202 r.id.GetPhysicalDeviceQueueFamilyProperties(device, &n, &families[0])
203 for j, f := range families[:n] {
204 if f.QueueCount > 0 && f.QueueFlags&vk.QueueGraphicsBit != 0 {
205 physical = device
206 family = uint32(j)
207 break
208 }
209 }
210 if physical != 0 {
211 break
212 }
213 }
214 if physical == 0 {
215 err = fmt.Errorf("no Vulkan 1.3 device with a graphics queue")
216 return
217 }
218 r.id.GetPhysicalDeviceMemoryProperties(physical, &r.memory)
219 var properties vk.PhysicalDeviceProperties
220 r.id.GetPhysicalDeviceProperties(physical, &properties)
221 r.maxRange = int(properties.Limits.MaxStorageBufferRange)
222 r.family = family
223 priority := float32(1)
224 qi := vk.DeviceQueueCreateInfo{SType: vk.StructureTypeDeviceQueueCreateInfo, QueueFamilyIndex: family, QueueCount: 1, QueuePriorities: &priority}
225 dynamicRendering := vk.PhysicalDeviceDynamicRenderingFeatures{SType: vk.StructureTypePhysicalDeviceDynamicRenderingFeatures, DynamicRendering: 1}
226 indexing := descriptorIndexingFeatures{SType: descriptorIndexingFeaturesType}
227 features := vk.PhysicalDeviceFeatures2{SType: vk.StructureTypePhysicalDeviceFeatures2, Next: unsafe.Pointer(&indexing)}
228 r.id.GetPhysicalDeviceFeatures2(physical, &features)
229 r.compose.resources = 1
230 coreFeatures := vk.PhysicalDeviceFeatures{}
231 if features.Features.ShaderSampledImageArrayDynamicIndexing != 0 && features.Features.ShaderStorageBufferArrayDynamicIndexing != 0 && indexing.ShaderSampledImageArrayNonUniformIndexing != 0 && indexing.ShaderStorageBufferArrayNonUniformIndexing != 0 {
232 limit := composeResourceLimit(properties.Limits)
233 if limit > 1 {
234 r.compose.resources, r.compose.indexing = limit, true
235 coreFeatures.ShaderSampledImageArrayDynamicIndexing = 1
236 coreFeatures.ShaderStorageBufferArrayDynamicIndexing = 1
237 // Enable only the array indexing features used by these shaders.
238 indexing = descriptorIndexingFeatures{SType: descriptorIndexingFeaturesType, ShaderSampledImageArrayNonUniformIndexing: 1, ShaderStorageBufferArrayNonUniformIndexing: 1}
239 dynamicRendering.Next = unsafe.Pointer(&indexing)
240 }
241 }
242 di := vk.DeviceCreateInfo{SType: vk.StructureTypeDeviceCreateInfo, Next: unsafe.Pointer(&dynamicRendering), QueueCreateInfoCount: 1, QueueCreateInfos: &qi, EnabledFeatures: &coreFeatures}
243 // dmabuf import needs every extension; without them clients use wl_shm.
244 var extNames [][]byte
245 var extPtrs []*byte
246 have := r.extensions(physical)
247 dmabufSupported := true
248 for _, e := range deviceExtensions {
249 if !have[e] {
250 dmabufSupported = false
251 }
252 }
253 if dmabufSupported {
254 for _, e := range deviceExtensions {
255 extNames = append(extNames, append([]byte(e), 0))
256 extPtrs = append(extPtrs, &extNames[len(extNames)-1][0])
257 }
258 di.EnabledExtensionCount = uint32(len(extPtrs))
259 di.PpEnabledExtensionNames = &extPtrs[0]
260 }
261 globalExt := ""
262 if have[vk.KHRGlobalPriorityExtensionName] {
263 globalExt = vk.KHRGlobalPriorityExtensionName
264 } else if have[vk.EXTGlobalPriorityExtensionName] {
265 globalExt = vk.EXTGlobalPriorityExtensionName
266 }
267 var global vk.DeviceQueueGlobalPriorityCreateInfo
268 chosen := "default"
269 for _, attempt := range priorityAttempts(globalExt != "") {
270 qi.Next = nil
271 count := len(extPtrs)
272 if attempt != 0 {
273 global = vk.DeviceQueueGlobalPriorityCreateInfo{SType: vk.StructureTypeDeviceQueueGlobalPriorityCreateInfo, GlobalPriority: attempt}
274 qi.Next = unsafe.Pointer(&global)
275 extNames = append(extNames, append([]byte(globalExt), 0))
276 extPtrs = append(extPtrs, &extNames[len(extNames)-1][0])
277 chosen = "realtime"
278 if attempt == vk.QueueGlobalPriorityHigh {
279 chosen = "high"
280 }
281 } else {
282 chosen = "default"
283 }
284 di.EnabledExtensionCount = uint32(len(extPtrs))
285 if len(extPtrs) > 0 {
286 di.PpEnabledExtensionNames = &extPtrs[0]
287 } else {
288 di.PpEnabledExtensionNames = nil
289 }
290 result := r.id.CreateDevice(physical, &di, nil, &r.device)
291 runtime.KeepAlive(global)
292 runtime.KeepAlive(coreFeatures)
293 runtime.KeepAlive(indexing)
294 runtime.KeepAlive(dynamicRendering)
295 runtime.KeepAlive(qi)
296 runtime.KeepAlive(extNames)
297 runtime.KeepAlive(extPtrs)
298 extPtrs = extPtrs[:count]
299 extNames = extNames[:count]
300 if result == vk.Success {
301 err = nil
302 break
303 }
304 err = checked("vkCreateDevice", result)
305 if attempt == 0 {
306 return
307 }
308 }
309 r.queuePriority = chosen
310 r.dd, err = vk.LoadDeviceDispatch(r.id, r.device)
311 if err != nil {
312 err = fmt.Errorf("LoadDeviceDispatch: %w", err)
313 return
314 }
315 runtime.KeepAlive(extNames)
316 runtime.KeepAlive(extPtrs)
317 if dmabufSupported {
318 // Clients need the render node to allocate on the right GPU.
319 if sup := r.probeDMABuf(physical); sup.Device != 0 {
320 r.dmabuf = sup
321 r.probeRenderModifiers(physical)
322 r.physical = physical
323 }
324 // VK_KHR_external_semaphore_fd (enabled above): frames export
325 // their fence as a sync file.
326 r.syncFD = r.dd.HasGetSemaphoreFdKHR()
327 }
328 r.dd.GetDeviceQueue(r.device, family, 0, &r.queue)
329 extent := vk.Extent3D{Width: uint32(width), Height: uint32(height), Depth: 1}
330 ii := vk.ImageCreateInfo{SType: vk.StructureTypeImageCreateInfo, ImageType: vk.ImageType2d, Format: vk.FormatB8g8r8a8Unorm, Extent: extent, MipLevels: 1, ArrayLayers: 1, Samples: vk.SampleCount1Bit, Tiling: vk.ImageTilingOptimal, Usage: targetUsage | vk.ImageUsageSampledBit, SharingMode: vk.SharingModeExclusive, InitialLayout: vk.ImageLayoutUndefined}
331 if err = checked("vkCreateImage", r.dd.CreateImage(r.device, &ii, nil, &r.own.image)); err != nil {
332 return
333 }
334 var req vk.MemoryRequirements
335 r.dd.GetImageMemoryRequirements(r.device, r.own.image, &req)
336 var kind uint32
337 kind, err = r.findMemoryType(req.MemoryTypeBits, vk.MemoryPropertyDeviceLocalBit)
338 if err != nil {
339 return
340 }
341 alloc := vk.MemoryAllocateInfo{SType: vk.StructureTypeMemoryAllocateInfo, AllocationSize: req.Size, MemoryTypeIndex: kind}
342 if err = checked("vkAllocateMemory(image)", r.dd.AllocateMemory(r.device, &alloc, nil, &r.own.memory)); err != nil {
343 return
344 }
345 if err = checked("vkBindImageMemory", r.dd.BindImageMemory(r.device, r.own.image, r.own.memory, 0)); err != nil {
346 return
347 }
348 pi := vk.CommandPoolCreateInfo{SType: vk.StructureTypeCommandPoolCreateInfo, Flags: vk.CommandPoolCreateResetCommandBufferBit, QueueFamilyIndex: family}
349 if err = checked("vkCreateCommandPool", r.dd.CreateCommandPool(r.device, &pi, nil, &r.pool)); err != nil {
350 return
351 }
352 if err = r.createSlots(); err != nil {
353 return
354 }
355 if err = r.createView(&r.own); err != nil {
356 return
357 }
358 if err = r.createComposer(); err != nil {
359 return
360 }
361 runtime.KeepAlive(priority)
362 return
363}
364
365func (r *Renderer) findMemoryType(bits uint32, props vk.MemoryPropertyFlags) (uint32, error) {
366 for i := uint32(0); i < r.memory.MemoryTypeCount; i++ {
367 if bits&(uint32(1)<<i) != 0 && r.memory.MemoryTypes[i].PropertyFlags&props == props {
368 return i, nil
369 }
370 }
371 return 0, fmt.Errorf("no Vulkan memory type for bits %#x and properties %#x", bits, props)
372}
373
374// CopiedBytes reports cumulative wl_shm bytes copied into GPU storage.
375// It is diagnostic only and must be read by the renderer's owner goroutine.
376func (r *Renderer) CopiedBytes() int { return r.copied }
377
378// TakeRedrawn returns the target pixels drawn since the last call and
379// resets the count.
380func (r *Renderer) TakeRedrawn() int {
381 n := r.redrawn
382 r.redrawn = 0
383 if f := r.fullRedraws.take(); f != (fullRedraws{}) {
384 r.log.Debug().Int("target", f[fullTarget]).Int("scene", f[fullScene]).Int("new_window", f[fullNewWindow]).
385 Int("empty", f[fullEmpty]).Int("children", f[fullChildren]).Int("history", f[fullHistory]).Msg("full redraws")
386 }
387 return n
388}
389
390// SetLogger sets the logger (with its component field) for render target
391// events. Not on the port; the zero logger discards.
392func (r *Renderer) SetLogger(l zerowrap.Logger) { r.log = l }
393
394// SetTraceDamage logs the redrawn pixels, draw count and partial flag of
395// every frame at debug level (--debug=render-damage). Off by default.
396func (r *Renderer) SetTraceDamage(on bool) { r.traceDamage = on }
397
398// QueuePriority is the global priority the queue got: realtime, high or
399// default.
400func (r *Renderer) QueuePriority() string { return r.queuePriority }
401
402// Pixels reads the last frame back through a capture slot, waiting for
403// the GPU on the calling goroutine (slot fence, no sync file needed):
404// headless screenshots and tests only. Nil when no slot is free (both
405// leased to captures) or the copy failed: never a silent black image.
406func (r *Renderer) Pixels() *image.RGBA {
407 cf, err := r.debugCapture()
408 if err != nil {
409 return nil
410 }
411 defer r.EndCapture(cf)
412 out := image.NewRGBA(image.Rect(0, 0, r.width, r.height))
413 if err := cf.Read(image.Rect(0, 0, r.width, r.height), out.Pix, out.Stride); err != nil {
414 return nil
415 }
416 for i := 0; i < len(out.Pix); i += 4 {
417 out.Pix[i], out.Pix[i+2] = out.Pix[i+2], out.Pix[i]
418 }
419 return out
420}
421
422// SetHDRReadback makes the exported HDR targets transfer sources so that
423// HDRPixels can read them (they are not in production: the DRM path never
424// sets it). Call before ExportTargets. Not on the port.
425func (r *Renderer) SetHDRReadback(on bool) { r.hdrReadback = on }
426
427// HDRPixels reads the last HDR frame's XRGB2101010 target back, each 10-bit
428// code scaled to 16 bits (c<<6 | c>>4), alpha opaque. It waits for the GPU:
429// debug only. Nil when the output is SDR, the target is not readable (no
430// SetHDRReadback), no HDR frame was rendered, every slot is leased or the
431// copy failed.
432func (r *Renderer) HDRPixels() *image.RGBA64 {
433 var out *image.RGBA64
434 err := r.readHDRTarget(func(words []uint32) {
435 out = image.NewRGBA64(image.Rect(0, 0, r.width, r.height))
436 for i, w := range words {
437 p := out.Pix[i*8 : i*8+8]
438 for ch, c := range [3]uint32{w >> 20 & 1023, w >> 10 & 1023, w & 1023} {
439 v := uint16(c<<6 | c>>4)
440 p[ch*2], p[ch*2+1] = byte(v>>8), byte(v)
441 }
442 p[6], p[7] = 0xff, 0xff
443 }
444 })
445 if err != nil {
446 return nil
447 }
448 return out
449}
450
451// Missing from the bindings: VK_QUEUE_FAMILY_IGNORED, VK_WHOLE_SIZE.
452const (
453 queueFamilyIgnored = ^uint32(0)
454 wholeSize = ^vk.DeviceSize(0)
455)
456
457func (r *Renderer) Close() {
458 if r == nil {
459 return
460 }
461 if r.dd != nil {
462 d := r.dd
463 if r.device != 0 {
464 _ = checked("vkDeviceWaitIdle", d.DeviceWaitIdle(r.device))
465 r.idle()
466 }
467 r.dropTargets()
468 r.destroyHDR()
469 r.dropCursors()
470 for id, im := range r.imports {
471 r.release(im)
472 delete(r.imports, id)
473 }
474 r.frame += importTTL + 1
475 r.dropPools()
476 r.dropShm()
477 r.destroySlots()
478 r.destroyCaptures()
479 if r.pool != 0 {
480 d.DestroyCommandPool(r.device, r.pool, nil)
481 r.pool = 0
482 }
483 r.destroyComposer()
484 r.freeTarget(&r.hdrOwn)
485 r.freeTarget(&r.own)
486 d.DestroyDevice(r.device, nil)
487 r.device = 0
488 } else if r.device != 0 && r.id != nil {
489 if vk.VkDestroyDevice != nil {
490 vk.VkDestroyDevice(r.device, nil)
491 }
492 r.device = 0
493 }
494 if r.instance != 0 {
495 if r.id != nil {
496 r.id.DestroyInstance(r.instance, nil)
497 } else if vk.VkDestroyInstance != nil {
498 vk.VkDestroyInstance(r.instance, nil)
499 }
500 r.instance = 0
501 }
502}