1package drm
2
3import (
4 "context"
5 "errors"
6 "fmt"
7 "os"
8 "slices"
9 "time"
10 "unsafe"
11
12 "github.com/bnema/neferwl/internal/adapters/clock"
13 "github.com/bnema/neferwl/internal/adapters/outputkit/capture"
14 "github.com/bnema/neferwl/internal/adapters/outputkit/corewake"
15 "github.com/bnema/neferwl/internal/adapters/outputkit/presented"
16 "github.com/bnema/neferwl/internal/adapters/outputkit/surfaces"
17
18 "github.com/bnema/neferwl/internal/ports"
19 "github.com/bnema/zerowrap"
20 "golang.org/x/sys/unix"
21)
22
23// Output is one KMS connector on a CRTC, driven by atomic commits (ADR
24// 015). One goroutine runs it (Run) and closes it. Every commit that
25// changes the screen asks for an event and leaves the output pending until
26// it arrives: the next commit waits for it. The kernel may still refuse
27// it with EBUSY (it retries soon), and an event that never comes is
28// bounded (stuckTimeout, then a modeset).
29type Output struct {
30 // Set before Run. The backend owns instance registration and wake routing.
31 Security ports.SessionSecurity
32 SecurityChanges <-chan ports.SecurityState
33 Instance ports.OutputInstance
34 SecurityEvents chan<- ports.SecurityBackendEvent
35 // Frames, set before Run, tells core a flip of an animating scene
36 // completed (nil: nobody is told). Sends never block.
37 Frames chan<- ports.OutputFrame
38 securityState ports.SecurityState
39 securityPrepared bool
40 securityInvalid bool
41 inactiveOnClose bool // affirmative terminal KMS result, owned by Close
42 runContext context.Context // bounds compositor clear waits during Run startup
43 // kept reports a window drawn from a content its client withdrew (it
44 // closed and fades out; surfaces.Table.Kept): composed only, never on
45 // a plane, never reported shown. Set by Run; nil before.
46 kept func(ports.WindowID) bool
47
48 k kms
49 flipped <-chan flipEvent // commit events of this CRTC, from Card.ReadEvents
50 crtc uint32
51 conn connector
52 mode modeInfo
53 saved modeCrtc
54 monitor Monitor
55 hdr hdrState
56 log zerowrap.Logger
57 // renderLog is the render-component logger (see Card.renderLog).
58 renderLog zerowrap.Logger
59 // clock paces the periodic stats and renderer trim (nil: system).
60 clock ports.Clock
61 // Properties: CRTC and connector property IDs by name.
62 crtcProps map[string]uint32
63 connCrtc uint32 // the connector's CRTC_ID property
64 modeBlob uint32
65 // primary shows frames; cursor is the cursor plane (nil: none);
66 // stray are other planes that can show on this CRTC, turned off at
67 // every modeset when a previous master left them on it.
68 primary *plane
69 cursor *Cursor
70 stray []*plane
71 // overlay shows one window's buffer above the composed frame
72 // (overlay.go); overlayOn is the buffer on it in the pending or last
73 // frame (0: off), overlayReason why it is off.
74 overlay *plane
75 overlayOn uint64
76 overlayReason string
77 // ctmBlob is the 3x4 matrix blob of the planes' colour pipelines
78 // (BT.709 to BT.2020), made at setup and destroyed by Close (0: none);
79 // colorMult is the S31.32 multiplier, SDR white over 80 nits.
80 ctmBlob uint32
81 colorMult uint64
82 // seatDisable carries a request to leave the planes on Bypass before
83 // the seat is disabled (PrepareSeatDisable); Run answers it.
84 seatDisable chan chan struct{}
85 // ping carries watchdog probes (Ping); Run answers each one.
86 ping chan chan struct{}
87 // fbs are the two renderer images frames alternate between, back the
88 // one the next frame draws into (ADR 014: zero copy).
89 fbs [2]uint32
90 back int
91 frame frameLifecycle // pending commit, serials, deadlines and timer
92 flips int
93 // NewCaptureRenderer makes the child renderer of a hidden workspace
94 // capture session (Scene.CaptureScene). Nil: such captures fail closed.
95 // Set before Run.
96 NewCaptureRenderer func(w, h int) (ports.Renderer, error)
97 // StartOff, set before Run, keeps the display off until a scene turns
98 // it on: a display that reconnects while turned off never lights up.
99 StartOff bool
100 // capHidden limits a report to what the child renderer of a hidden
101 // workspace was given, window by window, while it may still read (its
102 // device is not ordered with the display's fences). capped is set when
103 // the last report had child reads: it is repeated until they finish.
104 capHidden func(map[ports.WindowID]uint64) (map[ports.WindowID]uint64, map[ports.WindowID]uint64, bool)
105 capped bool
106 // readFences are fences of frames rendered but not committed: the
107 // GPU may still read client buffers until they signal, so what was
108 // seen is reported only then (or after a later frame flipped).
109 readFences []*os.File
110 // Direct scanout: client framebuffers by DMABuf ID and the buffer on
111 // screen and queued (0: the composed image).
112 scanout bool
113 clientFBs map[uint64]*clientFB
114 shown, queued uint64
115 planeRect planeRect
116 reason string // why the last frame was composed ("" = scanout)
117 reports presented.Queue
118 // Last immutable shows snapshot sent in a flip. Never mutate it: wayland
119 // may still be reading a prior report on another goroutine.
120 shows surfaces.ShowsSnapshot
121 directShowsSnapshot map[ports.WindowID]uint64
122 // Tearing (ADR 006): async commits of scanned-out buffers that ask for
123 // them, when the card has DRM_CAP_ATOMIC_ASYNC_PAGE_FLIP. asyncFence
124 // is set when an async commit may carry IN_FENCE_FD (probed once).
125 tearing bool
126 asyncFence bool
127 asyncProbed bool
128 async bool
129 // VRR: the CRTC's VRR_ENABLED property (0: none), on while a buffer is
130 // scanned out or a fullscreen window covers the output, set inside
131 // frame commits. vrrGame is that state for the last frame.
132 vrrProp uint32
133 vrrOn bool
134 vrrGame bool
135 contentProp uint32
136 contentValues [5]uint64
137 contentValue uint64
138 contentWanted uint64
139 composedSince time.Time
140 // Reusable atomic requests, owned by the output goroutine (zero value
141 // ready). frameReq builds frame commits (commitWithRect), which retry
142 // by recursing: a retry resets it, so a caller never reads it after a
143 // recursive call. stateReq builds commitState. Paths that hold a second
144 // request while one is alive use their own: withoutReq (commitState's
145 // retry test) and probeReq (every TEST_ONLY probe: frameTest, the scale,
146 // overlay and colour probes).
147 frameReq, stateReq, withoutReq, probeReq atomicReq
148 // restWindows is overlayFrame's scene windows, valid until its next call.
149 restWindows []ports.SceneWindow
150 // front is what the output image on screen holds (front.go).
151 front frontImage
152 // overlayPassed are the last overlay probes KMS took (testOverlay):
153 // enough for a client cycling through four buffers.
154 overlayPassed [4]overlayProbe
155 overlayNext int
156 // lastFrame is when the last frame was committed. cursorHeld: a cursor
157 // move waits for the next frame (see cursorWaits).
158 lastFrame time.Time
159 cursorHeld bool
160 // stateAt is when a deferred state commit is due (zero: none);
161 // lastVblankAt is the kernel timestamp of the last vsynced flip, frame
162 // or state (CLOCK_MONOTONIC, 0: unknown). See state_schedule.go.
163 stateAt time.Time
164 lastVblankAt time.Duration
165 // traceFlips logs every completion with its timing; lastFlipAt is
166 // the previous frame flip's kernel timestamp (always kept).
167 // flipStats are reported and reset with the periodic stats. wantedAt
168 // is when the output first needed a frame since the last frame commit
169 // (0: none), taken by the commit into pendingFrame.
170 traceFlips bool
171 lastFlipAt time.Duration
172 flipStats flipStats
173 wantedAt time.Duration
174 // vrrFlipGap is the minimum time between a game frame's flip event
175 // and the next frame commit under VRR (render.vrr-flip-gap, 0: off);
176 // flipGapUntil is when the next frame may commit: the gap from the
177 // time the flip event was read, as the workaround was measured. See
178 // vrr_flip_gap.go.
179 vrrFlipGap time.Duration
180 flipGapUntil time.Time
181 // flipGapAt is the end of the gap from the flip's kernel timestamp
182 // (CLOCK_MONOTONIC), kept until the frame it held commits. It differs
183 // from flipGapUntil on purpose: the flip stats must not hide a late
184 // read of the event inside the gap.
185 flipGapAt time.Duration
186 // wantOff is the latest Scene.Off: a client turned the display off.
187 // off: the CRTC is inactive for it. Every modeset (resume, recovery)
188 // follows wantOff, so a display turned off never lights up.
189 wantOff, off bool
190 // protected is an output-owned latch. Ordinary modesets must not expose
191 // a previous desktop target while it is set; only protectedModeset may
192 // activate a target after clearing it and waiting for the GPU.
193 protected bool
194 // protectBackoff is the current retry delay after a refused protected
195 // commit; protectNotBefore the earliest next attempt (zero: none).
196 protectBackoff time.Duration
197 protectNotBefore time.Time
198 // kind is how the images were made.
199 kind imageKind
200 // planes is the memory plane count of the last exported images.
201 planes int
202 // imageMod is the modifier of the exported images (0: linear).
203 imageMod uint64
204 // formats receives the direct scanout formats after each modeset;
205 // sampled and device are what they are built from (Want).
206 formats chan<- ports.OutputFormats
207 sampled []ports.DMABufFormat
208 device uint64
209 ready chan error // first successful modeset, or a startup failure
210 readySent bool
211}
212
213// Ready reports the result of the first modeset exactly once.
214func (o *Output) Ready() <-chan error { return o.ready }
215
216// CursorLoader returns the image of a cursor at an output scale, already
217// rotated by the output transform t (the cursor plane is not rotated by the
218// hardware), at most limit pixels on a side; an empty image hides the cursor.
219type CursorLoader func(c ports.CursorChange, scale float64, t ports.BufferTransform, limit int) (ports.CursorImage, error)
220
221// newOutput reads the CRTC's planes and properties for a connector on crtc.
222func newOutput(card *Card, c connector, mode modeInfo, crtc uint32) (*Output, error) {
223 log := card.log
224 pipe := slices.Index(card.crtcs, crtc)
225 o := &Output{k: card.k, flipped: card.flips[crtc], frame: frameLifecycle{serials: &card.serials}, formats: card.formats, sampled: card.want.Sampled, device: card.want.Device, crtc: crtc, conn: c, mode: mode, log: log, renderLog: card.renderLog, monitor: readMonitor(card.path, c.name), scanout: !card.want.NoScanout, clientFBs: map[uint64]*clientFB{}, reason: "start", ready: make(chan error, 1), seatDisable: make(chan chan struct{}), ping: make(chan chan struct{}), traceFlips: card.want.TraceFlips, vrrFlipGap: card.want.VRRFlipGap}
226 var err error
227 if o.saved, err = getCrtc(card.fd, crtc); err != nil {
228 log.Warn().Err(err).Uint32("crtc", crtc).Msg("save crtc; it will not be restored on exit")
229 }
230 if err := o.readProps(card.fd, pipe, card.taken, cursorSize(card.fd)); err != nil {
231 return nil, err
232 }
233 o.hdr.settings = normalizedHDRSettings(card.want.HDR[c.name])
234 o.hdr.cap = detectHDR(o.monitor, o.hdr.props)
235 o.readColor()
236 o.setupColor()
237 log.Info().Str("connector", c.name).Bool("hdr_capable", o.hdr.cap.Capable).Str("reason", o.hdr.cap.Reason).Float64("max_luminance", o.hdr.cap.MaxLuminance).Float64("max_frame_average", o.hdr.cap.MaxFrameAverage).Float64("min_luminance", o.hdr.cap.MinLuminance).Msg("HDR capability")
238 if o.hdr.settings.Enabled && !o.hdr.cap.Capable {
239 log.Warn().Str("connector", c.name).Str("reason", o.hdr.cap.Reason).Msg("HDR requested but unavailable")
240 }
241 o.tearing = card.async && !card.want.NoTearing
242 if !card.want.NoVRR {
243 o.vrrProp = vrrProperty(o.k, c.id, crtc)
244 }
245 log.Info().Str("card", card.path).Str("connector", c.name).Str("mode", mode.String()).Str("make", o.monitor.Make).Str("model", o.monitor.Model).Uint32("crtc", crtc).Uint32("primary", o.primary.id).Bool("cursor", o.cursor != nil).Bool("tearing", o.tearing).Bool("vrr", o.vrrProp != 0).Msg("output")
246 return o, nil
247}
248
249// readProps finds the output's planes and property IDs. taken are planes
250// other outputs drive.
251func (o *Output) readProps(fd, pipe int, taken map[uint32]bool, cursorSide int) error {
252 cp, err := o.k.objProps(o.crtc, objCrtc)
253 if err != nil {
254 return fmt.Errorf("crtc properties: %w", err)
255 }
256 o.crtcProps = map[string]uint32{}
257 for name, v := range cp {
258 o.crtcProps[name] = uint32(v[0])
259 }
260 np, err := o.k.objProps(o.conn.id, objConnector)
261 if err != nil {
262 return fmt.Errorf("connector properties: %w", err)
263 }
264 o.connCrtc = uint32(np["CRTC_ID"][0])
265 // The connector's property metadata is needed for enum/range capability.
266 o.hdr.props = readConnectorHDRProps(fd, np)
267 o.contentProp, o.contentValues = readContentTypeProp(fd, np)
268 o.contentValue = np["content type"][1]
269 o.contentWanted = o.contentValue
270 if o.crtcProps["MODE_ID"] == 0 || o.crtcProps["ACTIVE"] == 0 || o.connCrtc == 0 {
271 return errors.New("crtc or connector lacks atomic properties")
272 }
273 planes, err := readPlanes(o.k, pipe)
274 if err != nil {
275 return fmt.Errorf("planes: %w", err)
276 }
277 pick := func(typ uint32) *plane {
278 var best *plane
279 for _, p := range planes {
280 if p.typ != typ || taken[p.id] {
281 continue
282 }
283 // Prefer the plane already on this CRTC.
284 if best == nil || p.crtc == o.crtc && best.crtc != o.crtc {
285 best = p
286 }
287 }
288 return best
289 }
290 if o.primary = pick(planePrimary); o.primary == nil {
291 return errors.New("no free primary plane")
292 }
293 if p := pick(planeCursor); p != nil {
294 o.cursor = newCursor(p, cursorSide)
295 }
296 for _, p := range planes {
297 if p != o.primary && (o.cursor == nil || p != o.cursor.plane) && !taken[p.id] {
298 o.stray = append(o.stray, p)
299 }
300 }
301 o.pickOverlay()
302 return nil
303}
304
305// owned are the planes this output drives.
306func (o *Output) owned() []*plane {
307 out := []*plane{o.primary}
308 if o.cursor != nil {
309 out = append(out, o.cursor.plane)
310 }
311 if o.overlay != nil {
312 out = append(out, o.overlay)
313 }
314 return out
315}
316
317// cursorSize is the cursor plane side: the smaller of the driver's width
318// and height caps, 64 when it does not say.
319func cursorSize(fd int) int {
320 size := 0
321 for _, c := range []uint64{capCursorW, capCursorH} {
322 v := getCap{capability: c}
323 if ioctl(fd, ioctlGetCap, unsafe.Pointer(&v)) == nil && v.value >= 16 && v.value <= 512 {
324 if size == 0 || int(v.value) < size {
325 size = int(v.value)
326 }
327 }
328 }
329 if size == 0 {
330 size = 64
331 }
332 return size
333}
334
335// Cursor is the hardware cursor, or nil when the CRTC has no cursor plane.
336func (o *Output) Cursor() *Cursor { return o.cursor }
337
338// Ping returns once Run handled a probe, or once ctx ends: the watchdog
339// uses it to detect a stuck output loop.
340func (o *Output) Ping(ctx context.Context) {
341 ack := make(chan struct{})
342 select {
343 case o.ping <- ack:
344 case <-ctx.Done():
345 return
346 }
347 select {
348 case <-ack:
349 case <-ctx.Done():
350 }
351}
352
353// Info describes the output for core and clients. Unknown EDID fields are empty.
354func (o *Output) Info() ports.OutputInfo {
355 known := func(v string) string {
356 if v == "Unknown" {
357 return ""
358 }
359 return v
360 }
361 return ports.OutputInfo{Name: o.conn.name, Make: known(o.monitor.Make), Model: known(o.monitor.Model), Serial: o.monitor.Serial, Width: o.Width(), Height: o.Height(), RefreshMilli: o.mode.refreshMilli(), PhysicalW: o.conn.mmW, PhysicalH: o.conn.mmH}
362}
363
364func (o *Output) Width() int { return int(o.mode.HDisplay) }
365func (o *Output) Height() int { return int(o.mode.VDisplay) }
366
367// primaryProps shows fb full screen on the primary plane.
368func (o *Output) primaryProps(req *atomicReq, fb uint32) {
369 o.primaryRectProps(req, fb, fullPlaneRect(o.Width(), o.Height()))
370}
371
372func (o *Output) primaryRectProps(req *atomicReq, fb uint32, rect planeRect) {
373 p := o.primary
374 req.set(p.id, p.prop("FB_ID"), uint64(fb))
375 req.set(p.id, p.prop("CRTC_ID"), uint64(o.crtc))
376 for i, name := range [...]string{"SRC_X", "SRC_Y", "SRC_W", "SRC_H"} {
377 req.set(p.id, p.prop(name), rect.src[i])
378 }
379 for i, name := range [...]string{"CRTC_X", "CRTC_Y", "CRTC_W", "CRTC_H"} {
380 req.set(p.id, p.prop(name), rect.crtc[i])
381 }
382}
383
384// modeset shows the front image with the output's mode, turning off every
385// other plane on the CRTC; needed at start and after every VT resume.
386// It blocks until the kernel applied it.
387func (o *Output) modeset() error {
388 o.observeSecurity()
389 return o.modesetImage(o.fbs[1-o.back], !o.wantOff && !o.protected)
390}
391
392// modesetImage is called with an active protected image only by
393// protectedModeset, after its clear fence has signalled.
394func (o *Output) modesetImage(fb uint32, active bool) error {
395 if o.protected && o.frame.pendingCommit() {
396 return errProtectionPending
397 }
398 if o.protected && !active {
399 if err := o.disableProtected(); err != nil {
400 return err
401 }
402 o.frame.resetAfterModeset()
403 o.forgetColor()
404 o.planeRect = fullPlaneRect(o.Width(), o.Height())
405 o.signalReady()
406 return nil
407 }
408 blob, err := o.modeBlobFor(o.mode)
409 if err != nil {
410 if o.protected {
411 return protectedCommitError{err}
412 }
413 return err
414 }
415 var req *atomicReq
416 if o.protected {
417 req = o.modesetBaseReq(blob, active)
418 if err := o.detachProtectedPlanes(req); err != nil {
419 _ = o.k.destroyBlob(blob)
420 return protectedCommitError{err}
421 }
422 if active {
423 o.primaryProps(req, fb)
424 }
425 } else {
426 req = o.modesetReq(blob, active)
427 }
428 // Do not admit a desktop-front modeset after an engage observed while
429 // constructing the request. Protected activation only contains black.
430 wasProtected := o.protected
431 o.observeSecurity()
432 if active && !wasProtected && o.protected {
433 _ = o.k.destroyBlob(blob)
434 return errSecurityScene
435 }
436 if err := o.k.commit(req, atomicAllowModes, 0); err != nil {
437 _ = o.k.destroyBlob(blob)
438 if o.protected {
439 return protectedCommitError{fmt.Errorf("modeset: %w", err)}
440 }
441 return fmt.Errorf("modeset: %w", err)
442 }
443 // Only now is nothing of ours pending or on screen: on failure the
444 // previous buffers may still show.
445 o.frame.resetAfterModeset()
446 // Plane scaling and colour pipeline capabilities may change after a
447 // modeset or VT resume; the planes' colour state is not assumed.
448 for _, fb := range o.clientFBs {
449 fb.scaleTestedOK, fb.scaleRefused = false, false
450 }
451 o.front.forget()
452 o.forgetOverlayProbes()
453 o.forgetColor()
454 o.forgetColorVerdicts()
455 o.shown, o.queued = 0, 0
456 if o.modeBlob != 0 {
457 _ = o.k.destroyBlob(o.modeBlob)
458 }
459 o.modeBlob = blob
460 o.vrrOn, o.vrrGame, o.overlayOn, o.off = false, false, 0, !active
461 // The gap and the flip interval belonged to the old state.
462 o.flipGapUntil, o.flipGapAt, o.lastFlipAt = time.Time{}, 0, 0
463 o.stateAt, o.lastVblankAt = time.Time{}, 0
464 o.contentValue, o.contentWanted = o.contentValues[0], o.contentValues[0]
465 o.planeRect = fullPlaneRect(o.Width(), o.Height())
466 if o.cursor != nil {
467 o.cursor.applied = cursorState{}
468 o.cursor.screen, o.cursor.flying = 0, false
469 }
470 o.log.Info().Str("connector", o.conn.name).Bool("off", o.off).Msg("modeset")
471 if !o.off && !o.protected {
472 // An inactive CRTC tells nothing about the cursor plane. Protected
473 // activation keeps it detached; do not test desktop cursor images.
474 o.testCursor()
475 }
476 o.sendFormats()
477 o.signalReady()
478 return nil
479}
480
481// signalReady reports the first successful modeset once.
482func (o *Output) signalReady() {
483 if o.ready != nil && !o.readySent {
484 o.ready <- nil
485 o.readySent = true
486 }
487}
488
489// powerOff turns the display off: the CRTC goes inactive and keeps its
490// mode, as DPMS off. The commit blocks, so no event is pending after it;
491// a modeset turns the display back on.
492func (o *Output) powerOff() error {
493 req := &atomicReq{}
494 req.set(o.crtc, o.crtcProps["ACTIVE"], 0)
495 req.set(o.crtc, o.vrrProp, 0)
496 req.set(o.conn.id, o.contentProp, o.contentValues[0])
497 o.forceBypass(req)
498 if err := o.k.commit(req, atomicAllowModes, 0); err != nil {
499 return fmt.Errorf("power off: %w", err)
500 }
501 o.colorBypassed()
502 o.off, o.vrrGame = true, false
503 // No vblank to aim at: the modeset that turns it on commits the state.
504 o.stateAt, o.lastVblankAt = time.Time{}, 0
505 o.setVRR(false)
506 o.contentValue, o.contentWanted = o.contentValues[0], o.contentValues[0]
507 o.log.Info().Str("connector", o.conn.name).Msg("power off")
508 // An inactive output offers neither HDR nor direct scanout.
509 o.sendFormats()
510 return nil
511}
512
513// sendFormats reports the formats clients may allocate for direct
514// scanout on this output (none when scanout is off). It never blocks:
515// wayland keeps the latest per output, a dropped report is resent at the
516// next modeset.
517func (o *Output) sendFormats() {
518 if o.formats == nil {
519 return
520 }
521 f := ports.OutputFormats{Output: o.conn.name, Device: o.device}
522 if o.hdr.on && !o.off {
523 f.HDR = &ports.OutputHDR{MaxLuminance: o.hdr.cap.MaxLuminance, MaxFrameAverage: o.hdr.cap.MaxFrameAverage, MinLuminance: o.hdr.cap.MinLuminance}
524 }
525 if o.scanout && !o.off {
526 for _, format := range o.scanoutFormats(o.sampled) {
527 // The feedback is not widened for SDR buffers on HDR: HDR
528 // clients pick from it and need 10-bit formats. An SDR
529 // buffer is still scanned out through the pipeline when the
530 // plane lists its format.
531 if !isYUVFormat(format.Format) && isTenBit(format.Format) == o.hdr.on {
532 f.Formats = append(f.Formats, format)
533 }
534 }
535 }
536 select {
537 case o.formats <- f:
538 default:
539 o.log.Warn().Str("connector", o.conn.name).Msg("scanout formats not sent: wayland busy")
540 }
541}
542
543// testModeset asks KMS whether it would take the modeset of the output
544// images without applying it.
545func (o *Output) testModeset() error {
546 blob, err := o.modeBlobFor(o.mode)
547 if err != nil {
548 return err
549 }
550 defer func() { _ = o.k.destroyBlob(blob) }()
551 // Tested active: an inactive CRTC would accept any image.
552 if err := o.k.commit(o.modesetReq(blob, true), atomicTestOnly|atomicAllowModes, 0); err != nil {
553 return fmt.Errorf("modeset test: %w", err)
554 }
555 return nil
556}
557
558func (o *Output) modeBlobFor(mode modeInfo) (uint32, error) {
559 blob, err := o.k.createBlob(unsafe.Slice((*byte)(unsafe.Pointer(&mode)), unsafe.Sizeof(mode)))
560 if err != nil {
561 return 0, fmt.Errorf("mode blob: %w", err)
562 }
563 return blob, nil
564}
565
566// testCursor turns the hardware cursor off when KMS refuses its images on
567// the cursor plane (e.g. a pitch the plane cannot scan out).
568func (o *Output) testCursor() {
569 c := o.cursor
570 if c == nil || c.off || c.fbs[0] == 0 {
571 return
572 }
573 req := &atomicReq{}
574 c.props(req, o.crtc, cursorState{on: true, fb: c.fbs[0]})
575 if err := o.k.commit(req, atomicTestOnly, 0); refused(err) {
576 o.cursorOff(err)
577 }
578}
579
580// cursorOff stops using the cursor plane for the rest of the output's life.
581func (o *Output) cursorOff(err error) {
582 o.cursor.off = true
583 o.log.Warn().Err(err).Str("connector", o.conn.name).Msg("cursor plane refused; hardware cursor off")
584}
585
586// modesetBaseReq sets the CRTC and connector without attaching any plane.
587func (o *Output) modesetBaseReq(blob uint32, active bool) *atomicReq {
588 req := &atomicReq{}
589 req.set(o.crtc, o.crtcProps["MODE_ID"], uint64(blob))
590 req.set(o.crtc, o.crtcProps["ACTIVE"], boolValue(active))
591 req.set(o.crtc, o.vrrProp, 0)
592 req.set(o.conn.id, o.contentProp, o.contentValues[0])
593 req.set(o.conn.id, o.connCrtc, uint64(o.crtc))
594 o.hdrConnectorProps(req, o.hdr.on)
595 // A modeset starts from the planes' Bypass pipeline: whatever the
596 // previous master left is not ours.
597 o.forceBypass(req)
598 return req
599}
600
601// modesetReq is the unlocked modeset of the front image with mode blob;
602// active false keeps the display off (output power).
603func (o *Output) modesetReq(blob uint32, active bool) *atomicReq {
604 req := o.modesetBaseReq(blob, active)
605 o.primaryProps(req, o.fbs[1-o.back])
606 if o.cursor != nil {
607 o.cursor.props(req, o.crtc, cursorState{})
608 }
609 o.overlayProps(req, overlayWin{})
610 for _, p := range o.stray {
611 // Only planes a previous master left on this CRTC.
612 props, err := o.k.objProps(p.id, objPlane)
613 if err != nil || uint32(props["CRTC_ID"][1]) != o.crtc {
614 continue
615 }
616 req.set(p.id, p.prop("FB_ID"), 0)
617 req.set(p.id, p.prop("CRTC_ID"), 0)
618 }
619 return req
620}
621
622// commitFrame flips fb in one commit with the cursor and VRR. An async
623// commit carries only the primary plane (kernel rule): a frame that must
624// also move the cursor or change VRR flips at vblank. fence is the
625// frame's GPU fence (nil: none); the caller keeps and closes it.
626func (o *Output) commitFrame(fb uint32, fence *os.File, async bool, vrr bool, f pendingFrame) error {
627 return o.commitWithRect(fb, fence, async, vrr, f, overlayWin{}, fullPlaneRect(o.Width(), o.Height()), colorUse{})
628}
629
630// commitWith is commitFrame with ov on the overlay plane (zero: off). The
631// primary plane shows a composed image: Bypass.
632func (o *Output) commitWith(fb uint32, fence *os.File, async bool, vrr bool, f pendingFrame, ov overlayWin) error {
633 return o.commitWithRect(fb, fence, async, vrr, f, ov, fullPlaneRect(o.Width(), o.Height()), colorUse{})
634}
635
636// commitWithRect commits fb on the primary plane at rect, with ov on the
637// overlay, and the planes' colour pipelines as pc (primary) and ov.color
638// (overlay) say. A frame that changes a plane's colour state is never async:
639// only FB_ID may change in an async commit.
640func (o *Output) commitWithRect(fb uint32, fence *os.File, async bool, vrr bool, f pendingFrame, ov overlayWin, rect planeRect, pc colorUse) error {
641 o.observeSecurity()
642 if o.Security != nil && (f.security != o.securityState || o.protected && !o.securityPrepared) {
643 return errSecurityScene
644 }
645 cur := cursorState{}
646 if o.cursor != nil && !o.protected {
647 cur = o.cursor.desired()
648 }
649 if o.protected {
650 async, vrr, ov, pc = false, false, overlayWin{}, colorUse{}
651 }
652 if async && (vrr != o.vrrOn || o.cursor != nil && cur != o.cursor.applied || fence != nil && !o.asyncFence || ov.buf != o.overlayOn || rect != o.planeRect || o.contentWanted != o.contentValue || o.colorStaleFor(pc, ov)) {
653 async = false
654 }
655 req := &o.frameReq
656 req.reset()
657 flags := uint32(atomicNonblock | flipEventFlag)
658 if async {
659 flags |= flipAsyncFlag
660 req.set(o.primary.id, o.primary.prop("FB_ID"), uint64(fb))
661 } else {
662 o.primaryRectProps(req, fb, rect)
663 o.primaryColorProps(req, pc.mode)
664 req.set(o.crtc, o.vrrProp, boolValue(vrr))
665 if o.cursor != nil {
666 o.cursor.props(req, o.crtc, cur)
667 }
668 if ov.buf != 0 || o.overlayOn != 0 {
669 o.overlayProps(req, ov)
670 } else {
671 o.overlayColorProps(req, ov.color.mode)
672 }
673 }
674 o.contentProps(req)
675 if fence != nil {
676 req.set(o.primary.id, o.primary.prop("IN_FENCE_FD"), uint64(fence.Fd()))
677 }
678 // Admission linearizes at this last defensive snapshot. An already
679 // admitted native ioctl can complete, but precedes the black proof.
680 o.observeSecurity()
681 if o.Security != nil && (f.security != o.securityState || o.protected && !o.securityPrepared) {
682 return errSecurityScene
683 }
684 // The kernel takes its own reference on the fence.
685 if err := o.k.commit(req, flags, o.frame.userData(userFrame)); err != nil {
686 // The pipeline first: with it in the frame, its refusal must not
687 // be taken for the cursor's, which would turn the hardware cursor
688 // off for good. colorRefused tests the frame with Bypass and the
689 // cursor as it is, so a real cursor refusal still reaches below.
690 if !async && errors.Is(err, unix.EINVAL) && o.colorRefused(err, fb, fence, vrr, cur, ov, rect, pc) {
691 return errColorRefused
692 }
693 if !async && cur.on && errors.Is(err, unix.EINVAL) && o.cursorRefused() {
694 return o.commitWithRect(fb, fence, false, vrr, f, ov, rect, pc)
695 }
696 // The same frame without the connector hint tells whether the hint
697 // is the cause.
698 if o.contentProp != 0 && o.contentWanted != o.contentValue && errors.Is(err, unix.EINVAL) && o.frameTest(fb, fence, vrr, cur, ov, rect, pc, false) {
699 o.log.Warn().Err(err).Msg("content type refused; disabled")
700 o.contentProp = 0
701 return o.commitWithRect(fb, fence, false, vrr, f, ov, rect, pc)
702 }
703 if !async && vrr && !o.vrrOn && errors.Is(err, unix.EINVAL) {
704 // Retry without turning VRR on: if that passes, the driver
705 // refuses VRR on this output. The overlay stays: the composed
706 // image left its window out.
707 if o.commitWithRect(fb, fence, false, false, f, ov, rect, pc) == nil {
708 o.log.Warn().Err(err).Str("connector", o.conn.name).Msg("vrr refused; disabled")
709 o.vrrProp = 0
710 return nil
711 }
712 }
713 return err
714 }
715 if !async {
716 if o.vrrOn != vrr {
717 o.log.Info().Bool("vrr", vrr).Str("connector", o.conn.name).Msg("vrr")
718 }
719 o.setVRR(vrr)
720 }
721 if o.cursor != nil {
722 // An async commit leaves the cursor plane as applied.
723 o.cursor.committed(cur)
724 }
725 if !async {
726 o.overlayOn = ov.buf
727 o.colorCommitted(pc, ov)
728 }
729 o.contentValue = o.contentWanted
730 o.planeRect = rect
731 o.lastFrame, o.cursorHeld = time.Now(), false
732 // The frame carries every deferred state change.
733 if !o.stateAt.IsZero() {
734 o.flipStats.stateCarried++
735 }
736 o.stateAt = time.Time{}
737 f.frame, f.async = true, async
738 f.wantedAt, o.wantedAt = o.wantedAt, 0
739 f.gapEnd, o.flipGapAt = o.flipGapAt, 0
740 f.fences = dupFences(fence, ov.acquire)
741 if o.traceFlips {
742 f.fenceReady = signalled(f.fences)
743 }
744 o.setAsync(async)
745 o.frame.begin(f, time.Now())
746 return nil
747}
748
749// frameTest is a TEST_ONLY of the frame commit as commitWithRect builds it,
750// with the colour state pc and ov.color, and the connector content hint only
751// when withContent. It reports whether KMS takes it. frameReq is still the
752// caller's: it uses probeReq.
753func (o *Output) frameTest(fb uint32, fence *os.File, vrr bool, cur cursorState, ov overlayWin, rect planeRect, pc colorUse, withContent bool) bool {
754 req := &o.probeReq
755 req.reset()
756 o.primaryRectProps(req, fb, rect)
757 o.primaryColorProps(req, pc.mode)
758 req.set(o.crtc, o.vrrProp, boolValue(vrr))
759 if o.cursor != nil {
760 o.cursor.props(req, o.crtc, cur)
761 }
762 if ov.buf != 0 || o.overlayOn != 0 {
763 o.overlayProps(req, ov)
764 } else {
765 o.overlayColorProps(req, ov.color.mode)
766 }
767 if withContent {
768 o.contentProps(req)
769 }
770 if fence != nil {
771 req.set(o.primary.id, o.primary.prop("IN_FENCE_FD"), uint64(fence.Fd()))
772 }
773 return o.k.commit(req, atomicTestOnly, 0) == nil
774}
775
776// cursorRefused reports, after a commit carrying the cursor failed with
777// EINVAL, whether the cursor plane was the cause; then the cursor is off
778// and the commit can be retried without it.
779func (o *Output) cursorRefused() bool {
780 o.testCursor()
781 return o.cursor.off
782}
783
784// commitState commits a cursor or VRR change without a new frame.
785func (o *Output) commitState(vrr bool) error {
786 o.observeSecurity()
787 if o.protected {
788 return errSecurityScene
789 }
790 if !o.stateChanged(vrr) {
791 // A change undone before its commit holds nothing.
792 o.stateAt, o.cursorHeld = time.Time{}, false
793 return nil
794 }
795 if !o.stateDue(time.Now(), monotonic()) {
796 return nil // a frame before then carries it; else the state timer
797 }
798 req := &o.stateReq
799 req.reset()
800 cur := cursorState{}
801 // A move back to the applied state holds nothing.
802 o.cursorHeld = false
803 if o.cursor != nil {
804 cur = o.cursor.desired()
805 if cur != o.cursor.applied {
806 if o.cursorWaits(time.Now()) {
807 o.cursorHeld = true
808 cur = o.cursor.applied
809 } else {
810 o.cursor.props(req, o.crtc, cur)
811 }
812 }
813 }
814 if vrr != o.vrrOn {
815 req.set(o.crtc, o.vrrProp, boolValue(vrr))
816 }
817 o.contentProps(req)
818 if len(req.objs) == 0 {
819 return nil
820 }
821 o.observeSecurity()
822 if o.protected {
823 return errSecurityScene
824 }
825 if err := o.k.commit(req, atomicNonblock|flipEventFlag, o.frame.userData(userState)); err != nil {
826 if o.colorConflict(err, o.cursor != nil && cur.on && cur != o.cursor.applied) {
827 // A plane shows the pipeline and KMS refuses the cursor with
828 // it: the next frame is composed.
829 return errOverlayDropped
830 }
831 if o.overlayConflict(err, o.overlayOn) {
832 // The next frame is composed without the overlay.
833 return errOverlayDropped
834 }
835 if cur.on && errors.Is(err, unix.EINVAL) && o.cursorRefused() {
836 return o.commitState(vrr)
837 }
838 if o.contentProp != 0 && o.contentWanted != o.contentValue && errors.Is(err, unix.EINVAL) {
839 without := &o.withoutReq
840 without.reset()
841 if o.cursor != nil && cur != o.cursor.applied {
842 o.cursor.props(without, o.crtc, cur)
843 }
844 if vrr != o.vrrOn {
845 without.set(o.crtc, o.vrrProp, boolValue(vrr))
846 }
847 if len(without.objs) == 0 || o.k.commit(without, atomicTestOnly, 0) == nil {
848 o.contentProp = 0
849 o.log.Warn().Err(err).Msg("content type refused; disabled")
850 return o.commitState(vrr)
851 }
852 }
853 return err
854 }
855 if o.cursor != nil {
856 o.cursor.committed(cur)
857 }
858 if vrr != o.vrrOn {
859 o.log.Info().Bool("vrr", vrr).Str("connector", o.conn.name).Msg("vrr")
860 }
861 o.setVRR(vrr)
862 o.contentValue = o.contentWanted
863 o.flipStats.stateCommits++
864 o.frame.begin(pendingFrame{}, time.Now())
865 return nil
866}
867
868// cursorMinInterval bounds how long a held cursor move waits for a frame.
869const cursorMinInterval = time.Second / 24
870
871// cursorWaits reports whether a cursor move rides on the next frame
872// instead of its own commit. Under VRR a cursor-only commit refreshes the
873// panel at its slowest rate, which delays the game's next frame by up to
874// one such refresh. A game that stops drawing still gets the cursor at
875// cursorMinInterval.
876func (o *Output) cursorWaits(now time.Time) bool {
877 return o.vrrOn && o.vrrGame && now.Sub(o.lastFrame) < cursorMinInterval
878}
879
880func boolValue(b bool) uint64 {
881 if b {
882 return 1
883 }
884 return 0
885}
886
887// commitScanout flips a client buffer. It reports false when KMS refused
888// the buffer: the frame must be composed.
889func (o *Output) commitScanout(fb uint32, c ports.SurfaceContent, f pendingFrame) (bool, error) {
890 return o.commitScanoutRect(fb, c, f, fullPlaneRect(o.Width(), o.Height()), colorBypass)
891}
892
893// commitScanoutRect flips a client buffer at rect, shown with colour mode.
894func (o *Output) commitScanoutRect(fb uint32, c ports.SurfaceContent, f pendingFrame, rect planeRect, mode colorMode) (bool, error) {
895 cfb := o.clientFBs[c.DMABuf.ID]
896 // Drivers refuse async flips that change the format or modifier: the
897 // flip from the composed image into scanout waits for vblank.
898 async := c.Async && o.tearing && o.shown != 0 && !cfb.noAsync
899 vrr := o.wantVRR(true)
900 f.queued, f.zeroCopy = c.DMABuf.ID, c.ID
901 // Explicit sync: KMS waits on the client's acquire fence. Wayland may
902 // close its file at any time: commit a duplicate taken while it is
903 // held open (the kernel keeps its own reference).
904 fence := dupFence(c.Acquire)
905 if fence != nil {
906 defer fence.Close()
907 }
908 pc := colorUse{mode: mode, format: c.DMABuf.Format}
909 err := o.commitWithRect(fb, fence, async, vrr, f, overlayWin{}, rect, pc)
910 if err != nil && async && errors.Is(err, unix.EINVAL) {
911 // Refused (e.g. not a fast update): this buffer flips at vblank
912 // from now on.
913 o.log.Debug().Err(err).Str("connector", o.conn.name).Msg("async flip refused")
914 cfb.noAsync = true
915 err = o.commitWithRect(fb, fence, false, vrr, f, overlayWin{}, rect, pc)
916 }
917 if errors.Is(err, errColorRefused) {
918 // KMS takes the buffer without the pipeline: compose it; the
919 // refusal is cached, the buffer is not marked failed.
920 o.setScanoutReason("color_refused")
921 return false, nil
922 }
923 if err != nil && (errors.Is(err, unix.EINVAL) || errors.Is(err, unix.ERANGE)) {
924 // KMS refuses this buffer on the plane: compose it instead.
925 o.renderLog.Info().Err(err).Uint32("format", c.DMABuf.Format).Uint64("modifier", c.DMABuf.Modifier).Msg("scanout flip refused")
926 cfb.failed = "flip_refused"
927 o.setScanoutReason("flip_refused")
928 return false, nil
929 }
930 if err == nil {
931 o.queued = c.DMABuf.ID
932 }
933 return true, err
934}
935
936// Close frees buffers and disables scanout. Without a security gate it
937// retains legacy restoration of the CRTC state found at startup; a wired
938// gate never admits saved desktop pixels, even if its snapshot is unlocked.
939func (o *Output) Close() {
940 o.inactiveOnClose = false
941 // Close may run after Run stopped, without receiving the engage wake.
942 o.observeSecurity()
943 req := &atomicReq{}
944 if o.cursor != nil {
945 o.cursor.props(req, o.crtc, cursorState{})
946 }
947 o.overlayProps(req, overlayWin{})
948 // Whoever drives the planes next starts from Bypass.
949 o.forceBypass(req)
950 o.hdrConnectorProps(req, false)
951 req.set(o.crtc, o.vrrProp, 0)
952 s := o.saved
953 var blob uint32
954 p := o.primary
955 restored := false
956 if o.Security == nil && !o.protected && s.crtcID != 0 && s.modeValid != 0 && s.fbID != 0 {
957 if b, err := o.modeBlobFor(s.mode); err == nil {
958 blob, restored = b, true
959 req.set(o.crtc, o.crtcProps["MODE_ID"], uint64(b))
960 req.set(o.crtc, o.crtcProps["ACTIVE"], 1)
961 req.set(o.conn.id, o.connCrtc, uint64(o.crtc))
962 req.set(p.id, p.prop("FB_ID"), uint64(s.fbID))
963 req.set(p.id, p.prop("CRTC_ID"), uint64(o.crtc))
964 req.set(p.id, p.prop("SRC_X"), 0)
965 req.set(p.id, p.prop("SRC_Y"), 0)
966 req.set(p.id, p.prop("SRC_W"), uint64(s.mode.HDisplay)<<16)
967 req.set(p.id, p.prop("SRC_H"), uint64(s.mode.VDisplay)<<16)
968 req.set(p.id, p.prop("CRTC_X"), 0)
969 req.set(p.id, p.prop("CRTC_Y"), 0)
970 req.set(p.id, p.prop("CRTC_W"), uint64(s.mode.HDisplay))
971 req.set(p.id, p.prop("CRTC_H"), uint64(s.mode.VDisplay))
972 }
973 }
974 if !restored {
975 // Nothing to give back: leave the CRTC off.
976 req.set(o.crtc, o.crtcProps["ACTIVE"], 0)
977 req.set(o.crtc, o.crtcProps["MODE_ID"], 0)
978 req.set(p.id, p.prop("FB_ID"), 0)
979 req.set(p.id, p.prop("CRTC_ID"), 0)
980 req.set(o.conn.id, o.connCrtc, 0)
981 }
982 // Only the validated detachment path can certify terminal inactivity.
983 // Legacy nil-gate restoration/disable remains best effort, not evidence.
984 detached := false
985 if o.protected || o.Security != nil {
986 if err := o.detachProtectedPlanes(req); err != nil {
987 o.log.Warn().Err(err).Str("connector", o.conn.name).Msg("protected close plane detachment")
988 } else {
989 detached = o.primary != nil && o.crtc != 0 && o.crtcProps["ACTIVE"] != 0
990 }
991 }
992 // A live metadata blob is freed only once an SDR modeset succeeds.
993 hdrShown := o.hdr.shown()
994 // Best effort: at exit the card may already belong to another session.
995 if err := o.k.commit(req, atomicAllowModes, 0); err != nil {
996 ev := o.log.Debug()
997 if hdrShown {
998 // The display may stay in HDR mode until the next modeset.
999 ev = o.log.Warn()
1000 }
1001 ev.Err(err).Str("connector", o.conn.name).Bool("hdr", hdrShown).Msg("restore crtc")
1002 } else {
1003 active, hasActive := req.value(o.crtc, o.crtcProps["ACTIVE"])
1004 o.inactiveOnClose = !restored && detached && hasActive && active == 0
1005 if hdrShown {
1006 o.log.Info().Str("connector", o.conn.name).Msg("HDR10 off")
1007 }
1008 }
1009 if blob != 0 {
1010 _ = o.k.destroyBlob(blob)
1011 }
1012 if o.modeBlob != 0 {
1013 _ = o.k.destroyBlob(o.modeBlob)
1014 o.modeBlob = 0
1015 }
1016 o.hdr.releaseBlob(o.k)
1017 if o.ctmBlob != 0 {
1018 _ = o.k.destroyBlob(o.ctmBlob)
1019 o.ctmBlob = 0
1020 }
1021 if o.cursor != nil {
1022 o.cursor.free(o.k)
1023 }
1024 o.freeImages()
1025 o.frame.endPending()
1026 o.dropRead()
1027 o.shown, o.queued = 0, 0
1028 o.dropClientFBs(time.Now(), true)
1029}
1030
1031// InactiveOnClose reports affirmative terminal inactivity, not cached power
1032// state. Call only after Close's owner has finished and a done channel (or
1033// equivalent owner-completion synchronization) has been observed. It is not
1034// safe to read concurrently with the output owner; no shared mutation is added.
1035func (o *Output) InactiveOnClose() bool { return o.inactiveOnClose }
1036
1037// Run renders scenes and commits them until ctx ends. active reports seat
1038// enable/disable. What the output shows and read is pushed to reports.
1039func (o *Output) Run(ctx context.Context, newRenderer func(w, h int) (ports.Renderer, error), loadCursor CursorLoader, active <-chan bool, scenes <-chan ports.Scene, contents <-chan *ports.ContentBatch, returns ports.ContentReturn, cursor <-chan ports.CursorChange, reports ports.PresentationSink, captures <-chan ports.CaptureRequest, captured chan<- ports.CaptureDone) (runErr error) {
1040 defer func() {
1041 p := recover()
1042 if p != nil {
1043 runErr = fmt.Errorf("output panic: %v", p)
1044 }
1045 if o.ready != nil && !o.readySent {
1046 o.ready <- runErr
1047 o.readySent = true
1048 }
1049 if p != nil {
1050 panic(p)
1051 }
1052 }()
1053 o.runContext = ctx
1054 o.wantOff = o.StartOff
1055 o.observeSecurity()
1056 r, err := newRenderer(o.Width(), o.Height())
1057 if err != nil {
1058 return fmt.Errorf("create renderer: %w", err)
1059 }
1060 defer r.Close()
1061 if o.cursor != nil {
1062 if err := o.cursor.setup(o.k, r); err != nil {
1063 o.cursor.off = true
1064 o.log.Warn().Err(err).Str("connector", o.conn.name).Msg("no hardware cursor images; hardware cursor off")
1065 }
1066 }
1067 // The seat sends its state first; while switched away (e.g. a monitor
1068 // plugged in on another VT) the modeset waits for the enable.
1069 enabled := true
1070 select {
1071 case enabled = <-active:
1072 default:
1073 }
1074 // Output images, from the best to the linear fallback (ADR 014).
1075 // While switched away they are made on the first enable.
1076 if enabled {
1077 if err := o.showImages(r, imagesDriver, nil); err != nil {
1078 // While protected, a recoverable KMS failure keeps the output
1079 // registered and dark; the loop retries with bounded backoff.
1080 if !o.protected || !o.commitFailed(err, &enabled) {
1081 return err
1082 }
1083 }
1084 if !o.protected {
1085 o.probeAsync(r)
1086 }
1087 }
1088 table := surfaces.New()
1089 drawn := table.Map()
1090 o.kept = table.Kept
1091 var scene ports.Scene
1092 haveScene, dirty := false, false
1093 var want ports.CursorChange
1094 cursorScale := -1.0 // not loaded yet
1095 var cursorTransform ports.BufferTransform
1096 frame := 0
1097 var requests capture.Requests
1098 ctx, cancelCaptures := context.WithCancel(ctx)
1099 requests.Init(ctx, captured)
1100 pipeline := capture.NewPipeline(ctx, captured)
1101 pipeline.Security = o.Security
1102 if o.NewCaptureRenderer != nil {
1103 pipeline.EnableOffscreen(o.NewCaptureRenderer)
1104 o.capHidden = pipeline.CapHiddenSeen
1105 }
1106 defer func() {
1107 cancelCaptures()
1108 pipeline.Close(r)
1109 requests.Stop(captures)
1110 }()
1111 clk := o.clock
1112 if clk == nil {
1113 clk = clock.System{}
1114 }
1115 stats := clk.NewTicker(10 * time.Second)
1116 defer stats.Stop()
1117 // Timer channels are enabled only when their condition holds. Go 1.27
1118 // guarantees a stopped/reset timer cannot deliver its old value.
1119 retryTimer := time.NewTimer(time.Hour)
1120 vrrTimer := time.NewTimer(time.Hour)
1121 cursorTimer := time.NewTimer(time.Hour)
1122 stateTimer := time.NewTimer(time.Hour)
1123 stateTimer.Stop()
1124 defer stateTimer.Stop()
1125 protectTimer := time.NewTimer(time.Hour)
1126 o.frame.startTimer()
1127 gapTimer := time.NewTimer(time.Hour)
1128 gapTimer.Stop()
1129 defer gapTimer.Stop()
1130 // A request waits for a scene that shows its capture indicator, at most
1131 // capture.HoldFor (see capture.Hold).
1132 holdTimer := time.NewTimer(time.Hour)
1133 holdTimer.Stop()
1134 defer holdTimer.Stop()
1135 retryTimer.Stop()
1136 vrrTimer.Stop()
1137 cursorTimer.Stop()
1138 protectTimer.Stop()
1139 defer protectTimer.Stop()
1140 defer retryTimer.Stop()
1141 defer vrrTimer.Stop()
1142 defer cursorTimer.Stop()
1143 defer o.frame.stopTimer()
1144 // seen is the latest content Seq per window. It is reported only while
1145 // no frame is in flight: then the GPU has finished every frame that
1146 // read older buffers (the kernel waited for their fence to flip).
1147 seen := map[ports.WindowID]uint64{}
1148 reportDirty := false
1149 var cursorWake <-chan struct{}
1150 if o.cursor != nil {
1151 cursorWake = o.cursor.wake
1152 }
1153 securityChanges := o.SecurityChanges
1154 reacquireTimer := time.NewTimer(time.Hour)
1155 reacquireTimer.Stop()
1156 defer reacquireTimer.Stop()
1157 var reacquire masterRetry
1158 // refuse records an enable refused for lack of DRM master.
1159 refuseMaster := func(err error) {
1160 if reacquire.refused(err, enabled, time.Now()) {
1161 o.log.Warn().Err(err).Str("connector", o.conn.name).Int("attempts", masterRetryLimit).Msg("DRM master not back; output stays off until the next seat change")
1162 }
1163 }
1164 // setActive applies a seat change. skip reports that the loop must
1165 // go on without touching the frame state (refused or protected).
1166 setActive := func(on bool) (skip bool, err error) {
1167 o.observeSecurity()
1168 if o.protected {
1169 o.invalidateSecurity()
1170 enabled = on
1171 if on && o.fbs[0] == 0 {
1172 if err := o.showImages(r, imagesDriver, nil); err != nil {
1173 if !o.commitFailed(err, &enabled) {
1174 return true, err
1175 }
1176 refuseMaster(err)
1177 return true, nil
1178 }
1179 }
1180 // With images, the protected modeset runs at the top of the
1181 // loop (prepareSecurity): it ends or reschedules the retry.
1182 return true, nil
1183 }
1184 // Modeset on every enable: a fast disable+enable can coalesce to one true.
1185 if on {
1186 enabled = true // this enable may supersede a previous master loss
1187 var err error
1188 if o.fbs[0] == 0 {
1189 err = o.showImages(r, imagesDriver, nil)
1190 } else {
1191 err = o.modeset()
1192 }
1193 if err != nil {
1194 if !o.commitFailed(err, &enabled) {
1195 return true, err
1196 }
1197 refuseMaster(err)
1198 return true, nil // epoch retry or wait disabled for master reacquisition
1199 }
1200 reacquire.done()
1201 o.observeSecurity()
1202 if o.protected {
1203 return true, nil
1204 }
1205 o.probeAsync(r)
1206 dirty = haveScene
1207 }
1208 enabled = on
1209 o.log.Info().Bool("enabled", on).Msg("output")
1210 return false, nil
1211 }
1212 for {
1213 if o.observeSecurity() || haveScene && !o.sceneCurrent(scene) {
1214 scene, haveScene, dirty = ports.Scene{}, false, false
1215 o.wantedAt = 0
1216 want, cursorScale, cursorTransform = ports.CursorChange{}, -1, 0
1217 if o.cursor != nil {
1218 o.cursor.image, o.cursor.later = false, nil
1219 }
1220 }
1221 if o.protected && requests.Len() > 0 {
1222 requests.FailPending(capture.ErrSessionProtected)
1223 }
1224 if o.protected && enabled && !o.frame.pendingCommit() && !o.securityPrepared && !o.protectBackoffActive() {
1225 if err := o.prepareSecurity(ctx, r); err != nil {
1226 if ctx.Err() != nil {
1227 return nil
1228 }
1229 if !o.commitFailed(err, &enabled) {
1230 return err
1231 }
1232 // A lock screen refused for lack of DRM master after a VT
1233 // switch is retried like an ordinary enable.
1234 refuseMaster(err)
1235 } else {
1236 reacquire.done()
1237 }
1238 // Recheck immediately: a takeover during the clear needs its own
1239 // transaction, not another scene or wake to make progress.
1240 if enabled && !o.frame.pendingCommit() && !o.securityPrepared && !o.protectBackoffActive() {
1241 continue
1242 }
1243 }
1244 if o.securityInvalid {
1245 if err := o.sendSecurityInvalid(ctx); err != nil {
1246 return nil
1247 }
1248 }
1249 readWait := false
1250 if reportDirty || o.capped {
1251 published := false
1252 if !o.frame.pendingCommit() {
1253 if readWait = !o.readDone(); !readWait {
1254 o.report(nil, seen)
1255 reportDirty, published = false, true
1256 // A report limited by the child's reads is repeated soon.
1257 readWait = o.capped
1258 }
1259 }
1260 if !published && o.capped {
1261 // A display frame or fence that has not finished must not
1262 // keep a finished child read held: child holds do not expire.
1263 // Report them alone, at the last Seen that was safe.
1264 o.report(nil, o.reports.Seen())
1265 readWait = readWait || o.capped
1266 }
1267 }
1268 o.reports.Flush(reports)
1269 // A report that waits for an uncommitted frame's fence is retried
1270 // soon, not only on the next event.
1271 var retry <-chan time.Time
1272 if readWait {
1273 retryTimer.Reset(time.Millisecond)
1274 retry = retryTimer.C
1275 } else {
1276 retryTimer.Stop()
1277 }
1278 // A composed screen that stopped changing still drops VRR on time.
1279 var vrrOff <-chan time.Time
1280 if enabled && o.vrrOn && !o.frame.pendingCommit() && !o.composedSince.IsZero() {
1281 vrrTimer.Reset(max(0, vrrHold-time.Since(o.composedSince)))
1282 vrrOff = vrrTimer.C
1283 } else {
1284 vrrTimer.Stop()
1285 }
1286 // A held cursor move commits alone if no frame comes in time.
1287 var cursorDue <-chan time.Time
1288 if enabled && !o.protected && o.cursorHeld && !o.frame.pendingCommit() {
1289 cursorTimer.Reset(max(0, cursorMinInterval-time.Since(o.lastFrame)))
1290 cursorDue = cursorTimer.C
1291 } else {
1292 cursorTimer.Stop()
1293 }
1294 // A deferred state change commits alone if no frame carried it.
1295 var stateDue <-chan time.Time
1296 if enabled && !o.protected && !o.off && !o.stateAt.IsZero() && !o.frame.pendingCommit() {
1297 stateTimer.Reset(max(0, time.Until(o.stateAt)))
1298 stateDue = stateTimer.C
1299 } else {
1300 stateTimer.Stop()
1301 }
1302 // A game frame waiting out the VRR flip gap commits when it ends.
1303 var gapDue <-chan time.Time
1304 if enabled && dirty && !o.frame.pendingCommit() && !o.flipGapUntil.IsZero() {
1305 gapTimer.Reset(max(0, time.Until(o.flipGapUntil)))
1306 gapDue = gapTimer.C
1307 } else {
1308 gapTimer.Stop()
1309 }
1310 var holdDue <-chan time.Time
1311 if requests.Len() > 0 {
1312 var wait time.Duration
1313 requests.List, wait = pipeline.Expire(scene, requests.List, time.Now())
1314 if wait > 0 {
1315 holdTimer.Reset(wait)
1316 holdDue = holdTimer.C
1317 }
1318 }
1319 if holdDue == nil {
1320 holdTimer.Stop()
1321 }
1322 // A refused protected commit is retried after a bounded backoff.
1323 var protectDue <-chan time.Time
1324 if o.protected && enabled && !o.securityPrepared && o.protectBackoffActive() {
1325 protectTimer.Reset(max(0, time.Until(o.protectNotBefore)))
1326 protectDue = protectTimer.C
1327 } else {
1328 protectTimer.Stop()
1329 }
1330 // An enable refused for lack of DRM master is retried on time.
1331 var reacquireDue <-chan time.Time
1332 if d, ok := reacquire.next(time.Now()); ok {
1333 reacquireTimer.Reset(d)
1334 reacquireDue = reacquireTimer.C
1335 } else {
1336 reacquireTimer.Stop()
1337 }
1338 // A commit whose event never comes must not stop the output.
1339 stuck := o.frame.wait(enabled)
1340 stateDirty := false
1341 select {
1342 case _, ok := <-securityChanges:
1343 if !ok {
1344 securityChanges = nil
1345 }
1346 continue // gate is authoritative, not the queued wake payload
1347 case <-retry:
1348 continue
1349 case <-protectDue:
1350 continue
1351 case <-stuck:
1352 o.observeSecurity()
1353 if !o.frame.deadlineReached(time.Now()) {
1354 continue // waits for a fence
1355 }
1356 if o.expire() {
1357 if o.protected {
1358 o.invalidateSecurity()
1359 continue // fresh black transaction at the top of the loop
1360 }
1361 if err := o.modeset(); err != nil {
1362 if !o.commitFailed(err, &enabled) {
1363 return fmt.Errorf("recovery modeset: %w", err)
1364 }
1365 continue
1366 }
1367 }
1368 dirty = haveScene
1369 stateDirty = true
1370 case <-vrrOff:
1371 stateDirty = true
1372 case <-cursorDue:
1373 stateDirty = true
1374 case <-stateDue:
1375 stateDirty = true
1376 case <-cursorWake:
1377 stateDirty = true
1378 case <-gapDue:
1379 case <-holdDue:
1380 continue
1381 case <-ctx.Done():
1382 return nil
1383 case b := <-pipeline.Completed():
1384 pipeline.Recycle(b, r)
1385 continue
1386 case b := <-pipeline.HiddenCompleted():
1387 pipeline.RecycleHidden(b)
1388 pipeline.Retire(scene)
1389 continue
1390 case q := <-captures:
1391 o.observeSecurity()
1392 var refuse error
1393 if o.protected || !enabled || o.off || o.wantOff {
1394 refuse = capture.ErrOutputOff
1395 }
1396 dirty = requests.Admit(q, refuse, scene, haveScene) || dirty
1397 case ack := <-o.ping:
1398 close(ack)
1399 continue
1400 case ack := <-o.seatDisable:
1401 o.bypassForSeatDisable()
1402 // The seat is about to be disabled: no frame may select the
1403 // pipeline again before the next enable (a modeset), not even a
1404 // retried one.
1405 reacquire.done()
1406 enabled = false
1407 ack <- struct{}{}
1408 continue
1409 case on := <-active:
1410 reacquire.done()
1411 skip, err := setActive(on)
1412 if err != nil {
1413 return err
1414 }
1415 if skip {
1416 continue
1417 }
1418 stateDirty = on
1419 case <-reacquireDue:
1420 // The seat reported active before the kernel gave DRM master
1421 // back: the enable modeset was refused. Try it again.
1422 reacquire.fire()
1423 skip, err := setActive(true)
1424 if err != nil {
1425 return err
1426 }
1427 if skip {
1428 continue
1429 }
1430 stateDirty = true
1431 case ev := <-o.flipped:
1432 o.observeSecurity()
1433 if !o.completed(ev, seen) {
1434 continue // stale event from before a modeset
1435 }
1436 stateDirty = true
1437 if o.cursor != nil && !o.protected {
1438 // An image that waited for a free slot loads now.
1439 if _, err := o.cursor.flushLater(r); err != nil {
1440 o.log.Warn().Err(err).Msg("cursor")
1441 }
1442 }
1443 case s := <-scenes:
1444 o.observeSecurity()
1445 if !o.sceneCurrent(s) {
1446 continue
1447 }
1448 // The core keeps the Seq of a scene that draws the same: nothing to do.
1449 if haveScene && s.Seq != 0 && s.Seq == scene.Seq && s.Security == scene.Security {
1450 continue
1451 }
1452 if o.protected {
1453 s.CaptureScene = nil
1454 if o.wantOff != s.Off {
1455 o.invalidateSecurity()
1456 }
1457 }
1458 if !o.protected && o.cursor != nil && loadCursor != nil && (s.Scale != cursorScale || s.Transform != cursorTransform) {
1459 cursorScale, cursorTransform = s.Scale, s.Transform
1460 o.setCursor(r, loadCursor, want, s.Scale, s.Transform)
1461 stateDirty = true
1462 }
1463 scene, haveScene, dirty = s, true, true
1464 o.wantOff = s.Off
1465 // A closed window's last content goes once no scene draws it.
1466 table.Prune(scene)
1467 case c := <-cursor:
1468 o.observeSecurity()
1469 if o.protected {
1470 continue
1471 }
1472 want = c
1473 // Before the first scene the scale is unknown: loaded then.
1474 if o.cursor != nil && loadCursor != nil && cursorScale > 0 {
1475 o.setCursor(r, loadCursor, want, cursorScale, cursorTransform)
1476 stateDirty = true
1477 }
1478 case b := <-contents:
1479 // Windows on other outputs or workspaces do not need a frame.
1480 batchDirty, batchSeen := table.Apply(b, scene, seen, returns)
1481 dirty = dirty || batchDirty
1482 reportDirty = reportDirty || batchSeen
1483 case <-stats.C():
1484 now := clk.Now()
1485 pipeline.Retire(scene) // an idle child of an ended session goes
1486 o.dropClientFBs(now, false)
1487 // An idle output renders nothing: free what windows that left
1488 // it held.
1489 if err := r.Trim(now); err != nil {
1490 return fmt.Errorf("trim renderer: %w", err)
1491 }
1492 ev := o.log.Info().Str("connector", o.conn.name).Int("frames", frame).Int("flips", o.flips).Bool("pending", o.frame.pendingCommit())
1493 if o.cursor != nil {
1494 cs := o.cursor.TakeStats()
1495 ev = ev.Int("cursor_moves", cs.Moves).Int("cursor_commits", cs.Commits)
1496 }
1497 fs := o.takeFlipStats()
1498 ev = ev.Int("missed_vblanks", fs.missedVblanks).Float64("max_flip_interval_ms", ms(fs.maxInterval))
1499 ev = ev.Float64("max_commit_delay_ms", ms(fs.maxCommitDelay)).Float64("max_flip_to_read_ms", ms(fs.maxFlipToRead))
1500 ev = ev.Int("state_commits", fs.stateCommits).Int("state_carried", fs.stateCarried)
1501 ev = ev.Int("redrawn_pixels", r.TakeRedrawn())
1502 if o.traceFlips {
1503 ev = ev.Int("late_fences", fs.lateFences)
1504 }
1505 ev.Msg("stats")
1506 }
1507 // A frame wanted while the previous one is in flight is due at
1508 // that flip: accountFlip never counts it before.
1509 if dirty && haveScene && o.wantedAt == 0 {
1510 o.wantedAt = monotonic()
1511 }
1512 o.observeSecurity()
1513 if haveScene && !o.sceneCurrent(scene) {
1514 continue
1515 }
1516 if o.protected && o.wantOff != o.off {
1517 o.invalidateSecurity()
1518 }
1519 if o.protected && !o.securityPrepared {
1520 continue
1521 }
1522 if requests.Len() > 0 && (o.protected || !enabled || o.off || o.wantOff) {
1523 requests.FailPending(capture.ErrOutputOff)
1524 }
1525 if !enabled || o.frame.pendingCommit() {
1526 continue
1527 }
1528 // Output power: off waits for no commit in flight; on is a modeset
1529 // and a full frame.
1530 if o.wantOff != o.off {
1531 var err error
1532 if o.wantOff {
1533 err = o.powerOff()
1534 } else if err = o.modeset(); err == nil {
1535 dirty = true
1536 if !o.protected {
1537 o.probeAsync(r) // skipped while it started off
1538 }
1539 }
1540 if err != nil && !o.commitFailed(err, &enabled) {
1541 return err
1542 }
1543 }
1544 if o.off {
1545 continue
1546 }
1547 if !dirty || !haveScene {
1548 if stateDirty && !o.protected {
1549 err := o.commitState(o.stateVRR())
1550 if errors.Is(err, errSecurityScene) {
1551 continue
1552 }
1553 if errors.Is(err, errOverlayDropped) {
1554 dirty = haveScene
1555 continue
1556 }
1557 if err != nil && !o.commitFailed(err, &enabled) {
1558 return fmt.Errorf("commit: %w", err)
1559 }
1560 }
1561 continue
1562 }
1563 if !o.flipGapUntil.IsZero() {
1564 if time.Now().Before(o.flipGapUntil) {
1565 continue // gapDue commits it
1566 }
1567 o.flipGapUntil = time.Time{}
1568 }
1569 if !o.sceneCurrent(scene) {
1570 continue
1571 }
1572 start := time.Now()
1573 direct, err := o.submitFrame(ctx, r, scene, drawn, seen, requests.List, pipeline)
1574 var fatal renderError
1575 if errors.As(err, &fatal) || errors.Is(err, errSecurityScene) {
1576 // A failed render or epoch rejection did not hand these requests
1577 // to the worker; held ones go too: their frame is gone.
1578 requests.FailPending(err)
1579 }
1580 // Requests still waiting for their indicator stay; the rest is gone.
1581 requests.List = capture.Waiting(requests.List, scene)
1582 if errors.As(err, &fatal) {
1583 return err
1584 }
1585 if err != nil {
1586 if errors.Is(err, errSecurityScene) {
1587 continue
1588 }
1589 if errors.Is(err, errOverlayDropped) {
1590 dirty = true
1591 continue
1592 }
1593 if !o.commitFailed(err, &enabled) {
1594 return fmt.Errorf("page flip: %w", err)
1595 }
1596 continue // dirty stays true
1597 }
1598 dirty = false
1599 frame++
1600 if !o.traceFlips {
1601 continue
1602 }
1603 o.log.Debug().Int("frame", frame).Uint64("seq", scene.Seq).Int("windows", len(scene.Windows)).Bool("direct", direct).Dur("took", time.Since(start)).Msg("frame")
1604 }
1605}
1606
1607// commitFailed handles a commit error that does not stop the output: a
1608// commit still in flight (EBUSY, retried after its event) or DRM master
1609// lost mid-switch (the seat disable follows).
1610func (o *Output) commitFailed(err error, enabled *bool) bool {
1611 switch {
1612 case errors.Is(err, errSecurityScene), errors.Is(err, errProtectionPending):
1613 return true // retry at the owner boundary, never certify stale work
1614 case errors.Is(err, unix.EBUSY):
1615 // Any CRTC event can unblock a refusal; absent one, retry on time.
1616 if o.frame.busy(time.Now()) {
1617 o.log.Info().Err(err).Str("connector", o.conn.name).Msg("commit busy")
1618 }
1619 if o.protected && !o.securityPrepared {
1620 o.deferProtection() // bounded: a busy protected attempt never spins
1621 }
1622 return true
1623 case retryableProtected(err):
1624 // Protection stays latched and the output stays registered, dark.
1625 // Never stop the output: that would remove it from Wayland while the
1626 // session stays locked. Retry later; no proof is sent meanwhile.
1627 o.deferProtection()
1628 o.log.Warn().Err(err).Str("connector", o.conn.name).Dur("retry_in", o.protectBackoff).Msg("protected commit refused; output stays protected, retrying")
1629 return true
1630 case lostMaster(err):
1631 o.observeSecurity()
1632 if o.protected {
1633 o.invalidateSecurity()
1634 }
1635 o.log.Warn().Err(err).Msg("commit refused")
1636 *enabled = false
1637 return true
1638 }
1639 o.log.Error().Err(err).Msg("commit")
1640 return false
1641}
1642
1643// flipDone times the completion ev of the commit f made at start: the
1644// flip stats, the flip trace and the VRR flip gap. It never allocates
1645// with tracing off.
1646func (o *Output) flipDone(ev flipEvent, f pendingFrame, start time.Time, ours, trace bool, fenceAt time.Duration) {
1647 readAt := time.Now()
1648 now := monotonic()
1649 commitAt := now - readAt.Sub(start)
1650 if ours {
1651 o.accountRead(ev, now)
1652 }
1653 if trace {
1654 o.traceFlip(ev, f, commitAt, now, o.lastFlipAt, fenceAt)
1655 }
1656 o.accountFlip(ev, f.frame, f.dueAt(), commitAt, fenceAt)
1657 o.startFlipGap(f, ev.when, readAt)
1658}
1659
1660// completed handles the event of the pending commit: a flipped frame is
1661// reported with its kernel timestamp. It reports false for an event of
1662// another commit, which is dropped.
1663func (o *Output) completed(ev flipEvent, seen map[ports.WindowID]uint64) bool {
1664 // Only a commit of ours is traced: not a stale event, nor the end of
1665 // an EBUSY wait. Its fences are read before flip closes them.
1666 ours := o.frame.ours(ev.user)
1667 trace := o.traceFlips && ours
1668 var fenceAt time.Duration
1669 if trace {
1670 fenceAt = fencesSignalledAt(o.frame.pendingFrame.fences)
1671 }
1672 f, start, ok := o.frame.flip(ev.user)
1673 if !ok {
1674 return false
1675 }
1676 o.flipDone(ev, f, start, ours, trace, fenceAt)
1677 if ours {
1678 o.noteVblank(f, ev.when)
1679 }
1680 if o.cursor != nil {
1681 o.cursor.landed()
1682 }
1683 if !f.frame {
1684 return true
1685 }
1686 if f.composed {
1687 // Frames render in order: this flip's fence signalled after
1688 // those of frames rendered before it and not committed.
1689 o.dropRead()
1690 }
1691 o.flips++
1692 o.shown = f.queued
1693 if o.queued == f.queued {
1694 o.queued = 0
1695 }
1696 o.report(&ports.FlipInfo{When: ev.when, Seq: uint64(ev.seq), Refresh: o.refreshPeriod(), ZeroCopy: f.zeroCopy, Async: f.async, HardwareClock: true, Shows: f.shows}, seen)
1697 if f.animating {
1698 o.wakeCore()
1699 }
1700 return true
1701}
1702
1703// wakeCore tells core the next frame of a running animation is due. A frame
1704// core has not read yet is enough, so the owner never waits.
1705func (o *Output) wakeCore() { corewake.Send(o.Frames, o.conn.name) }
1706
1707// setCursor loads a cursor for scale and output transform t into the cursor
1708// images.
1709func (o *Output) setCursor(r ports.Renderer, load CursorLoader, c ports.CursorChange, scale float64, t ports.BufferTransform) {
1710 o.observeSecurity()
1711 if o.protected || o.cursor.off {
1712 return
1713 }
1714 img, err := load(c, scale, t, o.cursor.Limit())
1715 if err == nil {
1716 err = o.cursor.setImage(r, img.Pixels, img.W, img.H, img.HotX, img.HotY)
1717 }
1718 if err != nil {
1719 o.log.Warn().Err(err).Float64("scale", scale).Int("transform", int(t)).Msg("cursor")
1720 return
1721 }
1722 if c.Image != nil {
1723 // Client cursors (games) reload on every change: too many to log.
1724 return
1725 }
1726 o.log.Debug().Float64("scale", scale).Int("transform", int(t)).Str("shape", c.Shape).Int("w", img.W).Int("h", img.H).Msg("cursor image")
1727}
1728
1729// refused reports a modeset error meaning KMS rejects the images.
1730func refused(err error) bool {
1731 return errors.Is(err, unix.EINVAL) || errors.Is(err, unix.ERANGE)
1732}
1733
1734// lostMaster reports a commit error meaning the seat is switched away.
1735func lostMaster(err error) bool {
1736 return errors.Is(err, unix.EACCES) || errors.Is(err, unix.EPERM)
1737}