Immersive: the stream edge to edge on an iPad with a hardware keyboard

.github/workflows/ci.yml

@@ -134,7 +134,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        check: [addresses, ask-limits, away-copy, away-quality, away-wire, cable-link, clientlink, clipboard, compatibility, device-gate, dmg-layout, door-policy, duo, encoder-mailbox, encoder-slowstate, fence, gesture-chords, gestures, goodbye, hardware-keyboard, home-device, home-model, home-records, home-txt, key-strokes, keychain, link-judge, menu-state, menus, message-reader, origin, pairing-address, phone-portrait, pointer-control, pointer-presence, pointer-watch, policy, protocol, remote-rules, settings-window, tour, update-policy, window-restore]
+        check: [addresses, ask-limits, away-copy, away-quality, away-wire, cable-link, clientlink, clipboard, compatibility, device-gate, dmg-layout, door-policy, duo, encoder-mailbox, encoder-slowstate, fence, gesture-chords, gestures, goodbye, hardware-keyboard, home-device, home-model, home-records, home-txt, immersive, key-strokes, keychain, link-judge, menu-state, menus, message-reader, origin, pairing-address, phone-portrait, pointer-control, pointer-presence, pointer-watch, policy, protocol, remote-rules, settings-window, tour, update-policy, window-restore]
     steps:
       - name: Check out
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

Sources/SillHost/DeviceDesktop.swift

@@ -0,0 +1,157 @@
+import Foundation
+import AppKit
+import CoreGraphics
+import ScreenCaptureKit
+import StreamProtocol
+
+/// The Desktop in full screen (`Viewport.deviceDesktop`): a virtual display at the device's own
+/// size, made the Mac's main display while it streams, as Screen Sharing's High Performance mode
+/// does. The menu bar and the Dock move onto it, so the device shows a whole Mac desktop edge to
+/// edge at its own resolution instead of the real screen fitted with black bars.
+///
+/// Geometry: the display is the device's screen in device points divided by its text scale (Aa;
+/// none means 1:1), at 2× HiDPI, grown evenly to the private API's least 320×240 pt.
+///
+/// Lifetime: created when the Desktop starts with the flag on, recreated when the wanted size
+/// changes (a rotation, an Aa step), removed when anything else streams, the flag goes off, the
+/// last device leaves or the host quits. Before it goes, the display that was main becomes main
+/// again; the window server then moves the windows that were on it back to a real display.
+///
+/// Main actor throughout, like the coordinator. Uses the same private API as `VirtualStage`, with
+/// the same vendor (so the app keeps its own windows off it) and a product ID of its own (so macOS
+/// remembers its arrangement apart from the window stage's).
+@MainActor
+final class DeviceDesktop {
+    static let productID: UInt32 = 0x0002, serialNum: UInt32 = 1
+
+    enum Failure: Error, CustomStringConvertible {
+        case displayCreate(Error)
+        case neverOnline
+        case notListedBySCK
+
+        var description: String {
+            switch self {
+            case .displayCreate(let e): return "displayCreate: \(e)"
+            case .neverOnline: return "neverOnline (the device's display never came up)"
+            case .notListedBySCK: return "notListedBySCK (ScreenCaptureKit never listed the device's display)"
+            }
+        }
+    }
+
+    private var display: VirtualDisplay?
+    private(set) var scDisplay: SCDisplay?
+    /// The display that was main before this one took over, to give the role back.
+    private var previousMain: CGDirectDisplayID?
+    private var generation = 0
+    /// The window server took the display away on its own: the coordinator restarts the Desktop.
+    var onLost: (() -> Void)?
+
+    var isLive: Bool { display?.isAlive == true && scDisplay != nil }
+    var displayID: CGDirectDisplayID? { display?.displayID }
+    /// The display in CG global points (top-left origin), for input and the pointer.
+    var frame: CGRect? { display.map { CGDisplayBounds($0.displayID) } }
+    /// The display's size in points.
+    var size: CGSize? { display.map { CGSize(width: $0.widthPt, height: $0.heightPt) } }
+
+    /// The display's size for a viewport: the panel in Mac points, even, at least 320×240.
+    static func wantedSize(for v: Viewport) -> CGSize? {
+        guard v.width > 0, v.height > 0 else { return nil }
+        let scale = (v.scale ?? 1) > 0 ? (v.scale ?? 1) : 1
+        var w = v.width / scale, h = v.height / scale
+        let grow = max(1, 320 / w, 240 / h)
+        w *= grow; h *= grow
+        func even(_ x: Double) -> CGFloat { CGFloat(min(8192, max(2, (x / 2).rounded() * 2))) }
+        return CGSize(width: even(w), height: even(h))
+    }
+
+    /// Whether the live display already is `size` at `refreshHz`.
+    func matches(size: CGSize, refreshHz: Int) -> Bool {
+        guard isLive, let d = display else { return false }
+        return d.widthPt == Int(size.width) && d.heightPt == Int(size.height) && Int(d.refreshHz) == refreshHz
+    }
+
+    /// A display of `size` points, online, main, and listed by ScreenCaptureKit.
+    func ensure(size: CGSize, refreshHz: Int) async throws -> SCDisplay {
+        if matches(size: size, refreshHz: refreshHz), let sc = scDisplay { return sc }
+        release()
+
+        let t0 = Date()
+        let d: VirtualDisplay
+        do {
+            d = try VirtualDisplay(name: "Sill Desktop", widthPt: Int(size.width), heightPt: Int(size.height), scale: 2,
+                                   refreshHz: Double(refreshHz), vendorID: VirtualStage.vendorID,
+                                   productID: Self.productID, serialNum: Self.serialNum)
+        } catch {
+            throw Failure.displayCreate(error)
+        }
+        generation += 1
+        let gen = generation
+        display = d
+        d.onTerminated = { [weak self] in Task { @MainActor [weak self] in self?.lost(generation: gen) } }
+
+        let online = await d.waitUntilOnline(timeout: 3)
+        if !online {
+            if d.bounds.width == 0 { release(); throw Failure.neverOnline }
+            print("Device desktop is online but not at \(d.widthPt)×\(d.heightPt) @2×: backing \(d.pixelSize.width)×\(d.pixelSize.height) px.")
+        }
+        makeMain(d.displayID)
+
+        // ScreenCaptureKit lists new displays a little after CoreGraphics does (see VirtualStage).
+        let t1 = Date()
+        var listed: SCDisplay?
+        while Date().timeIntervalSince(t1) < 3 {
+            let remaining = max(0.2, 3 - Date().timeIntervalSince(t1))
+            if let c = await WindowCatalog.shareableContent(excludingDesktopWindows: false, onScreenWindowsOnly: true, timeout: remaining),
+               let found = c.displays.first(where: { $0.displayID == d.displayID }) { listed = found; break }
+            try? await Task.sleep(for: .milliseconds(100))
+        }
+        guard let listed, display === d else { release(); throw Failure.notListedBySCK }
+        scDisplay = listed
+        Stats.shared.bump("dd.created")
+        print("Device desktop \(d.widthPt)×\(d.heightPt) pt @2× id \(d.displayID): main, ready after \(Int(Date().timeIntervalSince(t0) * 1000)) ms")
+        return listed
+    }
+
+    /// The main display's role back to the one that had it, then the display gone. Idempotent.
+    func release() {
+        guard let d = display else { scDisplay = nil; return }
+        let id = d.displayID
+        if let previous = previousMain, previous != id, CGMainDisplayID() == id { makeMain(previous) }
+        previousMain = nil
+        display = nil
+        scDisplay = nil
+        d.destroy()
+        print("Removed device desktop \(id)")
+    }
+
+    /// Makes `id` the main display: the main display is the one at the global origin, so it moves
+    /// there and every other display moves by the same amount, keeping the arrangement.
+    private func makeMain(_ id: CGDirectDisplayID) {
+        let main = CGMainDisplayID()
+        guard main != id else { return }
+        if display?.displayID == id, previousMain == nil { previousMain = main }
+        let shift = CGDisplayBounds(id).origin
+        var count: UInt32 = 0
+        guard CGGetActiveDisplayList(0, nil, &count) == .success, count > 0 else { return }
+        var ids = [CGDirectDisplayID](repeating: 0, count: Int(count))
+        guard CGGetActiveDisplayList(count, &ids, &count) == .success else { return }
+        var config: CGDisplayConfigRef?
+        guard CGBeginDisplayConfiguration(&config) == .success else { return }
+        for other in ids.prefix(Int(count)) {
+            let o = CGDisplayBounds(other).origin
+            CGConfigureDisplayOrigin(config, other, Int32(o.x - shift.x), Int32(o.y - shift.y))
+        }
+        if CGCompleteDisplayConfiguration(config, .forSession) != .success {
+            print("Device desktop: could not make display \(id) the main display.")
+        }
+    }
+
+    private func lost(generation gen: Int) {
+        guard gen == generation, display != nil else { return }
+        display = nil
+        scDisplay = nil
+        previousMain = nil
+        Stats.shared.bump("dd.lost")
+        onLost?()
+    }
+}

Sources/SillHost/StreamCoordinator.swift

@@ -70,6 +70,9 @@ package final class StreamCoordinator {
     /// The virtual display needs that loop too: without it the setting is forced off.
     private let appKitLoop: Bool
     let stage: VirtualStage
+    /// The Desktop in full screen on a device: a virtual display at its size, main while it
+    /// streams (`Viewport.deviceDesktop`; DeviceDesktop.swift).
+    let deviceDesktop = DeviceDesktop()
     /// What the menu bar app shows. Written here, at the moment things happen; never read back.
     package let status: HostStatus
     /// Set by the signal handler: a select in flight when Ctrl-C lands must not stage anything new.
@@ -340,6 +343,13 @@ package final class StreamCoordinator {
         remote?.attach(server: server, status: status, macName: macName, requirePairing: config.requirePairing)
         catalog.preferMainDisplay = virtualDisplay   // the Desktop source must never capture the virtual display
         stage.onLost = { [weak self] in Task { @MainActor in await self?.stageLost() } }
+        deviceDesktop.onLost = { [weak self] in
+            Task { @MainActor in
+                guard let self, self.active == .desktop else { return }
+                print("Device desktop: the display went away; streaming the Desktop again.")
+                await self.select(.desktop)
+            }
+        }
         status.update { $0.synthetic = synthetic; $0.virtualDisplayOn = config.virtualDisplay }
         // Every change of the snapshot re-publishes the devices' settings state (deduplicated, so
         // the once-a-second stats send nothing). One before `server.start()` reaches nobody, and
@@ -1053,6 +1063,7 @@ package final class StreamCoordinator {
             // The test pattern with the TEST ONLY scripted pointer: its own space, where a test
             // client's input moves that pointer (a synthetic host posts nothing).
             if synthetic, pointer.hasTestPointer { return Self.testPatternRect }
+            if deviceDesktop.isLive, let frame = deviceDesktop.frame { return frame }
             return catalog.display?.frame
         case .window(let id):
             // On the virtual display the video is the captured crop, not the whole window: an app
@@ -1096,6 +1107,7 @@ package final class StreamCoordinator {
             return nil
         case .desktop:
             if synthetic { return pointer.hasTestPointer ? .rect(Self.testPatternRect) : nil }
+            if deviceDesktop.isLive, let frame = deviceDesktop.frame { return .rect(frame) }
             return catalog.display.map { .rect($0.frame) }
         case .window(let id):
             guard !synthetic else { return nil }
@@ -1137,6 +1149,14 @@ package final class StreamCoordinator {
         // A different rate (a ProMotion device joined, Low Power Mode toggled) needs a new capture
         // interval and encoder session: one restart, whatever the source.
         if active != .none, effectiveFPS != fps { await select(active); return }
+        // The Desktop in full screen: a display of the device's size, made, resized or removed by a
+        // restart of the Desktop (the flag or the size changed: a toggle, a rotation, an Aa step).
+        if active == .desktop, !synthetic {
+            let want = wantedDeviceDesktop
+            let unchanged = want.map { deviceDesktop.matches(size: $0, refreshHz: fps) } ?? !deviceDesktop.isLive
+            if !unchanged { await select(.desktop) }
+            return
+        }
         if virtualDisplay, stage.isStaged, case .window(let id) = active {
             // One restart: prepare re-places the window on the same display (the display is only
             // recreated if the envelope grew). The client's identical re-send after a switch is a
@@ -1155,6 +1175,13 @@ package final class StreamCoordinator {
         if fitToViewport(w) != nil { await catalog.refreshWindows() }
     }
 
+    /// The device desktop's size for the last viewport, or nil for the Mac's real screen: the
+    /// device did not ask, or the display cannot be made here (no AppKit loop, a synthetic host).
+    private var wantedDeviceDesktop: CGSize? {
+        guard appKitLoop, !synthetic, let v = viewport, v.deviceDesktop == true else { return nil }
+        return DeviceDesktop.wantedSize(for: v)
+    }
+
     /// `--virtual-display`: how big the window should be on the virtual display and how big the
     /// display must be, from the last viewport (or the window's own size before any arrived).
     private func wantedStage(for w: SCWindow) -> (size: CGSize, envelope: CGSize) {
@@ -1312,6 +1339,8 @@ package final class StreamCoordinator {
                 stage.release()
             }
         }
+        // The device desktop lives only while the Desktop streams to a device that asked for it.
+        if source != .desktop || wantedDeviceDesktop == nil { deviceDesktop.release() }
         // In regular mode, every window Sill sized but the one about to stream goes back to its own
         // frame (WindowRestore, issue #57): another window or the Desktop picked, `.none` when the
         // last device leaves or the stream stops. A restart of the same window keeps it fitted.
@@ -1395,7 +1424,19 @@ package final class StreamCoordinator {
                 describe = "a synthetic test pattern\(useSoftwareEncoder ? " (software encoder)" : "")"
                 title = "Test Pattern"; kind = .testPattern
             } else {
-                guard let d = catalog.display else { active = .none; broadcastList(); return }
+                var shown = catalog.display
+                var deviceSize: CGSize? = nil
+                if let want = wantedDeviceDesktop {
+                    do {
+                        shown = try await deviceDesktop.ensure(size: want, refreshHz: fps)
+                        deviceSize = want
+                    } catch {
+                        print("Device desktop unavailable (\(error)); streaming the Mac's screen instead.")
+                        deviceDesktop.release()
+                    }
+                    guard !shuttingDown else { deviceDesktop.release(); active = .none; broadcastList(); return }
+                }
+                guard let d = shown else { active = .none; broadcastList(); return }
                 // Sill's own windows (Settings, Log, the pairing window with its code) never go out
                 // in a Desktop stream. Fixed here at pipeline start: a running SCStream is never
                 // reconfigured, so a host that can show a pairing code (remote access) looks for
@@ -1413,7 +1454,8 @@ package final class StreamCoordinator {
                     filter = SCContentFilter(display: d, excludingWindows: [])
                 }
                 width = evenPixels(CGFloat(d.width) * captureScale); height = evenPixels(CGFloat(d.height) * captureScale)
-                describe = "the whole desktop\(useSoftwareEncoder ? " (software encoder)" : "")"
+                describe = (deviceSize.map { "the device's own desktop, \(Int($0.width))×\(Int($0.height)) pt" } ?? "the whole desktop")
+                    + (useSoftwareEncoder ? " (software encoder)" : "")
                 title = "Whole Desktop"; kind = .desktop
             }
         }
@@ -1894,6 +1936,7 @@ package final class StreamCoordinator {
         shuttingDown = true
         restoreFits(keeping: nil)
         stage.release()
+        deviceDesktop.release()
     }
 
     /// The CLI's default path at a signal (HostShutdown.installKeyRelease), just before it dies of it:

Sources/SillHost/WindowCatalog.swift

@@ -242,9 +242,12 @@ final class WindowCatalog {
         guard seq > appliedSeq else { return }      // a newer refresh already landed; don't go backwards
         appliedSeq = seq
         ownApplication = content.applications.first { $0.processID == Self.ownPID }
+        // Sill's own virtual displays (the window stage's, the device desktop's, which can be the
+        // main display while it streams) are never the Desktop source's real screen.
+        let real = content.displays.filter { CGDisplayVendorNumber($0.displayID) != VirtualStage.vendorID }
         display = preferMainDisplay
-            ? (content.displays.first { $0.displayID == CGMainDisplayID() } ?? content.displays.first)
-            : content.displays.first
+            ? (real.first { $0.displayID == CGMainDisplayID() } ?? real.first ?? content.displays.first)
+            : (real.first ?? content.displays.first)
         var visible = content.windows.filter { w in
             guard let app = w.owningApplication, !app.applicationName.isEmpty else { return false }
             return w.windowLayer == 0                                  // drops widgets, wallpaper, backstop

Sources/StreamProtocol/Viewport.swift

@@ -20,9 +20,16 @@ public struct Viewport: Codable, Hashable {
     /// The host runs the stream at the highest rate among its connected clients and restarts the
     /// pipeline when that changes.
     public var fps: Int?
+    /// The device shows the Desktop in full screen and wants it as a display of its own: the host
+    /// makes a virtual display of this viewport's size (÷ `scale`) the Mac's main display while the
+    /// Desktop streams, as Screen Sharing's High Performance mode does. nil or false (an older
+    /// client, or not in full screen) is the Mac's real screen. An older host ignores the key.
+    public var deviceDesktop: Bool?
 
-    public init(width: Double, height: Double, scale: Double?, localCursor: Bool? = nil, fps: Int? = nil) {
+    public init(width: Double, height: Double, scale: Double?, localCursor: Bool? = nil, fps: Int? = nil,
+                deviceDesktop: Bool? = nil) {
         self.width = width; self.height = height; self.scale = scale; self.localCursor = localCursor; self.fps = fps
+        self.deviceDesktop = deviceDesktop
     }
 }