diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e2ae359e1..785dfb91b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -134,7 +134,7 @@ jobs: strategy: fail-fast: false matrix: - check: [addresses, ask-limits, away-copy, away-quality, away-wire, cable-link, clientlink, clipboard, compatibility, device-gate, dmg-layout, door-policy, duo, encoder-mailbox, encoder-slowstate, fence, gesture-chords, gestures, goodbye, hardware-keyboard, home-device, home-model, home-records, home-txt, key-strokes, keychain, link-judge, menu-state, menus, message-reader, origin, pairing-address, phone-portrait, pointer-control, pointer-presence, pointer-watch, policy, protocol, remote-rules, settings-window, tour, update-policy, window-restore] + check: [addresses, ask-limits, away-copy, away-quality, away-wire, cable-link, clientlink, clipboard, compatibility, device-gate, dmg-layout, door-policy, duo, encoder-mailbox, encoder-slowstate, fence, gesture-chords, gestures, goodbye, hardware-keyboard, home-device, home-model, home-records, home-txt, immersive, key-strokes, keychain, link-judge, menu-state, menus, message-reader, origin, pairing-address, phone-portrait, pointer-control, pointer-presence, pointer-watch, policy, protocol, remote-rules, settings-window, tour, update-policy, window-restore] steps: - name: Check out uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/Sources/SillHost/DeviceDesktop.swift b/Sources/SillHost/DeviceDesktop.swift new file mode 100644 index 000000000..442a81436 --- /dev/null +++ b/Sources/SillHost/DeviceDesktop.swift @@ -0,0 +1,157 @@ +import Foundation +import AppKit +import CoreGraphics +import ScreenCaptureKit +import StreamProtocol + +/// The Desktop in full screen (`Viewport.deviceDesktop`): a virtual display at the device's own +/// size, made the Mac's main display while it streams, as Screen Sharing's High Performance mode +/// does. The menu bar and the Dock move onto it, so the device shows a whole Mac desktop edge to +/// edge at its own resolution instead of the real screen fitted with black bars. +/// +/// Geometry: the display is the device's screen in device points divided by its text scale (Aa; +/// none means 1:1), at 2× HiDPI, grown evenly to the private API's least 320×240 pt. +/// +/// Lifetime: created when the Desktop starts with the flag on, recreated when the wanted size +/// changes (a rotation, an Aa step), removed when anything else streams, the flag goes off, the +/// last device leaves or the host quits. Before it goes, the display that was main becomes main +/// again; the window server then moves the windows that were on it back to a real display. +/// +/// Main actor throughout, like the coordinator. Uses the same private API as `VirtualStage`, with +/// the same vendor (so the app keeps its own windows off it) and a product ID of its own (so macOS +/// remembers its arrangement apart from the window stage's). +@MainActor +final class DeviceDesktop { + static let productID: UInt32 = 0x0002, serialNum: UInt32 = 1 + + enum Failure: Error, CustomStringConvertible { + case displayCreate(Error) + case neverOnline + case notListedBySCK + + var description: String { + switch self { + case .displayCreate(let e): return "displayCreate: \(e)" + case .neverOnline: return "neverOnline (the device's display never came up)" + case .notListedBySCK: return "notListedBySCK (ScreenCaptureKit never listed the device's display)" + } + } + } + + private var display: VirtualDisplay? + private(set) var scDisplay: SCDisplay? + /// The display that was main before this one took over, to give the role back. + private var previousMain: CGDirectDisplayID? + private var generation = 0 + /// The window server took the display away on its own: the coordinator restarts the Desktop. + var onLost: (() -> Void)? + + var isLive: Bool { display?.isAlive == true && scDisplay != nil } + var displayID: CGDirectDisplayID? { display?.displayID } + /// The display in CG global points (top-left origin), for input and the pointer. + var frame: CGRect? { display.map { CGDisplayBounds($0.displayID) } } + /// The display's size in points. + var size: CGSize? { display.map { CGSize(width: $0.widthPt, height: $0.heightPt) } } + + /// The display's size for a viewport: the panel in Mac points, even, at least 320×240. + static func wantedSize(for v: Viewport) -> CGSize? { + guard v.width > 0, v.height > 0 else { return nil } + let scale = (v.scale ?? 1) > 0 ? (v.scale ?? 1) : 1 + var w = v.width / scale, h = v.height / scale + let grow = max(1, 320 / w, 240 / h) + w *= grow; h *= grow + func even(_ x: Double) -> CGFloat { CGFloat(min(8192, max(2, (x / 2).rounded() * 2))) } + return CGSize(width: even(w), height: even(h)) + } + + /// Whether the live display already is `size` at `refreshHz`. + func matches(size: CGSize, refreshHz: Int) -> Bool { + guard isLive, let d = display else { return false } + return d.widthPt == Int(size.width) && d.heightPt == Int(size.height) && Int(d.refreshHz) == refreshHz + } + + /// A display of `size` points, online, main, and listed by ScreenCaptureKit. + func ensure(size: CGSize, refreshHz: Int) async throws -> SCDisplay { + if matches(size: size, refreshHz: refreshHz), let sc = scDisplay { return sc } + release() + + let t0 = Date() + let d: VirtualDisplay + do { + d = try VirtualDisplay(name: "Sill Desktop", widthPt: Int(size.width), heightPt: Int(size.height), scale: 2, + refreshHz: Double(refreshHz), vendorID: VirtualStage.vendorID, + productID: Self.productID, serialNum: Self.serialNum) + } catch { + throw Failure.displayCreate(error) + } + generation += 1 + let gen = generation + display = d + d.onTerminated = { [weak self] in Task { @MainActor [weak self] in self?.lost(generation: gen) } } + + let online = await d.waitUntilOnline(timeout: 3) + if !online { + if d.bounds.width == 0 { release(); throw Failure.neverOnline } + print("Device desktop is online but not at \(d.widthPt)×\(d.heightPt) @2×: backing \(d.pixelSize.width)×\(d.pixelSize.height) px.") + } + makeMain(d.displayID) + + // ScreenCaptureKit lists new displays a little after CoreGraphics does (see VirtualStage). + let t1 = Date() + var listed: SCDisplay? + while Date().timeIntervalSince(t1) < 3 { + let remaining = max(0.2, 3 - Date().timeIntervalSince(t1)) + if let c = await WindowCatalog.shareableContent(excludingDesktopWindows: false, onScreenWindowsOnly: true, timeout: remaining), + let found = c.displays.first(where: { $0.displayID == d.displayID }) { listed = found; break } + try? await Task.sleep(for: .milliseconds(100)) + } + guard let listed, display === d else { release(); throw Failure.notListedBySCK } + scDisplay = listed + Stats.shared.bump("dd.created") + print("Device desktop \(d.widthPt)×\(d.heightPt) pt @2× id \(d.displayID): main, ready after \(Int(Date().timeIntervalSince(t0) * 1000)) ms") + return listed + } + + /// The main display's role back to the one that had it, then the display gone. Idempotent. + func release() { + guard let d = display else { scDisplay = nil; return } + let id = d.displayID + if let previous = previousMain, previous != id, CGMainDisplayID() == id { makeMain(previous) } + previousMain = nil + display = nil + scDisplay = nil + d.destroy() + print("Removed device desktop \(id)") + } + + /// Makes `id` the main display: the main display is the one at the global origin, so it moves + /// there and every other display moves by the same amount, keeping the arrangement. + private func makeMain(_ id: CGDirectDisplayID) { + let main = CGMainDisplayID() + guard main != id else { return } + if display?.displayID == id, previousMain == nil { previousMain = main } + let shift = CGDisplayBounds(id).origin + var count: UInt32 = 0 + guard CGGetActiveDisplayList(0, nil, &count) == .success, count > 0 else { return } + var ids = [CGDirectDisplayID](repeating: 0, count: Int(count)) + guard CGGetActiveDisplayList(count, &ids, &count) == .success else { return } + var config: CGDisplayConfigRef? + guard CGBeginDisplayConfiguration(&config) == .success else { return } + for other in ids.prefix(Int(count)) { + let o = CGDisplayBounds(other).origin + CGConfigureDisplayOrigin(config, other, Int32(o.x - shift.x), Int32(o.y - shift.y)) + } + if CGCompleteDisplayConfiguration(config, .forSession) != .success { + print("Device desktop: could not make display \(id) the main display.") + } + } + + private func lost(generation gen: Int) { + guard gen == generation, display != nil else { return } + display = nil + scDisplay = nil + previousMain = nil + Stats.shared.bump("dd.lost") + onLost?() + } +} diff --git a/Sources/SillHost/StreamCoordinator.swift b/Sources/SillHost/StreamCoordinator.swift index 1e33dfa83..a6e96f958 100644 --- a/Sources/SillHost/StreamCoordinator.swift +++ b/Sources/SillHost/StreamCoordinator.swift @@ -70,6 +70,9 @@ package final class StreamCoordinator { /// The virtual display needs that loop too: without it the setting is forced off. private let appKitLoop: Bool let stage: VirtualStage + /// The Desktop in full screen on a device: a virtual display at its size, main while it + /// streams (`Viewport.deviceDesktop`; DeviceDesktop.swift). + let deviceDesktop = DeviceDesktop() /// What the menu bar app shows. Written here, at the moment things happen; never read back. package let status: HostStatus /// Set by the signal handler: a select in flight when Ctrl-C lands must not stage anything new. @@ -340,6 +343,13 @@ package final class StreamCoordinator { remote?.attach(server: server, status: status, macName: macName, requirePairing: config.requirePairing) catalog.preferMainDisplay = virtualDisplay // the Desktop source must never capture the virtual display stage.onLost = { [weak self] in Task { @MainActor in await self?.stageLost() } } + deviceDesktop.onLost = { [weak self] in + Task { @MainActor in + guard let self, self.active == .desktop else { return } + print("Device desktop: the display went away; streaming the Desktop again.") + await self.select(.desktop) + } + } status.update { $0.synthetic = synthetic; $0.virtualDisplayOn = config.virtualDisplay } // Every change of the snapshot re-publishes the devices' settings state (deduplicated, so // the once-a-second stats send nothing). One before `server.start()` reaches nobody, and @@ -1053,6 +1063,7 @@ package final class StreamCoordinator { // The test pattern with the TEST ONLY scripted pointer: its own space, where a test // client's input moves that pointer (a synthetic host posts nothing). if synthetic, pointer.hasTestPointer { return Self.testPatternRect } + if deviceDesktop.isLive, let frame = deviceDesktop.frame { return frame } return catalog.display?.frame case .window(let id): // On the virtual display the video is the captured crop, not the whole window: an app @@ -1096,6 +1107,7 @@ package final class StreamCoordinator { return nil case .desktop: if synthetic { return pointer.hasTestPointer ? .rect(Self.testPatternRect) : nil } + if deviceDesktop.isLive, let frame = deviceDesktop.frame { return .rect(frame) } return catalog.display.map { .rect($0.frame) } case .window(let id): guard !synthetic else { return nil } @@ -1137,6 +1149,14 @@ package final class StreamCoordinator { // A different rate (a ProMotion device joined, Low Power Mode toggled) needs a new capture // interval and encoder session: one restart, whatever the source. if active != .none, effectiveFPS != fps { await select(active); return } + // The Desktop in full screen: a display of the device's size, made, resized or removed by a + // restart of the Desktop (the flag or the size changed: a toggle, a rotation, an Aa step). + if active == .desktop, !synthetic { + let want = wantedDeviceDesktop + let unchanged = want.map { deviceDesktop.matches(size: $0, refreshHz: fps) } ?? !deviceDesktop.isLive + if !unchanged { await select(.desktop) } + return + } if virtualDisplay, stage.isStaged, case .window(let id) = active { // One restart: prepare re-places the window on the same display (the display is only // recreated if the envelope grew). The client's identical re-send after a switch is a @@ -1155,6 +1175,13 @@ package final class StreamCoordinator { if fitToViewport(w) != nil { await catalog.refreshWindows() } } + /// The device desktop's size for the last viewport, or nil for the Mac's real screen: the + /// device did not ask, or the display cannot be made here (no AppKit loop, a synthetic host). + private var wantedDeviceDesktop: CGSize? { + guard appKitLoop, !synthetic, let v = viewport, v.deviceDesktop == true else { return nil } + return DeviceDesktop.wantedSize(for: v) + } + /// `--virtual-display`: how big the window should be on the virtual display and how big the /// display must be, from the last viewport (or the window's own size before any arrived). private func wantedStage(for w: SCWindow) -> (size: CGSize, envelope: CGSize) { @@ -1312,6 +1339,8 @@ package final class StreamCoordinator { stage.release() } } + // The device desktop lives only while the Desktop streams to a device that asked for it. + if source != .desktop || wantedDeviceDesktop == nil { deviceDesktop.release() } // In regular mode, every window Sill sized but the one about to stream goes back to its own // frame (WindowRestore, issue #57): another window or the Desktop picked, `.none` when the // last device leaves or the stream stops. A restart of the same window keeps it fitted. @@ -1395,7 +1424,19 @@ package final class StreamCoordinator { describe = "a synthetic test pattern\(useSoftwareEncoder ? " (software encoder)" : "")" title = "Test Pattern"; kind = .testPattern } else { - guard let d = catalog.display else { active = .none; broadcastList(); return } + var shown = catalog.display + var deviceSize: CGSize? = nil + if let want = wantedDeviceDesktop { + do { + shown = try await deviceDesktop.ensure(size: want, refreshHz: fps) + deviceSize = want + } catch { + print("Device desktop unavailable (\(error)); streaming the Mac's screen instead.") + deviceDesktop.release() + } + guard !shuttingDown else { deviceDesktop.release(); active = .none; broadcastList(); return } + } + guard let d = shown else { active = .none; broadcastList(); return } // Sill's own windows (Settings, Log, the pairing window with its code) never go out // in a Desktop stream. Fixed here at pipeline start: a running SCStream is never // reconfigured, so a host that can show a pairing code (remote access) looks for @@ -1413,7 +1454,8 @@ package final class StreamCoordinator { filter = SCContentFilter(display: d, excludingWindows: []) } width = evenPixels(CGFloat(d.width) * captureScale); height = evenPixels(CGFloat(d.height) * captureScale) - describe = "the whole desktop\(useSoftwareEncoder ? " (software encoder)" : "")" + describe = (deviceSize.map { "the device's own desktop, \(Int($0.width))×\(Int($0.height)) pt" } ?? "the whole desktop") + + (useSoftwareEncoder ? " (software encoder)" : "") title = "Whole Desktop"; kind = .desktop } } @@ -1894,6 +1936,7 @@ package final class StreamCoordinator { shuttingDown = true restoreFits(keeping: nil) stage.release() + deviceDesktop.release() } /// The CLI's default path at a signal (HostShutdown.installKeyRelease), just before it dies of it: diff --git a/Sources/SillHost/WindowCatalog.swift b/Sources/SillHost/WindowCatalog.swift index 09d2f71e1..943fe8e0a 100644 --- a/Sources/SillHost/WindowCatalog.swift +++ b/Sources/SillHost/WindowCatalog.swift @@ -242,9 +242,12 @@ final class WindowCatalog { guard seq > appliedSeq else { return } // a newer refresh already landed; don't go backwards appliedSeq = seq ownApplication = content.applications.first { $0.processID == Self.ownPID } + // Sill's own virtual displays (the window stage's, the device desktop's, which can be the + // main display while it streams) are never the Desktop source's real screen. + let real = content.displays.filter { CGDisplayVendorNumber($0.displayID) != VirtualStage.vendorID } display = preferMainDisplay - ? (content.displays.first { $0.displayID == CGMainDisplayID() } ?? content.displays.first) - : content.displays.first + ? (real.first { $0.displayID == CGMainDisplayID() } ?? real.first ?? content.displays.first) + : (real.first ?? content.displays.first) var visible = content.windows.filter { w in guard let app = w.owningApplication, !app.applicationName.isEmpty else { return false } return w.windowLayer == 0 // drops widgets, wallpaper, backstop diff --git a/Sources/StreamProtocol/Viewport.swift b/Sources/StreamProtocol/Viewport.swift index cffff6ca2..720209ddc 100644 --- a/Sources/StreamProtocol/Viewport.swift +++ b/Sources/StreamProtocol/Viewport.swift @@ -20,9 +20,16 @@ public struct Viewport: Codable, Hashable { /// The host runs the stream at the highest rate among its connected clients and restarts the /// pipeline when that changes. public var fps: Int? + /// The device shows the Desktop in full screen and wants it as a display of its own: the host + /// makes a virtual display of this viewport's size (÷ `scale`) the Mac's main display while the + /// Desktop streams, as Screen Sharing's High Performance mode does. nil or false (an older + /// client, or not in full screen) is the Mac's real screen. An older host ignores the key. + public var deviceDesktop: Bool? - public init(width: Double, height: Double, scale: Double?, localCursor: Bool? = nil, fps: Int? = nil) { + public init(width: Double, height: Double, scale: Double?, localCursor: Bool? = nil, fps: Int? = nil, + deviceDesktop: Bool? = nil) { self.width = width; self.height = height; self.scale = scale; self.localCursor = localCursor; self.fps = fps + self.deviceDesktop = deviceDesktop } } diff --git a/Tests/checks/immersive/main.swift b/Tests/checks/immersive/main.swift new file mode 100644 index 000000000..d08b9f814 --- /dev/null +++ b/Tests/checks/immersive/main.swift @@ -0,0 +1,232 @@ +// ImmersivePolicy (iOSClient/ImmersivePolicy.swift) on its own: when Immersive applies, which of its three +// states the stream screen is in, what each event does to the bar and the auto-hide timer, the +// shortcut's chord and the top edge's swipe (docs/immersive-plan.md). Scenario by scenario, then +// 20,000 random sessions against the plan's invariants: +// I1 the state is off exactly when Immersive does not apply; +// I2 the panel's size (full screen or not) changes only when the state moves between off and the +// other two: revealing and hiding the bar never resize it; +// I3 while something holds the bar it is shown; +// I4 the shortcut is taken only while Immersive applies; +// I5 the auto-hide never hides a held bar, and the timer is never left running while held or off. +// swiftc -O iOSClient/ImmersivePolicy.swift Tests/checks/immersive/main.swift -o check && ./check +import Foundation + +typealias P = ImmersivePolicy +var fails = 0, passes = 0 +func check(_ name: String, _ ok: Bool) { + if ok { passes += 1; print("ok \(name)") } else { fails += 1; print("FAIL \(name)") } +} + +// MARK: - When it applies (R2, R12, Stage Manager) + +do { + var all = 0, on = 0 + for bits in 0..<64 { + let b = (0..<6).map { bits & (1 << $0) != 0 } + let got = P.applies(switchOn: b[0], keyboardAttached: b[1], landscape: b[2], streaming: b[3], + voiceOver: b[4], fillsScreen: b[5]) + let want = b[0] && b[1] && b[2] && b[3] && !b[4] && b[5] + all += 1 + if got { on += 1 } + if got != want { check("applies \(b)", false) } + } + check("applies: exactly one of 64 combinations (switch, keyboard, sideways, streaming, VoiceOver off, full screen)", on == 1 && all == 64) + check("applies: all six together", P.applies(switchOn: true, keyboardAttached: true, landscape: true, streaming: true, voiceOver: false, fillsScreen: true)) + check("applies: not with VoiceOver", !P.applies(switchOn: true, keyboardAttached: true, landscape: true, streaming: true, voiceOver: true, fillsScreen: true)) + check("applies: not with nothing streaming", !P.applies(switchOn: true, keyboardAttached: true, landscape: true, streaming: false, voiceOver: false, fillsScreen: true)) + check("applies: not with no keyboard", !P.applies(switchOn: true, keyboardAttached: false, landscape: true, streaming: true, voiceOver: false, fillsScreen: true)) + check("applies: not upright", !P.applies(switchOn: true, keyboardAttached: true, landscape: false, streaming: true, voiceOver: false, fillsScreen: true)) + check("applies: not in a Stage Manager window", !P.applies(switchOn: true, keyboardAttached: true, landscape: true, streaming: true, voiceOver: false, fillsScreen: false)) + check("applies: not with the switch off", !P.applies(switchOn: false, keyboardAttached: true, landscape: true, streaming: true, voiceOver: false, fillsScreen: true)) +} + +// MARK: - Filling the screen + +check("fills: 13\" iPad Pro sideways, the whole screen", P.fills(size: (1376, 1032), screen: (1376, 1032))) +check("fills: the screen's bounds the other way round", P.fills(size: (1376, 1032), screen: (1032, 1376))) +check("fills: within a point", P.fills(size: (1375.5, 1032), screen: (1376, 1032))) +check("fills: not a Stage Manager window", !P.fills(size: (1100, 800), screen: (1376, 1032))) +check("fills: not Split View's half", !P.fills(size: (684, 1032), screen: (1376, 1032))) +check("fills: not a slide-over window", !P.fills(size: (375, 1032), screen: (1376, 1032))) +check("fills: not a zero size", !P.fills(size: (0, 0), screen: (1376, 1032))) +check("fills: not a scene a column narrower", !P.fills(size: (1374, 1032), screen: (1376, 1032))) + +// MARK: - The shortcut + +let ctrl: UInt64 = 1 << 18, opt: UInt64 = 1 << 19, cmd: UInt64 = 1 << 20, shift: UInt64 = 1 << 17 +let caps: UInt64 = 1 << 16, keypad: UInt64 = 1 << 21 +check("chord: ⌃⌥⌘B", P.isToggle(usage: 0x05, modifiers: ctrl | opt | cmd)) +check("chord: ⌃⌥⌘B with caps lock on", P.isToggle(usage: 0x05, modifiers: ctrl | opt | cmd | caps)) +check("chord: ⌃⌥⌘B with the keypad bit", P.isToggle(usage: 0x05, modifiers: ctrl | opt | cmd | keypad)) +check("chord: not ⇧⌃⌥⌘B", !P.isToggle(usage: 0x05, modifiers: shift | ctrl | opt | cmd)) +check("chord: not ⌃⌘B", !P.isToggle(usage: 0x05, modifiers: ctrl | cmd)) +check("chord: not ⌥⌘B", !P.isToggle(usage: 0x05, modifiers: opt | cmd)) +check("chord: not ⌃⌥B", !P.isToggle(usage: 0x05, modifiers: ctrl | opt)) +check("chord: not ⌘B (bold on the Mac)", !P.isToggle(usage: 0x05, modifiers: cmd)) +check("chord: not B", !P.isToggle(usage: 0x05, modifiers: 0)) +check("chord: not ⌃⌥⌘N", !P.isToggle(usage: 0x11, modifiers: ctrl | opt | cmd)) +check("chord: not ⌃⌘F (the Mac's Enter Full Screen)", !P.isToggle(usage: 0x09, modifiers: ctrl | cmd)) +do { + var count = 0 + for usage in UInt16(0)...UInt16(0xE7) { + for mask in 0..<64 { + let mods = (0..<6).reduce(UInt64(0)) { mask & (1 << $1) != 0 ? $0 | (1 << UInt64(16 + $1)) : $0 } + if P.isToggle(usage: usage, modifiers: mods) { count += 1 } + } + } + // B with ⌃⌥⌘, and caps lock and the keypad bit either way: four. + check("chord: exactly four of every usage × 64 flag sets", count == 4) +} + +// MARK: - The edge + +check("edge: straight down from the very top", P.revealsFromEdge(startY: 0, dx: 0, dy: 30)) +check("edge: down from 20 pt", P.revealsFromEdge(startY: 20, dx: 3, dy: 30)) +check("edge: not from 21 pt", !P.revealsFromEdge(startY: 21, dx: 0, dy: 30)) +check("edge: not from mid-screen", !P.revealsFromEdge(startY: 400, dx: 0, dy: 30)) +check("edge: not sideways along the top (a browser's tabs)", !P.revealsFromEdge(startY: 5, dx: 40, dy: 10)) +check("edge: not diagonal at exactly 45°", !P.revealsFromEdge(startY: 5, dx: 20, dy: 20)) +check("edge: not up", !P.revealsFromEdge(startY: 5, dx: 0, dy: -30)) +check("edge: not a drag that has not moved", !P.revealsFromEdge(startY: 5, dx: 0, dy: 0)) +check("edge: not from above the screen", !P.revealsFromEdge(startY: -1, dx: 0, dy: 30)) + +// MARK: - Scenarios + +do { + var p = P() + check("start: off", p.state == .off) + check("start: toggle not taken while off", p.toggle() == (false, .keep) && p.state == .off) + check("start: edge does nothing while off", p.edgeSwipe() == .keep && p.state == .off) + + // Entering: the switch's change applies as Settings closes, so applies and held change together. + check("enter: immersed, timer stopped", p.update(applies: true, held: false) == .stop && p.state == .immersed) + check("enter again (repeat): nothing", p.update(applies: true, held: false) == .keep && p.state == .immersed) + + var t = p.toggle() + check("shortcut: reveals, timer starts", t == (true, .start) && p.state == .revealed) + check("touch: restarts the timer", p.touched() == .start) + t = p.toggle() + check("shortcut again: hides, timer stops", t == (true, .stop) && p.state == .immersed) + check("touch while hidden: nothing", p.touched() == .keep) + + check("edge: reveals, timer starts", p.edgeSwipe() == .start && p.state == .revealed) + p.timerFired() + check("timer: hides", p.state == .immersed) + + // Something opens over the stream (Settings from the revealed bar). + _ = p.edgeSwipe() + check("held: the bar stays, timer stopped", p.update(applies: true, held: true) == .stop && p.state == .revealed) + p.timerFired() + check("held: a late timer does not hide it", p.state == .revealed) + t = p.toggle() + check("held: the shortcut is taken but changes nothing", t == (true, .keep) && p.state == .revealed) + check("held: edge changes nothing", p.edgeSwipe() == .keep && p.state == .revealed) + check("held: touch does not start the timer", p.touched() == .keep) + check("released: the bar lingers, timer starts", p.update(applies: true, held: false) == .start && p.state == .revealed) + p.timerFired() + check("released: then hides", p.state == .immersed) + + // A taker opening while immersed (the tour, ⌘. opening Settings): the bar comes. + check("held from immersed: revealed", p.update(applies: true, held: true) == .stop && p.state == .revealed) + check("released: lingers", p.update(applies: true, held: false) == .start && p.state == .revealed) + + // Detaching the keyboard while revealed: off at once, the reveal forgotten. + check("detach: off, timer stopped", p.update(applies: false, held: false) == .stop && p.state == .off) + check("attach again: immersed, not revealed", p.update(applies: true, held: false) == .stop && p.state == .immersed) + + // Entering with something open: revealed, held. + var q = P() + check("enter held: revealed, timer stopped", q.update(applies: true, held: true) == .stop && q.state == .revealed) + check("enter held then released: lingers", q.update(applies: true, held: false) == .start && q.state == .revealed) + + // Held while off, then applies: revealed; held off while not applying: nothing pending. + var r = P() + _ = r.update(applies: false, held: true) + check("held while off: still off", r.state == .off) + _ = r.update(applies: false, held: false) + check("released while off: entering later is immersed", r.update(applies: true, held: false) == .stop && r.state == .immersed) + + // Rotating to portrait and back. + var s = P() + _ = s.update(applies: true, held: false) + _ = s.toggle() + _ = s.update(applies: false, held: false) + check("portrait: off", s.state == .off) + _ = s.update(applies: true, held: false) + check("back sideways: immersed", s.state == .immersed) + + // The timer firing while off never revives anything. + var u = P() + u.timerFired() + check("timer while off: off", u.state == .off) + + check("panel: full screen immersed and revealed, not off", + P.fillsScreen(.immersed) && P.fillsScreen(.revealed) && !P.fillsScreen(.off)) + check("bar: shown off and revealed, not immersed", + P.barShown(.off) && P.barShown(.revealed) && !P.barShown(.immersed)) + check("auto-hide: 4 s", P.autoHide == 4) +} + +// MARK: - Random sessions + +enum Ev { case update(Bool, Bool), toggle, edge, touch, fire } +var rng = SystemRandomNumberGenerator() +func pick() -> Ev { + switch Int.random(in: 0..<10, using: &rng) { + case 0, 1, 2: return .update(Bool.random(using: &rng) || Bool.random(using: &rng), Bool.random(using: &rng)) + case 3, 4: return .toggle + case 5: return .edge + case 6: return .touch + default: return .fire + } +} + +var broken: [String: Int] = [:] +func bad(_ name: String) { broken[name, default: 0] += 1 } +var timerRuns = 0 +for _ in 0..<20_000 { + var p = P() + var applies = false, held = false + var timer = false + for _ in 0..<40 { + let before = p.state + var resized = false + let ev = pick() + var out: P.Timer = .keep + switch ev { + case .update(let a, let h): + applies = a; held = h + out = p.update(applies: a, held: h) + resized = P.fillsScreen(before) != P.fillsScreen(p.state) + case .toggle: + let r = p.toggle() + out = r.timer + if r.taken != applies { bad("I4 the shortcut taken \(r.taken) while applying \(applies)") } + case .edge: out = p.edgeSwipe() + case .touch: out = p.touched() + case .fire: + if timer { + p.timerFired() + timer = false + timerRuns += 1 + if held && applies && p.state != .revealed { bad("I5 the timer hid a held bar") } + } + } + switch out { case .start: timer = true; case .stop: timer = false; case .keep: break } + let after = p.state + if (after == .off) != !applies { bad("I1 state \(after.rawValue) while applying \(applies)") } + if P.fillsScreen(before) != P.fillsScreen(after) { + if case .update = ev {} else { bad("I2 the panel resized on a reveal or hide") } + if !resized { bad("I2 resized without an update") } + } + if applies && held && after != .revealed { bad("I3 held but \(after.rawValue)") } + if timer && (held || !applies) { bad("I5 the timer left running while \(held ? "held" : "off")") } + if timer && after != .revealed { bad("I5 the timer running with the bar \(after.rawValue)") } + } +} +check("random: 20,000 sessions of 40 events hold I1–I5", broken.isEmpty) +for (k, v) in broken.sorted(by: { $0.key < $1.key }) { print(" \(v)× \(k)") } +check("random: the timer ran out sometimes", timerRuns > 1000) + +print("\(passes) passed, \(fails) failed") +exit(fails == 0 ? 0 : 1) diff --git a/Tests/checks/immersive/mutants.py b/Tests/checks/immersive/mutants.py new file mode 100644 index 000000000..c0720ad40 --- /dev/null +++ b/Tests/checks/immersive/mutants.py @@ -0,0 +1,69 @@ +"""Mutants of ImmersivePolicy.swift (iOSClient): each changes it in one place and must make the check +(main.swift) fail. usage: mutants.py""" +import os, subprocess, sys +SP = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.abspath(os.path.join(SP, "..", "..", "..")) # the repository +OUT = os.path.join(ROOT, ".build", "checks", "immersive") +os.makedirs(OUT, exist_ok=True) +SRC = os.path.join(ROOT, "iOSClient/ImmersivePolicy.swift") +orig = open(SRC).read() +APPLIES = "switchOn && keyboardAttached && landscape && streaming && !voiceOver && fillsScreen" +MUTANTS = [ + ("M1 applies with the switch off", APPLIES, "keyboardAttached && landscape && streaming && !voiceOver && fillsScreen"), + ("M2 applies with no keyboard", APPLIES, "switchOn && landscape && streaming && !voiceOver && fillsScreen"), + ("M3 applies upright", APPLIES, "switchOn && keyboardAttached && streaming && !voiceOver && fillsScreen"), + ("M4 applies with nothing streaming", APPLIES, "switchOn && keyboardAttached && landscape && !voiceOver && fillsScreen"), + ("M5 applies with VoiceOver", APPLIES, "switchOn && keyboardAttached && landscape && streaming && fillsScreen"), + ("M6 applies in a Stage Manager window", APPLIES, "switchOn && keyboardAttached && landscape && streaming && !voiceOver"), + ("M7 fills only one way round", " || (close(size.width, screen.height) && close(size.height, screen.width))\n", "\n"), + ("M8 fills with no tolerance", "abs(a - b) <= 1", "a == b"), + ("M9 fills with a loose tolerance", "abs(a - b) <= 1", "abs(a - b) <= 4"), + ("M10 held does not show the bar", "return revealed || held ? .revealed : .immersed", "return revealed ? .revealed : .immersed"), + ("M12 entering keeps an old reveal", " if !wasApplying {\n revealed = false\n", " if !wasApplying {\n"), + ("M13 leaving leaves the timer running", "guard nowApplies else { return .stop }", "guard nowApplies else { return .keep }"), + ("M14 held leaves the timer running", " if nowHeld { return .stop }", " if nowHeld { return .keep }"), + ("M15 the last holder closing snatches the bar", " if wasHeld {\n revealed = true\n", " if wasHeld {\n revealed = false\n"), + ("M16 the last holder closing starts no timer", " revealed = true\n return .start\n }\n return .keep", " revealed = true\n return .keep\n }\n return .keep"), + ("M17 the shortcut taken while off", "guard applies else { return (false, .keep) }", "guard applies else { return (true, .keep) }"), + ("M18 the shortcut acts while off", "guard applies else { return (false, .keep) }", ""), + ("M19 the shortcut hides a held bar", " guard !held else { return (true, .keep) }\n", ""), + ("M20 the shortcut given up while held", "guard !held else { return (true, .keep) }", "guard !held else { return (false, .keep) }"), + ("M21 a reveal starts no timer", "return (true, revealed ? .start : .stop)", "return (true, revealed ? .keep : .stop)"), + ("M22 a hide leaves the timer running", "return (true, revealed ? .start : .stop)", "return (true, revealed ? .start : .keep)"), + ("M23 the edge acts while off", "guard applies, !held else { return .keep }\n revealed = true", "guard !held else { return .keep }\n revealed = true"), + ("M24 the edge restarts the timer while held", "guard applies, !held else { return .keep }\n revealed = true", "guard applies else { return .keep }\n revealed = true"), + ("M25 a touch while hidden starts the timer", "applies && revealed && !held ? .start : .keep", "applies && !held ? .start : .keep"), + ("M26 a touch while held starts the timer", "applies && revealed && !held ? .start : .keep", "applies && revealed ? .start : .keep"), + ("M27 the timer hides nothing", " mutating func timerFired() {\n revealed = false\n", " mutating func timerFired() {\n"), + ("M29 the chord ignores shift", "(1 << 17) | (1 << 18) | (1 << 19) | (1 << 20)", "(1 << 18) | (1 << 19) | (1 << 20)"), + ("M30 the chord without option", "static let toggleModifiers: UInt64 = (1 << 18) | (1 << 19) | (1 << 20)", "static let toggleModifiers: UInt64 = (1 << 18) | (1 << 20)"), + ("M31 the chord any key", "usage == toggleUsage && modifiers & chordMask == toggleModifiers", "modifiers & chordMask == toggleModifiers"), + ("M32 the chord counts caps lock", "modifiers & chordMask == toggleModifiers", "modifiers == toggleModifiers"), + ("M33 the edge any height", "startY >= 0 && startY <= edgeZone && dy > 0 && dy > abs(dx)", "startY >= 0 && dy > 0 && dy > abs(dx)"), + ("M34 the edge sideways too", "startY >= 0 && startY <= edgeZone && dy > 0 && dy > abs(dx)", "startY >= 0 && startY <= edgeZone && dy > 0"), + ("M35 the edge upward too", "startY >= 0 && startY <= edgeZone && dy > 0 && dy > abs(dx)", "startY >= 0 && startY <= edgeZone && abs(dy) > abs(dx)"), + ("M36 the edge zone 40 pt", "static let edgeZone: Double = 20", "static let edgeZone: Double = 40"), + ("M37 auto-hide after 2 s", "static let autoHide: Double = 4", "static let autoHide: Double = 2"), + ("M38 the bar shown while immersed", "static func barShown(_ state: State) -> Bool { state != .immersed }", "static func barShown(_ state: State) -> Bool { true }"), + ("M39 revealed does not fill the screen", "static func fillsScreen(_ state: State) -> Bool { state != .off }", "static func fillsScreen(_ state: State) -> Bool { state == .immersed }"), +] +caught = 0 +for name, old, new in MUTANTS: + if orig.count(old) != 1: + print(f"{name}: NOT APPLIED (pattern found {orig.count(old)} times)"); continue + src = os.path.join(OUT, "ImmersivePolicy.swift") + open(src, "w").write(orig.replace(old, new, 1)) + exe = os.path.join(OUT, "mutant") + if os.path.exists(exe): os.remove(exe) + c = subprocess.run(["swiftc", "-O", src, os.path.join(SP, "main.swift"), "-o", exe], capture_output=True, text=True) + if c.returncode != 0: + print(f"{name}: did not compile: {c.stderr.strip().splitlines()[0] if c.stderr.strip() else ''}"); continue + r = subprocess.run([exe], capture_output=True, text=True, timeout=600) + failed = [l for l in r.stdout.splitlines() if l.startswith("FAIL")] + if r.returncode != 0: + caught += 1 + print(f"{name}: caught ({len(failed)} FAIL lines; first: {failed[0][:160] if failed else ''})") + else: + print(f"{name}: SURVIVED") +print(f"{caught} of {len(MUTANTS)} mutants caught") +sys.exit(0 if caught == len(MUTANTS) else 1) diff --git a/Tests/checks/immersive/run.sh b/Tests/checks/immersive/run.sh new file mode 100755 index 000000000..ad9968b60 --- /dev/null +++ b/Tests/checks/immersive/run.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# ImmersivePolicy (iOSClient/ImmersivePolicy.swift) on its own: when the iPad's Immersive mode applies, +# which of its three states the stream screen is in (off, immersed, revealed), what the shortcut, the +# top edge's swipe, a holder opening and closing and the auto-hide timer do, then 20,000 random +# sessions against the invariants (docs/immersive-plan.md). +# Tests/checks/immersive/run.sh compile and run the check +# Tests/checks/immersive/run.sh --mutants one-line mutants of ImmersivePolicy.swift; each must fail it +here="$(cd "$(dirname "$0")" && pwd)" +source "$here/../common.sh" +if [ "${1:-}" = "--mutants" ]; then run_mutants python3 "$here/mutants.py"; exit; fi +swiftc -O iOSClient/ImmersivePolicy.swift "$here/main.swift" -o "$out/check" +"$out/check" diff --git a/Tests/touchrig/main.swift b/Tests/touchrig/main.swift index 5cf249983..223870548 100644 --- a/Tests/touchrig/main.swift +++ b/Tests/touchrig/main.swift @@ -61,6 +61,8 @@ enum Step { case end([Int]) case cancel case wait(Int) + /// Runs between frames: what a scenario sets up before its touches. + case call(() -> Void) } enum Gesture { @@ -319,6 +321,7 @@ final class Runner { case .end(let idx): synth.end(IndexSet(idx)); RigLog.log("[touch] end \(idx.count) → \(synth.activeCount) down") case .cancel: synth.cancelAll(); RigLog.log("[touch] cancel all") case .wait(let n): waitLeft = n - 1 + case .call(let run): run() } } } @@ -338,6 +341,12 @@ final class AppDelegate: UIResponder, UIApplicationDelegate { final class Root: UIViewController { let trackpad = TrackpadSurface(frame: .zero) let overlay = InputOverlayView(frame: .zero) + #if IMMERSIVE + /// Immersive's top edge (docs/immersive-plan.md): an input view along the window's top, where the + /// edge's swipe begins, its proxy saying whether Immersive applies. + let edge = InputOverlayView(frame: .zero) + let edgeProxy = InputOverlayProxy() + #endif override func viewDidLoad() { super.viewDidLoad() view.backgroundColor = .black @@ -348,6 +357,14 @@ final class Root: UIViewController { trackpad.send = { RigLog.log("trackpad send \(describe($0))") } overlay.send = { RigLog.log("overlay send \(describe($0))") } overlay.videoSize = CGSize(width: 1512, height: 948) + #if IMMERSIVE + edge.backgroundColor = UIColor(white: 0.3, alpha: 1) + view.addSubview(edge) + edge.send = { RigLog.log("edge send \(describe($0))") } + edge.videoSize = CGSize(width: 1512, height: 948) + edge.keyboardProxy = edgeProxy + edgeProxy.immersiveEdge = { RigLog.log("edge immersive: the bar revealed") } + #endif #if GESTURES let on = ProcessInfo.processInfo.environment["RIG_SWITCH"] != "off" trackpad.sendGesture = { g, n in RigLog.log("trackpad gesture \(g.rawValue) fingers \(n)" + (on ? "" : " (switch off: not sent)")); return on } @@ -360,6 +377,10 @@ final class Root: UIViewController { let h = (b.height - 16) / 2 trackpad.frame = CGRect(x: b.minX, y: b.minY, width: b.width, height: h) overlay.frame = CGRect(x: b.minX, y: b.minY + h + 16, width: b.width, height: h) + #if IMMERSIVE + // The strip above the trackpad, from the window's very top: no other surface's touch lands in it. + edge.frame = CGRect(x: b.minX, y: 0, width: b.width, height: b.minY - 2) + #endif } } @@ -396,6 +417,23 @@ final class SceneDelegate: UIResponder, UIWindowSceneDelegate { let oc = CGPoint(x: root.overlay.bounds.midX, y: root.overlay.bounds.midY + 60) for (title, steps) in scenarios(tc) { runner.add("TRACKPAD \(title)", root.trackpad, steps) } for (title, steps) in scenarios(oc) { runner.add("OVERLAY \(title)", root.overlay, steps) } + #if IMMERSIVE + // Immersive's edge: from 5 pt below the window's top (in the 20 pt zone) and from 30 (out of + // it), down and sideways, with Immersive applying and not. Only the first reveals the bar, + // and nothing of it is sent; the rest are the strokes they were. + print("RIG edge \(root.edge.frame)") + let top: (CGFloat) -> CGPoint = { y in root.edge.convert(CGPoint(x: w.bounds.midX, y: y), from: w) } + let edgeCases: [(String, Bool, CGPoint, CGFloat, CGFloat)] = [ + ("on: 1-finger drag down 120 from 5 pt", true, top(5), 0, 120), + ("on: 1-finger drag down 120 from 30 pt", true, top(30), 0, 120), + ("on: 1-finger drag right 120 down 40 from 5 pt", true, top(5), 120, 40), + ("off: 1-finger drag down 120 from 5 pt", false, top(5), 0, 120), + ] + for (title, on, at, dx, dy) in edgeCases { + runner.add("IMMERSIVE \(title)", root.edge, [.call { root.edgeProxy.immersiveActive = on }] + + Gesture.stroke([at], to: [CGPoint(x: at.x + dx, y: at.y + dy)], frames: 12)) + } + #endif runner.start { print("RIG done"); exit(0) } } } diff --git a/Tests/touchrig/run.sh b/Tests/touchrig/run.sh index f3a5a3a64..7c1a2d4b6 100755 --- a/Tests/touchrig/run.sh +++ b/Tests/touchrig/run.sh @@ -42,7 +42,7 @@ fetch() { } for f in iOSClient/TrackpadView.swift iOSClient/InputOverlay.swift iOSClient/PointerPresence.swift \ iOSClient/TrackpadGestures.swift iOSClient/KeyChords.swift iOSClient/PortraitStreamScreen.swift \ - iOSClient/HardwareKeyboardPolicy.swift Sources/StreamProtocol/Input.swift; do + iOSClient/HardwareKeyboardPolicy.swift iOSClient/ImmersivePolicy.swift Sources/StreamProtocol/Input.swift; do fetch "$f" done cd "$work/src" @@ -55,12 +55,14 @@ rm PortraitStreamScreen.swift flags=() # The gestures when the surfaces have them (the stroke gate and `sendGesture`). if grep -q "var sendGesture" TrackpadView.swift; then flags+=(-D GESTURES); fi +# Immersive's top edge (docs/immersive-plan.md): its scenarios, once the input view has it. +if grep -q "immersiveEdge" InputOverlay.swift; then flags+=(-D IMMERSIVE); fi cp "$here/Stubs.swift" "$here/main.swift" "$here/TouchSynth.h" "$here/TouchSynth.m" "$here/Bridging.h" . target=arm64-apple-ios17.0-simulator xcrun -sdk iphonesimulator clang -target $target -fobjc-arc -Wall -c TouchSynth.m -o TouchSynth.o xcrun -sdk iphonesimulator swiftc -target $target -swift-version 5 -module-name TouchRig -Onone ${flags[@]+"${flags[@]}"} \ - -import-objc-header Bridging.h Stubs.swift Keys.swift Input.swift $(ls PointerPresence.swift TrackpadGestures.swift KeyChords.swift HardwareKeyboardPolicy.swift 2>/dev/null) \ + -import-objc-header Bridging.h Stubs.swift Keys.swift Input.swift $(ls PointerPresence.swift TrackpadGestures.swift KeyChords.swift HardwareKeyboardPolicy.swift ImmersivePolicy.swift 2>/dev/null) \ TrackpadView.swift InputOverlay.swift main.swift TouchSynth.o -o "$work/TouchRig.app/TouchRig" cat > "$work/TouchRig.app/Info.plist" <<'PLIST' diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 6213f9fd4..f7acae2fe 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -997,6 +997,16 @@ simulator: `-SillKeyProbe 1` adds GameController's own view of each key, `-SillEscViaCommand 1` hands Esc on from `pressesBegan` so its key commands take it, and `-SillTurnAt 6,12` turns an iPhone's scene at those seconds. +- Immersive (docs/immersive-plan.md): on an iPad `-SillLayout` screen held + sideways, with `-SillHardwareKeyboard 1` and a live session. The console + prints `immersive: …` for every state change (and why it does not apply), + and `viewport: … device desktop` while the Desktop asks for a display of its + own. ⌃⌥⌘B through `Tests/keyrig` is + `keys e0:down,e2:down,e3:down,05:down,05:up,e3:up,e2:up,e0:up`. + `-SillImmersiveExitAt ` taps the revealed bar's Exit s seconds in. To + test Exit, set the switch in the app's preferences plist rather than with + `-Sill.immersive YES`: an argument masks the app's own write, so Exit + would look like it did nothing. - `-SillInputTest 1`, with `-SillConnect` on this Mac's loopback: the portrait key row taps cmd, esc, shift and ctrl and the trackpad strokes, taps and scrolls, through their own code, once; in every layout the stream's input diff --git a/docs/immersive-plan.md b/docs/immersive-plan.md new file mode 100644 index 000000000..a9e48d2ea --- /dev/null +++ b/docs/immersive-plan.md @@ -0,0 +1,105 @@ +# Immersive on an iPad with a hardware keyboard — the plan + +2026-10-18. Branch `immersive` from origin/main at 5c323e6. Proposed by a contributor (Pearse +Taggart), from a PRD written against main on 2026-10-17; no issue yet. It carries the iPhone's +Larger Picture idea (issue #58, PR #79, docs/larger-picture-plan.md) to the iPad, which #79 left +unchanged. It builds on the host half of PR #156 (dr-apple, "Desktop in full screen on a display of +the device's own size"), carried here as a commit of its own under dr-apple's name: if #156 lands +first, a rebase drops that commit and nothing else changes. + +## The problem + +On an iPad Pro with a Magic Keyboard, sideways, the stream screen is `TopBar` (86 pt, +`BarMetrics.regular`) over `contentArea`, which insets the stream 8 pt and rounds it 12 pt, under +the status bar. With a keyboard and trackpad attached most of the bar is redundant, and on iPadOS +26 and later the Mac app's menus are in the iPad's own menu bar too. Someone using the iPad as a thin +client to a Mac wants the Mac's picture on the whole display, and the bar back only when asked for. + +The Desktop has a second problem: it keeps the Mac screen's shape (16:10 or 16:9), so on a 4:3 iPad +it letterboxes whatever the layout does. Only the Mac can fix that, by giving the Desktop a display +of the iPad's shape, which is what #156's host side does. + +## Decision + +### In one paragraph + +Settings › This iPad gains **Immersive** (off by default; `Sill.immersive`, +`StreamClient.immersiveKey`, never sent; offered on an iPad only, `immersiveOffered`). With it on, a +hardware keyboard attached, the iPad sideways, something streaming, VoiceOver off and Sill filling +the screen (`ImmersivePolicy.applies`), the landscape stream screen drops the bar, the inset, the +frame, the status bar and the home indicator, and the panel is the whole screen: one viewport, and +the Mac refits a window as after a rotation. ⌃⌥⌘B or a swipe down from the top edge brings the bar +back over the picture without resizing it; it hides after 4 s untouched, or on ⌃⌥⌘B again. The +revealed bar has an **Exit** button that turns the switch off at once. While the Desktop streams, +the viewport also says `deviceDesktop`, and a host that has #156's `DeviceDesktop` makes a display of +the iPad's size the Mac's main display, so the Desktop fills the iPad with no letterbox. + +### The rule + +1. Immersive applies only with all of: the switch on (as it was when Settings opened, while Settings + is open: `immersiveHeld`, as `largerPictureHeld`), a hardware keyboard attached + (`HardwareKeyboardMonitor`), the iPad sideways (`.innerLandscape`), a window or the Desktop + streaming (nothing streaming keeps the bar, so a first picture never lands in a layout that then + flips: Larger Picture's rule 12), VoiceOver off, and Sill's scene filling the screen + (`ImmersivePolicy.fills`: not a Stage Manager or Split View window). +2. Three states (`ImmersivePolicy.State`): `off`, main's layout; `immersed`, no bar and the panel the + whole screen; `revealed`, the bar over that same panel. Only a move between `off` and the other + two changes the panel's size. Revealing and hiding never send a viewport. +3. One view tree in all three (`landscape(…immersive:)`): the bar is drawn over a spacer of its own + height when off, so toggling never rebuilds the stream's input view and the hardware keyboard's + hold survives. +4. ⌃⌥⌘B (`ImmersivePolicy.isToggle`: HID 0x05 with control, option and command) toggles `immersed` and `revealed`, through both + a `UIKeyCommand` (priority over system behaviour) and `pressesBegan`, de-duplicated, and never + reaches the Mac while Immersive applies. When it does not apply, the chord is the Mac's. +5. A downward drag that starts within 20 pt of the top edge reveals the bar + (`ImmersivePolicy.revealsFromEdge`, `edgeZone`, judged from the touch's landing point). The top edge's system + gestures are deferred while Immersive applies, so the first swipe is Sill's, not Notification + Centre's. +6. Anything that opens over the stream (Settings, pairing, the tour, the Apps drawer, Menus, the Aa + ruler, a thumbnail's lights) holds the bar revealed until it closes; the last one closing leaves it + up for the full 4 s. +7. Exit (the revealed bar only, between Desktop and Settings) closes Settings if open, then turns the + switch off: the layout goes back to main's in one motion and one viewport. +8. The Desktop while Immersive applies: the viewport carries `deviceDesktop: true`; leaving Immersive + or picking a window sends one without it. An older host ignores the key and letterboxes as before. + +## Open questions (the default is taken unless Noah says otherwise) + +1. The shortcut. ⌃⌘F is macOS's Enter Full Screen and stays the Mac's; ⌃⌥⌘B is the default. +2. Immersive needs a hardware keyboard. Touch-only Immersive would hide the trackpad's controls. + Default: keyboard only. +3. Portrait, whose key row and trackpad could go with a keyboard attached. Default: later. +4. The device desktop makes a virtual display the Mac's MAIN display while it streams: the menu bar, + the Dock and windows move to it, and back when it goes. That is #156's design (Screen Sharing's + High Performance mode does the same). Default: taken with #156; a host setting could refuse it. +5. iPadOS-reserved chords (⌘Tab, ⌘Space, ⌘H) never reach `pressesBegan`. Out of scope. + +## Results + +- `Tests/checks/immersive`: 71 checks, 37 of 37 mutants caught (in CI's matrix). + `Tests/checks/run-all.sh`, all 45; `protocol` and `compatibility` with `deviceDesktop`. +- The iOS app Debug for the simulator and for a device, Release for the simulator; `swift build -c + release`; known warnings only. +- The touch rig against main's: 72 of 72 (the top edge's scenarios behind `-D IMMERSIVE`). +- Harness photos with the switch off against main's build (iPad Pro 13" simulator, mock Mac): + sideways, sideways with no keyboard and the switch on, upright, upright with the drawer: + identical. Sideways with the drawer open: within 16/255 at the edges of three thumbnail badges + (anti-aliasing), the same in two runs of each build. Settings: the new Immersive row and + footnote. +- Harness, iPad Pro 13" simulator, `-SillLayout 1376x1032`, live against a host on loopback: + switch off, `viewport: 1360×930`, main's layout; on with `-SillHardwareKeyboard 1`, one + `viewport: 1376×1032 … device desktop`, the host's "Device desktop 1376×1032 pt @2× … main" and + "Streaming the device's own desktop … 2752×2064"; ⌃⌥⌘B revealed and the bar hid itself after 4 s + with no new viewport and no capture restart; Exit, `immersive: Exit: the switch off`, one + `viewport: 1360×930`; quitting removed the display and the real one was main again. +- On an M4 iPad Pro 13" with the Magic Keyboard against `SillHost --virtual-display --pairing` + from this branch (2026-10-18): the Desktop edge to edge, the bar on demand, Exit back to the + standard layout. + +## Untested + +- Rotation and an Aa step while immersed on the Desktop (the host should remake its display). +- Two devices at once, one immersed on the Desktop. +- Stage Manager and Split View on a device (judged by `fills`; tested only by the harness bypass). +- VoiceOver on a device. +- Sill.app (rather than the CLI host) from this branch. diff --git a/iOSClient/HostSettingsPanel.swift b/iOSClient/HostSettingsPanel.swift index 215902a4c..c5a59ab35 100644 --- a/iOSClient/HostSettingsPanel.swift +++ b/iOSClient/HostSettingsPanel.swift @@ -47,6 +47,10 @@ struct HostSettingsPanel: View { /// Whether the switch is offered here, the stream screen's say (`largerPictureOffered`): only on an /// iPhone without a hinge whose picture it makes larger. @Environment(\.largerPictureOffered) private var offersLargerPicture + /// This iPad's Immersive switch (docs/immersive-plan.md): off unless turned on, read by the stream + /// screen, never sent; offered on an iPad only (`immersiveOffered`). + @AppStorage(StreamClient.immersiveKey) private var immersive = false + @Environment(\.immersiveOffered) private var offersImmersive /// This device's switch for the clipboard (docs/clipboard-plan.md): on unless turned off, read by /// StreamClient+Clipboard at each decision; a change subscribes or stops at once. @AppStorage(StreamClient.clipboardKey) private var clipboardOn = true @@ -485,6 +489,15 @@ struct HostSettingsPanel: View { } Footnote(text: Self.largerPictureFooter) } + if offersImmersive { + Rows { + Toggle(isOn: $immersive) { + Text("Immersive").foregroundStyle(Palette.text) + } + .rowFrame() + } + Footnote(text: Self.immersiveFooter) + } // The clipboard (docs/clipboard-plan.md): only while the Mac says it shares it. if ClipboardPolicy.hostTakes(client.hostClipboard) { Rows { @@ -530,6 +543,8 @@ struct HostSettingsPanel: View { /// it gives up, where windows come from instead, that the Mac fits a window to the taller picture /// once Aa sets its text size (or on the virtual display), and that a change applies as the panel /// closes (StreamScreen's `largerPictureHeld`). + static let immersiveFooter = "With a hardware keyboard attached and the iPad held sideways, the picture fills the whole screen and the bar hides. Press Control-Option-Command-B, or swipe down from the top edge, to show it; Exit in the bar turns Immersive off. A change here applies when you close Settings." + static let largerPictureFooter = "Held upright, a window’s picture takes the place of the window thumbnails and reaches down to where the keyboard comes up. The Desktop keeps the thumbnails. Switch windows with Apps: tap an open app again for its next window. To fill the taller picture with a window, set its text size with Aa. A change applies when you close Settings." /// What each gesture does on the Mac, and what the row does as an accessibility action: its one @@ -891,3 +906,16 @@ extension EnvironmentValues { set { self[LargerPictureOfferedKey.self] = newValue } } } + +/// Whether the Settings panel offers Immersive (This iPad; docs/immersive-plan.md): the stream screen's +/// say, true on an iPad, false on every iPhone. +private struct ImmersiveOfferedKey: EnvironmentKey { + static let defaultValue = false +} + +extension EnvironmentValues { + var immersiveOffered: Bool { + get { self[ImmersiveOfferedKey.self] } + set { self[ImmersiveOfferedKey.self] = newValue } + } +} diff --git a/iOSClient/ImmersivePolicy.swift b/iOSClient/ImmersivePolicy.swift new file mode 100644 index 000000000..aa779913b --- /dev/null +++ b/iOSClient/ImmersivePolicy.swift @@ -0,0 +1,144 @@ +import Foundation + +/// Immersive, the iPad's full-screen stream (docs/immersive-plan.md): with this device's switch on and a +/// hardware keyboard attached, the landscape stream screen drops its top bar, the panel's 8 pt inset, +/// its rounded frame and the status bar, and the picture fills the display edge to edge. The bar comes +/// back over the picture on demand (⌃⌥⌘B, or a swipe down from the top edge) without resizing it. +/// +/// Three states. `off`: today's layout. `immersed`: no bar, the panel the whole screen. `revealed`: the +/// bar floating over that same whole-screen panel. Only a move between `off` and the other two changes +/// the panel's size (one viewport to the Mac); revealing and hiding the bar never does. +/// +/// The stream screen owns one, tells it whether Immersive applies (`applies(…)`) and whether something +/// open over the stream holds the bar (the Settings panel, the pairing overlay, the tour, the Apps drawer, +/// the Menus pull-down, the Aa ruler, a thumbnail's lights), and runs the auto-hide timer it asks for. +/// Pure: Foundation only, checked in Tests/checks/immersive. +struct ImmersivePolicy: Equatable { + enum State: String, CaseIterable, Equatable { + case off, immersed, revealed + } + + /// What the auto-hide timer does after an event: nothing, (re)start its `autoHide` seconds, or stop. + enum Timer: String, Equatable { + case keep, start, stop + } + + // MARK: When it applies + + /// Immersive applies: the switch on (as it was when the Settings panel opened, while it is open), a + /// hardware keyboard attached, the iPad held sideways, a window or the Desktop streaming (nothing + /// streaming keeps the standard layout, so the first picture never lands in a layout that then + /// flips: Larger Picture's rule), VoiceOver off (every control stays in reach), and Sill filling + /// the screen (not a Stage Manager or Split View window). + static func applies(switchOn: Bool, keyboardAttached: Bool, landscape: Bool, streaming: Bool, + voiceOver: Bool, fillsScreen: Bool) -> Bool { + switchOn && keyboardAttached && landscape && streaming && !voiceOver && fillsScreen + } + + /// Whether a scene of `size` points fills a screen of `screen` points (either way round, within a + /// point): Stage Manager and Split View windows do not. + static func fills(size: (width: Double, height: Double), screen: (width: Double, height: Double)) -> Bool { + guard size.width > 0, size.height > 0, screen.width > 0, screen.height > 0 else { return false } + func close(_ a: Double, _ b: Double) -> Bool { abs(a - b) <= 1 } + return (close(size.width, screen.width) && close(size.height, screen.height)) + || (close(size.width, screen.height) && close(size.height, screen.width)) + } + + // MARK: The state + + /// Immersive applies now (`applies(…)`, as the stream screen last said). + private(set) var applies = false + /// Something open over the stream holds the bar. + private(set) var held = false + /// The bar was revealed on demand (the shortcut, the edge swipe, a holder's closing). + private(set) var revealed = false + + var state: State { + guard applies else { return .off } + return revealed || held ? .revealed : .immersed + } + + /// The panel fills the screen: no bar beside it, no inset, no frame, no status bar. + static func fillsScreen(_ state: State) -> Bool { state != .off } + /// The bar is on screen: beside the panel (`off`) or over it (`revealed`). + static func barShown(_ state: State) -> Bool { state != .immersed } + + /// The bar, once revealed, hides after this many seconds with nothing touched on it. + static let autoHide: Double = 4 + + // MARK: Events + + /// What applies, and what holds the bar, changed (the stream screen sends both together on any change + /// of either). Entering hides the bar (the panel grows to the edges in the same motion) unless + /// something holds it, forgetting any reveal from before. The last holder closing leaves the bar up for + /// `autoHide` seconds rather than snatching it away under the finger that closed it. + mutating func update(applies nowApplies: Bool, held nowHeld: Bool) -> Timer { + let wasApplying = applies, wasHeld = held + applies = nowApplies + held = nowHeld + guard nowApplies else { return .stop } + if !wasApplying { + revealed = false + return .stop + } + if nowHeld { return .stop } + if wasHeld { + revealed = true + return .start + } + return .keep + } + + /// The shortcut: reveal or hide the bar. False when Immersive does not apply, and the chord is the + /// Mac's like any other. While something holds the bar it stays (the chord is still taken, so it + /// never reaches the Mac half the time). + mutating func toggle() -> (taken: Bool, timer: Timer) { + guard applies else { return (false, .keep) } + guard !held else { return (true, .keep) } + revealed.toggle() + return (true, revealed ? .start : .stop) + } + + /// A swipe down from the top edge: reveals the bar, or keeps a revealed one up longer. + mutating func edgeSwipe() -> Timer { + guard applies, !held else { return .keep } + revealed = true + return .start + } + + /// A touch on the revealed bar: its auto-hide starts again. + func touched() -> Timer { + applies && revealed && !held ? .start : .keep + } + + /// The auto-hide timer ran out: the reveal ends (a bar something holds stays, by `state`). + mutating func timerFired() { + revealed = false + } + + // MARK: The shortcut and the edge + + /// HID usage of B (keyboard page). + static let toggleUsage: UInt16 = 0x05 + /// ⌃⌥⌘, as UIKeyModifierFlags and KeyModifiers carry them: control 1<<18, option 1<<19, command 1<<20. + static let toggleModifiers: UInt64 = (1 << 18) | (1 << 19) | (1 << 20) + /// Shift, control, option and command: the flags a chord is told by (caps lock and the keypad bit + /// are not). + private static let chordMask: UInt64 = (1 << 17) | (1 << 18) | (1 << 19) | (1 << 20) + + /// ⌃⌥⌘B, and only that: not ⇧⌃⌥⌘B, not ⌃⌘B, whatever caps lock says. (⌃⌘F is macOS's own Enter Full + /// Screen, so it stays the Mac's.) + static func isToggle(usage: UInt16, modifiers: UInt64) -> Bool { + usage == toggleUsage && modifiers & chordMask == toggleModifiers + } + + /// The edge swipe's zone: a drag must start within this many points of the screen's top edge. + static let edgeZone: Double = 20 + + /// A finger drag that started `startY` points from the top edge and has moved `dx`, `dy`: the edge + /// swipe when it began in the zone and goes mostly down. Anything else (a tap, a sideways drag on + /// a browser's tabs, a drag that starts lower) stays the Mac's. + static func revealsFromEdge(startY: Double, dx: Double, dy: Double) -> Bool { + startY >= 0 && startY <= edgeZone && dy > 0 && dy > abs(dx) + } +} diff --git a/iOSClient/InputOverlay.swift b/iOSClient/InputOverlay.swift index 3512064e9..792cd8416 100644 --- a/iOSClient/InputOverlay.swift +++ b/iOSClient/InputOverlay.swift @@ -207,9 +207,38 @@ final class InputOverlayView: UIView, UIKeyInput, UIGestureRecognizerDelegate { /// positive dy). @objc private func handlePan(_ gesture: UIPanGestureRecognizer) { guard !strokes.silent else { return } + // Immersive's top edge (docs/immersive-plan.md): a drag that began within 20 pt of the + // screen's top and goes mostly down reveals the bar, and none of it scrolls the Mac. + if gesture.state == .began { edgeSwipeTaken = edgeSwipe(gesture) } + if edgeSwipeTaken { + if gesture.state == .ended || gesture.state == .cancelled || gesture.state == .failed { edgeSwipeTaken = false } + return + } scroll(gesture, .finger) } + /// The finger pan in progress revealed Immersive's bar (`edgeSwipe`): the rest of it is not a scroll. + private var edgeSwipeTaken = false + /// Where the finger pan's first finger landed, in the window's space. + private var panLanding: CGPoint? + + /// Whether the pan beginning now is Immersive's edge swipe, and if so, the bar revealed: only while + /// Immersive applies (`keyboardProxy.immersiveActive`), judged in the window's space, which is the + /// screen's while Immersive applies (it applies only when Sill fills the screen). + private func edgeSwipe(_ gesture: UIPanGestureRecognizer) -> Bool { + // From where the finger landed (`panLanding`) to where the pan begins: UIKit begins it only + // after the finger has moved, and with a translation of zero. + guard let proxy = keyboardProxy, proxy.immersiveActive, let window, let start = panLanding else { return false } + let now = gesture.location(in: window) + guard ImmersivePolicy.revealsFromEdge(startY: Double(start.y), dx: Double(now.x - start.x), dy: Double(now.y - start.y)) + else { return false } + #if DEBUG + print("immersive: the top edge's swipe (from \(Int(start.y.rounded())) pt), not a scroll") + #endif + proxy.immersiveEdge?() + return true + } + /// The iPad's trackpad (two fingers) or a Magic Mouse over the picture: the finger pan's path, at /// the pointer, with its lift's coast. iPadOS hands a pan no momentum of its own (a scroll view /// decelerates by itself; in the simulator, scroll events marked as momentum after a scroll's end @@ -300,7 +329,12 @@ final class InputOverlayView: UIView, UIKeyInput, UIGestureRecognizerDelegate { /// The pointer's scrolls take scroll events only: no touch at all (UIKit asks this for touches, not /// for scroll events). func gestureRecognizer(_ gestureRecognizer: UIGestureRecognizer, shouldReceive touch: UITouch) -> Bool { - !(gestureRecognizer === trackpadScroll || gestureRecognizer === wheelScroll) + // The finger pan's first finger, where it landed: Immersive's edge swipe is judged from there + // (`edgeSwipe`), the pan's own `.began` coming only after it has moved. + if gestureRecognizer === scrollPan, gestureRecognizer.numberOfTouches == 0 { + panLanding = touch.location(in: window) + } + return !(gestureRecognizer === trackpadScroll || gestureRecognizer === wheelScroll) } /// The pointer's scrolls and its hover run together: the pointer rests over the picture while two @@ -661,15 +695,72 @@ final class InputOverlayView: UIView, UIKeyInput, UIGestureRecognizerDelegate { private var takesEscape: Bool { isFirstResponder && !EscapeKey.onScreen(in: window) } override var keyCommands: [UIKeyCommand]? { - guard takesEscape else { return super.keyCommands } - return (super.keyCommands ?? []) + Self.escapeCommands + var commands = super.keyCommands ?? [] + if takesEscape { commands += Self.escapeCommands } + if takesImmersiveToggle { commands.append(Self.immersiveCommand) } + return commands.isEmpty ? super.keyCommands : commands } override func canPerformAction(_ action: Selector, withSender sender: Any?) -> Bool { if action == #selector(InputOverlayView.escapeKeyCommand(_:)) { return takesEscape } + if action == #selector(InputOverlayView.immersiveKeyCommand(_:)) { return takesImmersiveToggle } return super.canPerformAction(action, withSender: sender) } + // MARK: Immersive's shortcut (docs/immersive-plan.md) + + /// ⌃⌥⌘B shows or hides the bar while Immersive applies, and never reaches the Mac (R5). Two ways + /// in, as Esc's: this command, with priority over the system's own handling and titled for the + /// ⌘ list and the menu bar, and `pressesBegan` (`takeImmersiveToggle`), whichever comes first; + /// one press toggles once. While Immersive does not apply the chord is the Mac's like any other. + private static let immersiveCommand: UIKeyCommand = { + let command = UIKeyCommand(title: "Show or Hide the Sill Bar", action: #selector(InputOverlayView.immersiveKeyCommand(_:)), + input: "b", modifierFlags: [.control, .alternate, .command]) + command.wantsPriorityOverSystemBehavior = true + return command + }() + + /// This view takes the chord: it holds the keyboard and Immersive applies. + private var takesImmersiveToggle: Bool { isFirstResponder && keyboardProxy?.immersiveActive == true } + + /// The chord's keys taken from pressesBegan, so their ups are taken too. + private var immersiveKeysDown: Set = [] + /// When the chord last toggled, so the command and pressesBegan never both do for one press. + private var immersiveToggledAt: TimeInterval = -1 + + @objc private func immersiveKeyCommand(_ command: UIKeyCommand) { + guard takesImmersiveToggle else { return } + toggleImmersive(via: "keyCommand") + } + + private func toggleImmersive(via channel: String) { + let now = ProcessInfo.processInfo.systemUptime + guard now - immersiveToggledAt > 0.2 else { + Self.hwkey(ImmersivePolicy.toggleUsage, down: true, via: channel, to: "the bar's shortcut, already toggled") + return + } + immersiveToggledAt = now + Self.hwkey(ImmersivePolicy.toggleUsage, down: true, via: channel, to: "the bar's shortcut") + keyboardProxy?.immersiveToggle?() + } + + /// The presses that are not Immersive's chord: the chord itself (down, and later its up) toggles + /// the bar here and goes no further, while Immersive applies. + private func takeImmersiveToggle(_ presses: Set, down: Bool) -> Set { + guard down ? takesImmersiveToggle : !immersiveKeysDown.isEmpty else { return presses } + return presses.filter { press in + guard let key = press.key else { return true } + let usage = UInt16(key.keyCode.rawValue) + if down { + guard ImmersivePolicy.isToggle(usage: usage, modifiers: UInt64(key.modifierFlags.rawValue)) else { return true } + immersiveKeysDown.insert(usage) + toggleImmersive(via: "pressesBegan") + return false + } + return immersiveKeysDown.remove(usage) == nil + } + } + /// Esc to the Mac, down and up at once with the command's modifiers, through `ForwardedKeys` as a /// pressed key goes; nothing when its press came first (its own up follows). @objc private func escapeKeyCommand(_ command: UIKeyCommand) { @@ -762,11 +853,15 @@ final class InputOverlayView: UIView, UIKeyInput, UIGestureRecognizerDelegate { override func pressesBegan(_ presses: Set, with event: UIPressesEvent?) { // A hardware key hides this device's pointer, as typing does (`setOwnPointer`). if presses.contains(where: { $0.key != nil }) { setOwnPointer(nil, .none) } + let presses = takeImmersiveToggle(presses, down: true) + guard !presses.isEmpty else { return } let unhandled = forward(presses, down: true, via: "pressesBegan") if !unhandled.isEmpty { super.pressesBegan(unhandled, with: event) } } override func pressesEnded(_ presses: Set, with event: UIPressesEvent?) { + let presses = takeImmersiveToggle(presses, down: false) + guard !presses.isEmpty else { return } let unhandled = forward(presses, down: false, via: "pressesEnded") if !unhandled.isEmpty { super.pressesEnded(unhandled, with: event) } } @@ -774,6 +869,8 @@ final class InputOverlayView: UIView, UIKeyInput, UIGestureRecognizerDelegate { /// A press cancelled (the app going to the background, the keyboard disconnected): its key comes /// up on the Mac as if released, so neither it nor a modifier stays down there. override func pressesCancelled(_ presses: Set, with event: UIPressesEvent?) { + let presses = takeImmersiveToggle(presses, down: false) + guard !presses.isEmpty else { return } let unhandled = forward(presses, down: false, via: "pressesCancelled") if !unhandled.isEmpty { super.pressesCancelled(unhandled, with: event) } } @@ -800,6 +897,11 @@ final class InputOverlayProxy { /// The person's own keyboard came up or went: the Keyboard button lights, the tour counts it. /// The hardware keyboard's holding never calls it. var onOwnChange: ((Bool) -> Void)? + /// Immersive applies (StreamScreen, docs/immersive-plan.md): the input view takes ⌃⌥⌘B and the top + /// edge's swipe for the bar, through these, instead of sending them to the Mac. + var immersiveActive = false + var immersiveToggle: (() -> Void)? + var immersiveEdge: (() -> Void)? /// The Keyboard button (and the portrait key row's keyboard cap). func toggleKeyboard() { change("the Keyboard button") { $0.toggled() } } diff --git a/iOSClient/Sill.xcodeproj/project.pbxproj b/iOSClient/Sill.xcodeproj/project.pbxproj index 41c841cb2..722357972 100644 --- a/iOSClient/Sill.xcodeproj/project.pbxproj +++ b/iOSClient/Sill.xcodeproj/project.pbxproj @@ -19,6 +19,7 @@ A1000001000000000000A501 /* PhonePortraitLayout.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1000001000000000000F501 /* PhonePortraitLayout.swift */; }; A1000001000000000000A901 /* KeyChords.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1000001000000000000F901 /* KeyChords.swift */; }; A1000001000000000000AC01 /* HardwareKeyboardPolicy.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1000001000000000000FC01 /* HardwareKeyboardPolicy.swift */; }; + A1000001000000000000AC77 /* ImmersivePolicy.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1000001000000000000FC77 /* ImmersivePolicy.swift */; }; A1000001000000000000A00A /* TrackpadView.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1000001000000000000F00B /* TrackpadView.swift */; }; A1000001000000000000A701 /* TrackpadGestures.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1000001000000000000F701 /* TrackpadGestures.swift */; }; A1000001000000000000A00B /* MockCatalog.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1000001000000000000F00C /* MockCatalog.swift */; }; @@ -72,6 +73,7 @@ A1000001000000000000F501 /* PhonePortraitLayout.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PhonePortraitLayout.swift; sourceTree = ""; }; A1000001000000000000F901 /* KeyChords.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeyChords.swift; sourceTree = ""; }; A1000001000000000000FC01 /* HardwareKeyboardPolicy.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HardwareKeyboardPolicy.swift; sourceTree = ""; }; + A1000001000000000000FC77 /* ImmersivePolicy.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ImmersivePolicy.swift; sourceTree = ""; }; A1000001000000000000F00B /* TrackpadView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TrackpadView.swift; sourceTree = ""; }; A1000001000000000000F701 /* TrackpadGestures.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TrackpadGestures.swift; sourceTree = ""; }; A1000001000000000000F00C /* MockCatalog.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MockCatalog.swift; sourceTree = ""; }; @@ -133,6 +135,7 @@ A1000001000000000000F501 /* PhonePortraitLayout.swift */, A1000001000000000000F901 /* KeyChords.swift */, A1000001000000000000FC01 /* HardwareKeyboardPolicy.swift */, + A1000001000000000000FC77 /* ImmersivePolicy.swift */, A1000001000000000000F013 /* HostSettingsPanel.swift */, A1000001000000000000FC64 /* StreamClient+Clipboard.swift */, A1000001000000000000FD64 /* ClipboardChip.swift */, @@ -277,6 +280,7 @@ A1000001000000000000A501 /* PhonePortraitLayout.swift in Sources */, A1000001000000000000A901 /* KeyChords.swift in Sources */, A1000001000000000000AC01 /* HardwareKeyboardPolicy.swift in Sources */, + A1000001000000000000AC77 /* ImmersivePolicy.swift in Sources */, A1000001000000000000A013 /* HostSettingsPanel.swift in Sources */, A1000001000000000000AC64 /* StreamClient+Clipboard.swift in Sources */, A1000001000000000000AD64 /* ClipboardChip.swift in Sources */, diff --git a/iOSClient/StreamClient.swift b/iOSClient/StreamClient.swift index f90764736..852b66242 100644 --- a/iOSClient/StreamClient.swift +++ b/iOSClient/StreamClient.swift @@ -3311,6 +3311,10 @@ final class StreamClient: ObservableObject { /// the Mac; the stream screen reads it with what streams (`active`): it acts only while a window /// does (`PhonePortraitLayout.largerPicture(switchOn:streaming:)`). static let largerPictureKey = "Sill.largerPicture" + /// This iPad's Immersive switch (docs/immersive-plan.md): off unless turned on. The device's own + /// preference, never sent to the Mac; the stream screen reads it with a hardware keyboard, the + /// iPad's turn and what streams (`ImmersivePolicy.applies`). + static let immersiveKey = "Sill.immersive" /// A three- or four-finger gesture a surface decided (TrackpadGestures, StrokeObserver), to the /// Mac as kind 28, which turns it into its own shortcut (docs/trackpad-gestures-plan.md §6.3): diff --git a/iOSClient/StreamScreen.swift b/iOSClient/StreamScreen.swift index 9f0baa6c3..43458e5cf 100644 --- a/iOSClient/StreamScreen.swift +++ b/iOSClient/StreamScreen.swift @@ -180,6 +180,21 @@ struct StreamScreen: View { /// streams changing meanwhile (a window closed on the Mac, another device's pick). It applies as /// the panel goes, in the same motion. @State private var largerPictureHeld: Bool? = nil + /// This iPad's Immersive switch (the Settings panel's This iPad; docs/immersive-plan.md). It acts + /// only with a hardware keyboard, sideways, while something streams (`immersiveApplies`). + @AppStorage(StreamClient.immersiveKey) private var immersiveSwitch = false + /// The switch as it was when the Settings panel opened, kept until it closes (as + /// `largerPictureHeld`): a change applies as the panel goes, so nothing moves under the finger. + @State private var immersiveHeld: Bool? = nil + /// Immersive's three states and its reveal (ImmersivePolicy), and the reveal's auto-hide. + @State private var immersive = ImmersivePolicy() + @State private var immersiveHide: Task? = nil + /// Immersive applies (the panel the whole screen, immersed or revealed): with the Desktop + /// streaming, the viewport asks the Mac for a display of this screen's own size + /// (`Viewport.deviceDesktop`), so the Desktop fills it with no letterbox. + @State private var immersiveFull = false + /// A hardware keyboard is attached (HardwareKeyboardMonitor), for Immersive. + @State private var keyboardAttached = HardwareKeyboardMonitor.shared.attached #if DEBUG /// The console says once when a decision starts to wait. @State private var tourWaitLogged: Double? = nil @@ -234,11 +249,17 @@ struct StreamScreen: View { // makes larger, judged on this screen held upright, whichever way it is held now. let offersLarger = DuoLayout.phoneArrangement && posture.hinge == .unknown && PhonePortraitLayout.offersLargerPicture(screen: geo.size) + // Immersive (docs/immersive-plan.md): on an iPad held sideways with a hardware keyboard, + // the picture the whole screen and the bar on demand. Its rule is ImmersivePolicy's. + let immersiveApplies = immersiveApplies(duo: duo, geo: geo) + // Off in this very frame when it stops applying (a keyboard detached), not a turn later. + let immersiveState: ImmersivePolicy.State = immersiveApplies ? immersive.state : .off ZStack { Group { switch duo { case .innerLandscape: - landscape(bar: .regular, width: geo.size.width, fold: posture.info(in: geo.size)) + landscape(bar: .regular, width: geo.size.width, fold: posture.info(in: geo.size), + immersive: immersiveState) case .outerLandscape: landscape(bar: .compact, width: geo.size.width, fold: posture.info(in: geo.size)) case .innerPortrait: @@ -314,6 +335,20 @@ struct StreamScreen: View { .duoFold(posture, size: geo.size, screen: "stream screen") .duoStatusBar(posture, size: geo.size) .environment(\.largerPictureOffered, offersLarger) + .environment(\.immersiveOffered, UIDevice.current.userInterfaceIdiom == .pad) + // Immersive: no status bar or home indicator, and the top edge's first swipe Sill's, not + // Notification Centre's (R3, R6). Nothing at all while it is off. + .immersiveChrome(immersiveState) + // What Immersive is told: whether it applies, and whether something open over the stream + // holds the bar (R9); the proxy hears whether the input view takes the chord and the edge. + .onChange(of: [immersiveApplies, immersiveHolds], initial: true) { _, now in + immersiveTimer(immersive.update(applies: now[0], held: now[1])) + overlay.immersiveActive = now[0] + immersiveFull = now[0] + #if DEBUG + print("immersive: " + immersive.state.rawValue + (now[0] ? "" : " (" + immersiveWhyNot(duo: duo, geo: geo) + ")")) + #endif + } #if DEBUG // The console's line for the gates: what Larger Picture's rule made of the switch and of // what streams, on a phone held upright, each time one of them or the outcome changes. @@ -364,9 +399,18 @@ struct StreamScreen: View { .onChange(of: client.connected) { _, up in if up { sendViewport() } } + // Immersive with the Desktop on or off: the Mac makes or removes the device's own display. + .onChange(of: immersiveDeviceDesktop) { _, _ in sendViewport(after: 0.25) } // A hardware keyboard (docs/hardware-keyboard-plan.md §2): the input view takes the keyboard by // itself while one is attached and nothing open takes it, never as the person's own. - .onReceive(HardwareKeyboardMonitor.shared.$attached) { overlay.update(attached: $0) } + .onReceive(HardwareKeyboardMonitor.shared.$attached) { attached in + overlay.update(attached: attached) + keyboardAttached = attached + } + .onAppear { + overlay.immersiveToggle = { immersiveToggle() } + overlay.immersiveEdge = { immersiveEdge() } + } .onChange(of: keyboardTakers, initial: true) { _, takers in overlay.update(takers: takers) } // Rotation animates, and a Mac window resize restarts the stream, so let the size settle. .onChange(of: panelSize) { _, _ in sendViewport(after: 0.25) } @@ -465,15 +509,21 @@ struct StreamScreen: View { let fps = StreamClient.wantedFPS(remote: client.awayCapsFrameRate) #if DEBUG print("viewport: \(Self.points(panelSize.width))×\(Self.points(panelSize.height)) pt, scale " - + (textScale.map { TextScaleControl.label($0) } ?? "none") + ", \(fps) fps") + + (textScale.map { TextScaleControl.label($0) } ?? "none") + ", \(fps) fps" + + (immersiveDeviceDesktop ? ", device desktop" : "")) #endif client.sendViewport(Viewport(width: Double(panelSize.width), height: Double(panelSize.height), scale: textScale, - fps: fps)) + fps: fps, + deviceDesktop: immersiveDeviceDesktop ? true : nil)) } } + /// The Desktop in Immersive: the Mac makes a display of the panel's size (DeviceDesktop.swift on + /// the host). An older host ignores the key and the Mac's own screen letterboxes as before. + private var immersiveDeviceDesktop: Bool { immersiveFull && client.active == .desktop } + #if DEBUG /// A length for the console: whole points as whole numbers (within a hundredth: a scaled harness /// screen measures 693.99…), anything else to a tenth. @@ -583,6 +633,15 @@ struct StreamScreen: View { scaleOpen = true } } + // `-SillImmersiveExitAt `: the revealed bar's Exit tapped s seconds in (only while the bar + // is revealed, as the button shows only then). + let exitAt = defaults.double(forKey: "SillImmersiveExitAt") + if exitAt > 0 { + after(exitAt) { + print("harness: Exit (" + immersive.state.rawValue + ")") + if immersive.state == .revealed { exitImmersive() } + } + } } private static var launchArgumentsRan = false @@ -679,6 +738,117 @@ struct StreamScreen: View { PhonePortraitLayout.largerPicture(switchOn: largerPicture, streaming: .init(client.active)) } + // MARK: Immersive (docs/immersive-plan.md) + + /// Immersive applies (ImmersivePolicy.applies): the switch (as it was when Settings opened, while + /// it is open), a hardware keyboard, an iPad held sideways, something streaming, VoiceOver off, + /// and this scene the whole screen. + private func immersiveApplies(duo: DuoLayout, geo: GeometryProxy) -> Bool { + ImmersivePolicy.applies(switchOn: immersiveHeld ?? immersiveSwitch, keyboardAttached: keyboardAttached, + landscape: duo == .innerLandscape && UIDevice.current.userInterfaceIdiom == .pad, + streaming: client.active != .none, voiceOver: voiceOverEnabled, + fillsScreen: immersiveFillsScreen(geo)) + } + + /// This scene the whole screen: its size with the insets it keeps out of (the status bar's strip + /// above, while it shows) against its screen's (not a Stage Manager or Split View window). + private func immersiveFillsScreen(_ geo: GeometryProxy) -> Bool { + #if DEBUG + // The harness's fake screen (`-SillLayout`) is the whole screen by construction: iPadOS 27 will + // not turn an iPad simulator, so a sideways iPad is checked there (docs/DEVELOPMENT.md). + if UserDefaults.standard.string(forKey: "SillLayout") != nil { return true } + #endif + let scene = UIApplication.shared.connectedScenes.lazy.compactMap { $0 as? UIWindowScene } + .first { $0.activationState == .foregroundActive } ?? UIApplication.shared.connectedScenes.lazy + .compactMap { $0 as? UIWindowScene }.first + guard let screen = scene?.screen.bounds.size else { return false } + let insets = geo.safeAreaInsets + return ImmersivePolicy.fills(size: (Double(geo.size.width + insets.leading + insets.trailing), + Double(geo.size.height + insets.top + insets.bottom)), + screen: (Double(screen.width), Double(screen.height))) + } + + /// Something open over the stream holds Immersive's bar (R9): whatever takes the keyboard (the + /// Settings panel, the pairing overlay, the tour, the Apps drawer, the Menus pull-down), and the + /// Aa ruler and a thumbnail's lights, which hang from the bar. + private var immersiveHolds: Bool { + !keyboardTakers.isEmpty || scaleOpen || windowMenu != nil + } + + /// The revealed bar's Exit: Immersive's switch off at once, with no trip to Settings (a panel + /// open meanwhile closes first, so its hold lets go and the switch applies now). The panel goes + /// back to the bar's layout in one motion; its new size is the one viewport that tells the Mac, + /// which removes the device's own display if the Desktop streams. + private func exitImmersive() { + setSettings(false, restoreKeyboard: false) + immersiveHide?.cancel() + immersiveHide = nil + withAnimation(Self.immersiveMotion) { immersiveSwitch = false } + client.noteAction() + #if DEBUG + print("immersive: Exit: the switch off") + #endif + } + + /// ⌃⌥⌘B, from the input view (it takes the chord only while Immersive applies). + private func immersiveToggle() { + let result = immersive.toggle() + guard result.taken else { return } + immersiveTimer(result.timer) + #if DEBUG + print("immersive: the shortcut: " + immersive.state.rawValue) + #endif + } + + /// The top edge's swipe, from the input view. + private func immersiveEdge() { + immersiveTimer(immersive.edgeSwipe()) + #if DEBUG + print("immersive: the top edge: " + immersive.state.rawValue) + #endif + } + + /// A touch on the revealed bar: it stays up its full `autoHide` again. + private func immersiveTouched() { immersiveTimer(immersive.touched()) } + + /// The auto-hide's timer as the rule asks: started again, stopped, or left. + private func immersiveTimer(_ timer: ImmersivePolicy.Timer) { + switch timer { + case .keep: return + case .stop: + immersiveHide?.cancel() + immersiveHide = nil + case .start: + immersiveHide?.cancel() + immersiveHide = Task { @MainActor in + try? await Task.sleep(for: .seconds(ImmersivePolicy.autoHide)) + guard !Task.isCancelled else { return } + immersive.timerFired() + immersiveHide = nil + #if DEBUG + print("immersive: the bar hid itself: " + immersive.state.rawValue) + #endif + } + } + } + + /// The bar's slide, and the panel growing to the edges with it: 0.25 s (the plan's "one motion"). + private static let immersiveMotion: Animation = .easeOut(duration: 0.25) + + #if DEBUG + /// Why Immersive does not apply, for the console. + private func immersiveWhyNot(duo: DuoLayout, geo: GeometryProxy) -> String { + var why: [String] = [] + if !(immersiveHeld ?? immersiveSwitch) { why.append("the switch off") } + if !keyboardAttached { why.append("no hardware keyboard") } + if !(duo == .innerLandscape && UIDevice.current.userInterfaceIdiom == .pad) { why.append("not an iPad sideways") } + if client.active == .none { why.append("nothing streaming") } + if voiceOverEnabled { why.append("VoiceOver") } + if !immersiveFillsScreen(geo) { why.append("not the whole screen") } + return why.joined(separator: ", ") + } + #endif + /// Opens or closes the Settings panel. Opening puts the drawer, the traffic lights and the Aa /// ruler away and takes the keyboard down: with the overlay not first responder no hardware /// key reaches the Mac, and Esc (or ⌘.) reaches the panel's Done instead. Closing (Done, a tap @@ -703,6 +873,8 @@ struct StreamScreen: View { // panel never moves under the finger: a change of the switch, or of what streams, // applies as the panel goes. largerPictureHeld = open ? largerPictureNow : nil + // Immersive's switch too: flipped in the panel, it applies as the panel goes. + immersiveHeld = open ? immersiveSwitch : nil } if !open { if restoreKeyboard, keyboardBeforeSettings { overlay.setKeyboard(shown: true) } @@ -1024,22 +1196,40 @@ struct StreamScreen: View { reduceMotion ? .opacity : .scale(scale: 0.94, anchor: anchor).combined(with: .opacity) } - private func landscape(bar plain: BarMetrics, width: CGFloat, fold: FoldInfo) -> some View { + /// `immersive`: Immersive's state (docs/immersive-plan.md). Off, the bar over the content area as + /// always. Immersed, no bar and the panel the whole screen; revealed, the bar over that same panel, + /// which never resizes for it (R7). One tree in all three, the bar drawn over a spacer of its own + /// height when off, so the stream's input view is never rebuilt and keeps the keyboard. + private func landscape(bar plain: BarMetrics, width: CGFloat, fold: FoldInfo, + immersive: ImmersivePolicy.State = .off) -> some View { // The book pose (docs/iphone-duo-plan.md, (a)): nothing on the fold's band; elsewhere as before. let (bar, stripWidth) = plain.clearing(fold, width: width) - return VStack(spacing: 0) { - TopBar(client: client, metrics: bar, stripWidth: stripWidth, drawerOpen: $drawerOpen, - keyboardShown: $keyboardShown, - textScale: $textScale, scaleOpen: $scaleOpen, windowMenu: $windowMenu, - settingsOpen: settingsOpen, - // Apps, Menus, Aa, Keyboard, Desktop, Settings, and the strip. - menusFit: MacMenuButton.fits(width: width - 2 * bar.padding, buttons: 6, buttonWidth: bar.buttonWidth, - gap: bar.gap, thumbWidth: bar.thumbWidth), - toggleKeyboard: { overlay.toggleKeyboard() }, - setSettings: { setSettings($0, restoreKeyboard: $1) }, - menusOpened: menusOpened, menusClosed: menusClosed) - contentArea(bar: bar) + let full = ImmersivePolicy.fillsScreen(immersive) + return ZStack(alignment: .top) { + VStack(spacing: 0) { + Color.clear.frame(height: full ? 0 : bar.height) + contentArea(bar: bar, fullScreen: full, barOver: immersive == .revealed ? bar.height : 0) + } + if ImmersivePolicy.barShown(immersive) { + TopBar(client: client, metrics: bar, stripWidth: stripWidth, drawerOpen: $drawerOpen, + keyboardShown: $keyboardShown, + textScale: $textScale, scaleOpen: $scaleOpen, windowMenu: $windowMenu, + settingsOpen: settingsOpen, + // Apps, Menus, Aa, Keyboard, Desktop, Settings (and Exit, revealed), and the strip. + menusFit: MacMenuButton.fits(width: width - 2 * bar.padding, + buttons: immersive == .revealed ? 7 : 6, buttonWidth: bar.buttonWidth, + gap: bar.gap, thumbWidth: bar.thumbWidth), + toggleKeyboard: { overlay.toggleKeyboard() }, + setSettings: { setSettings($0, restoreKeyboard: $1) }, + menusOpened: menusOpened, menusClosed: menusClosed, + exitImmersive: immersive == .revealed ? exitImmersive : nil) + // A touch anywhere on the revealed bar keeps it up its full 4 s again. + .simultaneousGesture(TapGesture().onEnded { immersiveTouched() }, including: .all) + .transition(.move(edge: .top)) + } } + .ignoresSafeArea(edges: full ? .top : []) + .animation(Self.immersiveMotion, value: immersive) } /// `largerKeyboard`: Larger Picture's keyboard on a phone (see `body`), nil otherwise. @@ -1061,7 +1251,10 @@ struct StreamScreen: View { private var streamShape: RoundedRectangle { RoundedRectangle(cornerRadius: 12, style: .continuous) } - private func contentArea(bar: BarMetrics) -> some View { + /// `fullScreen`: Immersive's panel, edge to edge with no inset, corner or border. `barOver`: the + /// height of Immersive's revealed bar over the top, which the drawer and the Settings panel hang + /// under. + private func contentArea(bar: BarMetrics, fullScreen: Bool = false, barOver: CGFloat = 0) -> some View { ZStack(alignment: .topLeading) { Color.black @@ -1079,12 +1272,12 @@ struct StreamScreen: View { } .background(Palette.panel) .overlay(alignment: .top) { LinkLineView(text: shownLinkLine) } - .clipShape(streamShape) - .overlay(streamShape.strokeBorder(Color.white.opacity(0.09), lineWidth: 1)) + .clipShape(RoundedRectangle(cornerRadius: fullScreen ? 0 : 12, style: .continuous)) + .overlay(streamShape.strokeBorder(Color.white.opacity(fullScreen ? 0 : 0.09), lineWidth: 1)) // The panel itself, inside the padding: the size the host fits the Mac window to. .onGeometryChange(for: CGSize.self, of: { $0.size }, action: { panelSize = $0 }) .tourTarget(.stream) - .padding(8) + .padding(fullScreen ? 0 : 8) // The dim comes after the overlay on purpose: with the drawer open a tap on the dim // closes the drawer instead of clicking the Mac. @@ -1099,7 +1292,7 @@ struct StreamScreen: View { .frame(width: 380) .frame(maxHeight: .infinity) .padding(.leading, 22) - .padding(.top, 8) + .padding(.top, 8 + barOver) .padding(.bottom, 14) .transition(.opacity) } @@ -1117,7 +1310,7 @@ struct StreamScreen: View { takeTour: takeTour) .frame(width: bar.settingsWidth) .frame(maxHeight: .infinity, alignment: .top) - .padding(.top, 8) + .padding(.top, 8 + barOver) .padding(.bottom, 14) .padding(.trailing, bar.padding) .frame(maxWidth: .infinity, alignment: .trailing) @@ -1234,6 +1427,9 @@ private struct TopBar: View { /// The Menus pull-down opened and went (see `StreamScreen.menusOpened`). let menusOpened: () -> Void let menusClosed: () -> Void + /// Immersive's revealed bar: Exit, which turns Immersive off (`StreamScreen.exitImmersive`). + /// nil everywhere else, where the bar is main's. + var exitImmersive: (() -> Void)? = nil var body: some View { HStack(spacing: metrics.gap) { @@ -1286,6 +1482,13 @@ private struct TopBar: View { .opacity(scaleOpen ? 0 : 1) .allowsHitTesting(!scaleOpen) + if let exitImmersive { + button(open: false, symbol: "arrow.down.right.and.arrow.up.left", label: "Exit", + accessibilityLabel: "Leave Immersive", action: exitImmersive) + .opacity(scaleOpen ? 0 : 1) + .allowsHitTesting(!scaleOpen) + } + // Leave's old slot: Disconnect is the Settings panel's pinned last row now. button(open: settingsOpen, symbol: "gearshape", label: "Settings", accessibilityLabel: settingsOpen ? "Close settings" : "Settings for \(client.macName.isEmpty ? "the Mac" : client.macName)", @@ -2115,3 +2318,19 @@ extension PhonePortraitLayout.Streaming { } } } + +// MARK: - Immersive's chrome + +extension View { + /// Immersive's system overlays (docs/immersive-plan.md, R3 and R6): while it is on, no status bar, + /// no home indicator, and a swipe from the top edge Sill's first (the edge swipe), the system's + /// only on a second. Off, nothing at all: today's screen exactly, its status bar the Duo's say. + /// No branch around `self`: one would give the whole stream screen a new identity on entering, and + /// rebuild the input view out from under the hardware keyboard's hold. + func immersiveChrome(_ state: ImmersivePolicy.State) -> some View { + let on = ImmersivePolicy.fillsScreen(state) + return background { if on { Color.clear.statusBarHidden(true) } } + .persistentSystemOverlays(on ? .hidden : .automatic) + .defersSystemGestures(on: on ? .top : []) + } +}