From 2f4230af4f31fbdb4eee9746470fab280bb0b6f3 Mon Sep 17 00:00:00 2001 From: Damian Reeves <957246+DamianReeves@users.noreply.github.com> Date: Tue, 3 Nov 2026 13:43:39 -0600 Subject: [PATCH 01/14] docs: record workspace migration ownership Track campaign 6 in #479 from the verified PR #477 baseline.\nParent retains review, CI, landing and cleanup gates. --- NOTICE | 5 +++++ .../docs/contributing/design/prds/_index.md | 2 +- .../design/prds/layered-configuration.md | 14 +++++++++--- .../prds/workspace-migration-delivery.md | 22 +++++++++++++++++++ .../add-workspace-migration/proposal.md | 2 +- .../changes/add-workspace-migration/tasks.md | 2 +- 6 files changed, 41 insertions(+), 6 deletions(-) create mode 100644 docs/content/docs/contributing/design/prds/workspace-migration-delivery.md diff --git a/NOTICE b/NOTICE index efdc4472..98d4e02d 100644 --- a/NOTICE +++ b/NOTICE @@ -69,3 +69,8 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +Bounded workspace adoption, aggregate verification and recoverable configuration transactions +Copyright 2026 FINOS +Distributed under the repository Apache-2.0 license. No new third-party code or +license restrictions are introduced by workspace migration. diff --git a/docs/content/docs/contributing/design/prds/_index.md b/docs/content/docs/contributing/design/prds/_index.md index 52c4b629..cb648967 100644 --- a/docs/content/docs/contributing/design/prds/_index.md +++ b/docs/content/docs/contributing/design/prds/_index.md @@ -15,7 +15,7 @@ Product Requirements Documents (PRDs) track feature requirements, design decisio | [IR JSON Schema Verification BDD](ir-json-schema-verification-bdd) | 🚧 In Progress | BDD scenarios for schema verification | | [Deployment Architecture Refactor](deployment-architecture-refactor) | 📋 Draft | Build and deployment improvements | | [F# backend / Myriad delivery](https://github.com/finos/morphir-dotnet/blob/main/docs/design/PRD-fsharp-backend.md) | In Progress | Classic v1-v3 to F#, per-module Myriad generation and car rental tutorial | -| [Layered Configuration](layered-configuration) | In Progress | Campaigns 1 through 4 landed and cleaned; campaign 5 project migration locally qualified under #476, parent review/landing/cleanup pending; workspace migration remains planned | +| [Layered Configuration](layered-configuration) | In Progress | Campaigns 1 through 5 landed and cleaned; workspace migration in progress under #479 | | [Product Manager Skill](product-manager-skill) | 📋 Draft | AI skill for product management | ## Status Legend diff --git a/docs/content/docs/contributing/design/prds/layered-configuration.md b/docs/content/docs/contributing/design/prds/layered-configuration.md index 853fa990..a66904e7 100644 --- a/docs/content/docs/contributing/design/prds/layered-configuration.md +++ b/docs/content/docs/contributing/design/prds/layered-configuration.md @@ -7,7 +7,7 @@ description: "Shared configuration, project discovery, and configuration inspect # Layered configuration and workspaces -**Status**: In Progress; campaigns 1 through 4 landed and cleaned; campaign 5 project migration locally qualified under #476; parent gates pending +**Status**: In Progress; campaigns 1 through 5 landed and cleaned; campaign 6 workspace migration in progress under #479 **Created**: 2025-12-22 @@ -178,8 +178,8 @@ Follow red-green-refactor for implementation. Start with BDD scenarios and focus | Shared resolution, profiles, validation, provenance | Implemented and landed | Campaign 1 additive APIs; prior baseline core 95 and reader/BDD 66; campaign 2 requalification tracked separately | | Microsoft configuration integration | Implemented and landed | Campaign 3, PR #471; opt-in resolved provider, generated/explicit binding and managed/trimmed/native qualification retained | | Shared query and edit services | Implemented and landed | Campaigns 3 and 4, PR #475; shared plans, TOML editor, transactions and parent cleanup confirmed | -| Classic project migration | Implemented locally; parent gates pending | Campaign 5, #476; pure planner, independent verification, guarded IO, project migrate and managed/trimmed/native qualification | -| Workspace migration | Planned | Bounded inventory, coordinated project plans, membership, and recovery | +| Classic project migration | Implemented and landed | Campaign 5, #476; pure planner, independent verification, guarded IO, project migrate and managed/trimmed/native qualification | +| Workspace migration | In Progress | Bounded inventory, coordinated project plans, membership, and recovery | | Config CLI commands | Implemented locally; parent gates pending | Campaign 3 get/list/show/path/explain are landed. Campaign 4 set/unset use shared services; all 360 tooling tests pass, including 33 editing subprocess scenarios. Published-host qualification passes; parent delivery gates remain | | Documentation foundation | Updated for inspection and editing | Shared API and Microsoft consumer examples, read CLI schema/options, walkthrough and troubleshooting; editing API, preview/recovery and CLI examples are implemented; classic project migration reference and walkthrough implemented; workspace migration remains planned | | Compatibility and end-to-end verification | In Progress | Campaign 1 landed baseline tooling 245, reader 66, core 95 and warning-qualified runtime evidence; current campaign results recorded separately | @@ -316,3 +316,11 @@ context. The IO coordinator reuses exact-buffer observations, the TOML editor and transaction boundary. Read-only hosts receive a redacted `Presentation`. The CLI remains a separate thin project provider. Qualification and parent gates are recorded in the [delivery journal](project-migration-delivery.md). + +### Workspace migration campaign, 2026-11-03 + +Campaign 6 tracks [#479](https://github.com/finos/morphir-dotnet/issues/479). +The [delivery journal](workspace-migration-delivery.md) records ownership, +prerequisite ancestry, task progress and qualification. PR #477 landed at +`e3b9a053120a49e94c70c30cd4f62ef783672c64`; its receipt confirms campaign 5 +cleanup. Parent owns campaign 6 review, CI watch, landing and cleanup. diff --git a/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md b/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md new file mode 100644 index 00000000..8ba77db9 --- /dev/null +++ b/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md @@ -0,0 +1,22 @@ +# Workspace migration delivery journal + +Tracking issue: [#479](https://github.com/finos/morphir-dotnet/issues/479). +Change: `add-workspace-migration`, campaign 6 of six. + +## Ownership and baseline + +Owned checkout `/Users/damian/.t3/worktrees/morphir-dotnet/t3-f370bf10`, branch `t3/workspace-migration`, initially clean at exact required main baseline `e3b9a053120a49e94c70c30cd4f62ef783672c64`. GitHub main agrees. Prerequisite PR #477 is merged at that same SHA, and its [receipt](https://github.com/finos/morphir-dotnet/pull/477#issuecomment-6613145141) confirms review/checks and owned cleanup. The real open tracking issue was created and read before coding. + +OpenSpec list/status/instructions apply returned schema `html-design`, ready, 0/9 tasks. Full returned proposal/design/spec/tasks and layered configuration PRD, project/config editing journals and shared APIs were read. Only this change will be synchronized/archived when all implementation tasks are genuinely complete. Independent active plans remain untouched. + +Parent owns independent final review, CI watch, authorized landing and checkout/branch/artifact cleanup. This implementation worker does not watch, merge or delete checkout/branches. The campaign remains incomplete until parent gates finish. + +## Work and evidence ownership + +Lead owns bounded inventory, aggregate composition/verification, integration and qualification. Two Codex helpers own shared transaction/recovery and CLI/report/schema/subprocess tests respectively. Helpers preserve one another's changes. One heavy restore/build/test/publish at a time, `DOTNET_PROCESSOR_COUNT=2`, serial MSBuild/restore, sequenced Expecto and TUnit `--maximum-parallel-tests 1`. + +All owned external artifacts are under `/private/tmp/morphir-workspace-migration/`, including physical `tmp/`, `transactions/`, `cli/`, build/runner/coverage/publication logs and manifests. Global caches/tools are not changed. Process-local PowerShell 7.5.3 is used for F# SDK tests. Unresolved recovery journals/backups are preserved and inventoried. + +## Checkpoints + +Setup completed before runtime changes. Proposal now links actual issue #479. Task 1.1 was updated at returned line 8 and apply progress rechecked. diff --git a/openspec/changes/add-workspace-migration/proposal.md b/openspec/changes/add-workspace-migration/proposal.md index 40ae6ace..e04be309 100644 --- a/openspec/changes/add-workspace-migration/proposal.md +++ b/openspec/changes/add-workspace-migration/proposal.md @@ -26,7 +26,7 @@ None. Existing publication and virtual-file requirements remain unchanged. - [Layered configuration PRD](../../../docs/content/docs/contributing/design/prds/layered-configuration.md). - Implementation prerequisites: [`add-classic-project-migration`](../archive/2026-11-03-add-classic-project-migration/proposal.md). -- GitHub tracking issue: not yet assigned; link the actual issue before opening an implementation PR. No external issue is created by this planning work. +- GitHub tracking issue: [#479](https://github.com/finos/morphir-dotnet/issues/479). ## Impact diff --git a/openspec/changes/add-workspace-migration/tasks.md b/openspec/changes/add-workspace-migration/tasks.md index 019b7c1c..2e6471ae 100644 --- a/openspec/changes/add-workspace-migration/tasks.md +++ b/openspec/changes/add-workspace-migration/tasks.md @@ -5,7 +5,7 @@ Read [the design](design.html) and change-local specs before starting. Existing ## 1. Campaign setup -- [ ] 1.1 Prepare an owned worktree/branch after add-classic-project-migration lands; verify its merged SHA, clean baseline, and recorded campaign tracking/ownership. +- [x] 1.1 Prepare an owned worktree/branch after add-classic-project-migration lands; verify its merged SHA, clean baseline, and recorded campaign tracking/ownership. ## 2. Inventory and aggregate planning From 67241f13e7dfe267b35204196216ec51a4b08bba Mon Sep 17 00:00:00 2001 From: Damian Reeves <957246+DamianReeves@users.noreply.github.com> Date: Tue, 3 Nov 2026 14:20:42 -0600 Subject: [PATCH 02/14] feat: coordinate recoverable configuration batches Stage the full batch before replacement and retain durable overlap guards. Explicit recovery preserves uncertain ownership and user changes. Refs #479. --- .../configuration/transaction-recovery.md | 137 +++++++++ .../prds/workspace-migration-delivery.md | 60 ++++ .../changes/add-workspace-migration/tasks.md | 10 +- .../ConfigurationRecovery.cs | 289 ++++++++++++++++++ .../ConfigurationTransaction.cs | 255 ++++++++++++---- .../ConfigurationBatchRecovery.feature | 21 ++ .../ConfigurationBatchRecoverySteps.cs | 90 ++++++ .../ConfigurationBatchTransactionTests.cs | 174 +++++++++++ .../ConfigurationEditingTests.cs | 3 + .../ConfigurationRecoveryProcessTests.cs | 139 +++++++++ .../ConfigurationRecoveryTests.cs | 237 ++++++++++++++ .../ConfigurationTransactionTests.cs | 55 +++- .../ProjectMigrationTests.cs | 6 + 13 files changed, 1401 insertions(+), 75 deletions(-) create mode 100644 docs/content/docs/configuration/transaction-recovery.md create mode 100644 src/Morphir.Configuration.IO/ConfigurationRecovery.cs create mode 100644 tests/Morphir.Configuration.IO.Tests/ConfigurationBatchRecovery.feature create mode 100644 tests/Morphir.Configuration.IO.Tests/ConfigurationBatchRecoverySteps.cs create mode 100644 tests/Morphir.Configuration.IO.Tests/ConfigurationBatchTransactionTests.cs create mode 100644 tests/Morphir.Configuration.IO.Tests/ConfigurationRecoveryProcessTests.cs create mode 100644 tests/Morphir.Configuration.IO.Tests/ConfigurationRecoveryTests.cs diff --git a/docs/content/docs/configuration/transaction-recovery.md b/docs/content/docs/configuration/transaction-recovery.md new file mode 100644 index 00000000..e80e11c5 --- /dev/null +++ b/docs/content/docs/configuration/transaction-recovery.md @@ -0,0 +1,137 @@ +--- +title: "Configuration transactions and recovery" +linkTitle: "Transaction recovery" +weight: 55 +--- + +# Configuration transactions and recovery + +`WorkspaceMigrationService`, `ProjectMigrationService` and `ConfigurationEditService` +apply through the shared `ConfigurationTransaction` boundary. Configuration tools +coordinate cooperating writers on one machine. They do not provide a distributed +lock, or a crash-atomic replacement of several files. + +## Batch application + +A transaction first acquires writer coordination and checks pending recovery. It +checks exact source and target bytes, observed absences, directory inventories +and link topology. It creates durable write-ahead metadata before staging every +candidate and original backup on the corresponding target filesystem. All stages, +backups and preconditions must match before the first destination replacement. +The transaction rechecks remaining inputs between installations. Atomic native +replacement protects one destination at a time. + +If a replacement fails, the transaction attempts rollback in reverse order. +`RestoredPaths` identifies originals proven restored. `UnresolvedPaths` identifies +paths whose state is uncertain or whose bytes no longer match the recorded +original or candidate. The result retains artifact paths and their transaction +owner. Configuration bytes, backup bytes and secrets are never included in these +metadata results. + +`Written` means the complete candidate was installed and its owned temporary +artifacts were cleaned. `Unchanged` leaves equivalent target bytes and timestamps +alone. `Conflict` means input evidence changed. `RolledBack` means rollback proved +the originals restored and cleaned transaction artifacts. `RecoveryRequired` +means retained evidence needs explicit inspection and recovery. Do not treat a +partial workspace, successful command exit from an unrelated tool, or the presence +of a candidate TOML file as evidence that the batch completed. + +## Inspect and recover through the shared API + +Recovery is explicit. Keep the returned journal path and inspect it before taking +action. Use the same authorized storage host that owns the paths: + +```csharp +var transaction = new ConfigurationTransaction(new NativeConfigurationTransactionHost()); +var pending = transaction.InspectPending(new[] { workspaceRoot }); +foreach (var item in pending) +{ + Console.Error.WriteLine($"{item.JournalPath}: {item.Code}"); + // Show item.Targets, item.Artifacts, item.UnresolvedPaths to the operator. +} + +var inspection = transaction.InspectRecovery(journalPath); +if (inspection.Valid && inspection.UnresolvedPaths.IsEmpty) +{ + // This call is an explicit recovery request by the host/operator. + var outcome = transaction.Recover(journalPath); + // Check outcome.Status and preserve outcome.Artifacts on RecoveryRequired. +} +``` + +Inspection does not change files. Recovery reacquires writer coordination and +checks the metadata and file identities again. Without a valid committed witness, +recovery restores the recorded originals. A target that was absent is removed only +when its current bytes match this transaction's candidate. With a durable committed +witness, recovery verifies all installed candidates and finishes cleanup. + +A host can stop between creating a directory and returning its ownership receipt. +Inspection reports these paths in `UncertainDirectories`. Default recovery keeps +the journal and returns `RecoveryRequired`. After reviewing those paths, an +operator can explicitly retain them as user-owned directories: + +```csharp +var result = transaction.Recover(journalPath, inspection.UncertainDirectories); +// Check result.Status. result.PreservedDirectories lists directories retained +// by this explicit request; transaction cleanup never claims or removes them. +``` + +This option accepts only the journal's uncertain paths that remain regular +directories. Linked paths, file replacements and unrelated paths remain blocked. +It does not authorize deletion or discard user content. + +If a target, backup, stage or journal has unexpected content or link topology, +recovery preserves it and reports an unresolved path. Do not overwrite the user's +new bytes with an old backup or delete evidence to bypass the guard. Preserve both +versions, review the metadata and restore a specifically approved state before +retrying recovery. Invalid/truncated journals need manual review; automatic +recovery does not invent metadata or infer ownership from a filename alone. + +## Journals and interruption windows + +Native `.morphir-edit--...` files contain private stages, original +backups and metadata journals. Native creation uses exclusive new files and user +read/write permissions. Metadata and file bytes are flushed before proceeding. +Journals identify the complete target set, original and candidate hashes, stage +and backup paths, and guarded directories. They do not contain the source values. +The initial manifest is written in an existing common ancestor before any parent +directory creation. A durable directory receipt records directories whose creation +the host confirmed. The complete write-ahead manifest allows inspection after +interruption before a stage, between staging and replacement, or immediately after a replacement whose +callback did not return. + +A committed witness distinguishes a completed installation from an interrupted +commit. Cleanup preserves this witness until the other owned stages, backups and +journals have been removed. An interruption during cleanup therefore remains +identifiable. Complete success and complete rollback remove only owned temporary +files and empty owned directories. An incomplete outcome retains journals and +backups. Cooperating configuration and project/workspace editing paths refuse an +overlapping write until explicit recovery resolves that state. + +File flushes and atomic replacement do not promise group atomicity or recovery +from every filesystem, storage-controller or power-loss failure. Uncooperative +processes can modify files outside Morphir's writer lock. Their unexpected bytes +are treated as drift, not silently overwritten during recovery. + +## Custom storage hosts + +`IConfigurationTransactionHost` remains the storage boundary. Hosts serialize +ordered canonical guarded paths across their writers, report exact byte hashes, +and atomically install one file. Writable hosts explicitly opt in with +`SupportsDurableRecovery = true` and implement `FindRecoveryJournals` and +`EnumerateTransactionArtifacts`. Enumeration must remain +bounded to authorized locations, include retained transaction-owned artifacts, +and never follow links. Stages and backups use +`ConfigurationTransaction.GetArtifactPath(destination, transactionId, role)`; +recovery validates the owner, destination anchor and role, including when several +targets share identical bytes. Returning `null` for artifact enumeration means recovery +cannot prove cleanup completeness and returns `config.recovery_host_capability`. +Older adapters remain source compatible. Application refuses a host without this +capability before any write, returning `IoFailure` and +`config.recovery_host_capability`. Read-only planning remains available. Hosts +must add durable enumeration before opting into writes; an empty process-local +registry is not a substitute. + +Recovery metadata uses version 2. Automatic recovery refuses unknown versions and +invalid ownership. Preserve earlier journals for explicit review with their +original host/version. Do not rewrite or silently delete them during an upgrade. diff --git a/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md b/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md index 8ba77db9..72658c16 100644 --- a/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md +++ b/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md @@ -20,3 +20,63 @@ All owned external artifacts are under `/private/tmp/morphir-workspace-migration ## Checkpoints Setup completed before runtime changes. Proposal now links actual issue #479. Task 1.1 was updated at returned line 8 and apply progress rechecked. + +### Inventory and planning checkpoints + +- `inventory-red-build.log` proves the aggregate API was absent before production. + Initial BDD and API cases preceded implementation. `inventory-io-full-01.log` + passes 323/323 IO/TUnit/Reqnroll cases, including three workspace BDD cases. +- `planning-red-run.log` passes 10 and fails four of 14 focused cases, reproducing + repeated-inventory classification, non-adoption setting drift, glob reopening + skipped/nested directories and filtered-root identity. `planning-green-run.log` + passes all 14 after fixing those behaviors through shared discovery and mapping. +- `boundaries-red-run.log` passes four and fails four boundary cases. These expose + retained classic root context, configured output skipping, invalid defaults and + paths containing glob syntax. `recovery-preview-red-run.log` separately fails + pending-journal preview before read-only recovery inspection is added. + `boundaries-green-run.log` passes all 23 focused planner/application cases with + clean compilation, including source/target/listing/topology drift and repeat + source-byte/target-byte/timestamp checks. No generated output is moved. +- Early inventory build attempts while recovery production was incomplete and a + zero-test filter run are retained as unsuccessful attempts, not qualification. + A lightweight FSI probe had an FSharp.Core load mismatch; configured executables + provide correctness evidence instead. +- Tasks 2.1 and 2.2 were checked at fresh CLI-returned locations and progress + confirmed after each update. Aggregate equivalence/adoption qualification and + application/recovery qualification remain active. + +### Coordinated application and recovery + +- `transactions/batch-red-run.log` fails both first-stage ordering cases before + production; later staging and recovery runs prove every candidate and backup + exists before the first replacement. Complete preflight blocks a bad member, + stale source/target/listing/alternate primary and overlapping writers. +- `transactions/recovery-red-build.log` and `inventory-build-01.log` prove missing + explicit inspection/recovery APIs. `transactions/boundary-red-run.log` reproduces + read-only unrelated-directory staging and lost common-ancestor cleanup guards. +- `transactions/capability-red-run.log` proves a legacy custom host could write + without durable enumeration. Application now returns a typed capability failure + before artifacts unless a host explicitly supplies durable support. +- `transactions/evidence-red-run.log` reproduces null/overflow journal parser + exceptions. Recovery now reports invalid metadata without changing it. Stable + artifact names are tied to target, role and owner. A mounted serialized-state + host proves recovery is not process-memory-only; tampered backups/user changes + preserve evidence and block automatic restoration. +- `transactions/final-precondition-red-run-02.log` reproduces source drift after + the last install. Final rechecking now detects it and attempts rollback. + `transactions/parent-creation-red-run.log` reproduces a process exit after mkdir + but before the host returned its ownership receipt. A durable initial journal + now precedes mkdir; uncertain directories remain explicit and require reviewed + retention, never inferred cleanup ownership. +- `transactions/last-windows-green-run-02.log` passes 87 focused cases. Seven actual + process-interruption windows and separate-process pending-write refusal pass. + `planner-final-build-02.log` is clean, and `planner-final-io-02.log` passes the + complete current-source IO runner, 371/371, after the lead's later fixes. +- Success and proven rollback clean owned artifacts. Explicit recovery resolves + test-owned native faults. Retained malformed parser samples are synthetic; + a pre-fix mkdir-crash fixture retains generated empty directories and child + logs as review evidence, without a recoverable target or journal. Those samples + are preserved for parent cleanup. No real unresolved native journal/backup was + found in the helper's final inventory. +- Native coordination is machine-local, and file replacement is per-file atomic. + No cross-file crash-atomicity or distributed-lock guarantee is claimed. diff --git a/openspec/changes/add-workspace-migration/tasks.md b/openspec/changes/add-workspace-migration/tasks.md index 2e6471ae..078cb6c3 100644 --- a/openspec/changes/add-workspace-migration/tasks.md +++ b/openspec/changes/add-workspace-migration/tasks.md @@ -9,15 +9,15 @@ Read [the design](design.html) and change-local specs before starting. Existing ## 2. Inventory and aggregate planning -- [ ] 2.1 Write failing BDD/API cases for roots with no workspace, include/exclude patterns, default skipped directories, nested workspace boundaries, discovered links, existing migrated members, virtual/root projects, and stable ordering; confirm red before inventory implementation. -- [ ] 2.2 Implement bounded shared inventory and aggregate plans by composing the project migrator, preserving existing membership/defaults/settings and using explicit member paths; verify the inventory/planning suite and document the adoption command's scan rules. +- [x] 2.1 Write failing BDD/API cases for roots with no workspace, include/exclude patterns, default skipped directories, nested workspace boundaries, discovered links, existing migrated members, virtual/root projects, and stable ordering; confirm red before inventory implementation. +- [x] 2.2 Implement bounded shared inventory and aggregate plans by composing the project migrator, preserving existing membership/defaults/settings and using explicit member paths; verify the inventory/planning suite and document the adoption command's scan rules. - [ ] 2.3 Add failing then passing verification of project conversion in a common proposed workspace context, with separately reported ownership/membership/selection/output-root effects; verify source paths/non-adoption settings stay equivalent and document that generated files are not moved. ## 3. Coordinated application -- [ ] 3.1 Write failing cases for a blocked member, target/source changes, conflicting overlapping writers, staging failure, partial replacement failure, rollback failure, and interrupted journals; verify no destination is replaced before complete preflight/staging. -- [ ] 3.2 Implement aggregate application using ordered writer coordination, input preconditions, a durable recovery journal, and rollback attempts; verify full rollback or explicit recovery-required outcomes, never claimed cross-file crash atomicity. -- [ ] 3.3 Implement explicit journal inspection/recovery through the shared transaction API and guard overlapping writes while recovery is pending; verify success/complete rollback removes owned transaction artifacts and incomplete recovery preserves evidence, with recovery docs in this group. +- [x] 3.1 Write failing cases for a blocked member, target/source changes, conflicting overlapping writers, staging failure, partial replacement failure, rollback failure, and interrupted journals; verify no destination is replaced before complete preflight/staging. +- [x] 3.2 Implement aggregate application using ordered writer coordination, input preconditions, a durable recovery journal, and rollback attempts; verify full rollback or explicit recovery-required outcomes, never claimed cross-file crash atomicity. +- [x] 3.3 Implement explicit journal inspection/recovery through the shared transaction API and guard overlapping writes while recovery is pending; verify success/complete rollback removes owned transaction artifacts and incomplete recovery preserves evidence, with recovery docs in this group. ## 4. Workspace CLI and integration diff --git a/src/Morphir.Configuration.IO/ConfigurationRecovery.cs b/src/Morphir.Configuration.IO/ConfigurationRecovery.cs new file mode 100644 index 00000000..b79d64b6 --- /dev/null +++ b/src/Morphir.Configuration.IO/ConfigurationRecovery.cs @@ -0,0 +1,289 @@ +// Copyright 2026 FINOS +// SPDX-License-Identifier: Apache-2.0 +using System.Collections.Immutable; +using System.Text; +using System.Text.Json; + +namespace Morphir.Configuration.IO; + +/// Metadata only. No target, stage or backup bytes are exposed. +public sealed record ConfigurationRecoveryInspection( + string JournalPath, string? TransactionId, bool Valid, bool Committed, + ImmutableArray Targets, ImmutableArray Artifacts, + ImmutableArray UnresolvedPaths, string Code) +{ + public ImmutableArray UncertainDirectories { get; init; } = []; +} + +internal sealed record RecoveryEntry(string Target, ConfigurationObservationKind OriginalKind, + string OriginalIdentity, string CandidateIdentity, string Stage, string? Backup); + +internal sealed record RecoveryDocument(string TransactionId, bool Committed, + ImmutableArray Guards, ImmutableArray Entries, + ImmutableArray Artifacts) +{ + internal ImmutableArray PendingDirectories { get; init; } = []; + + // A committed witness must remain above every destination while cleanup runs. + internal string MetadataDestination => ProjectPath.join(Guards.First(g => Entries.All(e => ProjectPath.contains(g, e.Target))), ".morphir-transaction"); + + internal static string ArtifactPath(string destination, string transactionId, string role) => + ProjectPath.join(ProjectPath.parent(destination), ".morphir-edit-" + transactionId + "-" + + NativeConfigurationTransactionHost.Hash(Encoding.UTF8.GetBytes(destination))[..16] + "-" + role); + + internal static bool IsJournal(string path) => Path.GetFileName(path).Contains("-journal", StringComparison.Ordinal) + && Path.GetFileName(path).StartsWith(".morphir-edit-", StringComparison.Ordinal); + + internal byte[] Encode() + { + using var stream = new MemoryStream(); + using (var writer = new Utf8JsonWriter(stream)) + { + writer.WriteStartObject(); writer.WriteNumber("version", 2); + writer.WriteString("transactionId", TransactionId); writer.WriteBoolean("committed", Committed); + writer.WriteStartArray("pendingDirectories"); foreach (var path in PendingDirectories) writer.WriteStringValue(path); writer.WriteEndArray(); + writer.WriteStartArray("guards"); foreach (var path in Guards) writer.WriteStringValue(path); writer.WriteEndArray(); + writer.WriteStartArray("entries"); + foreach (var entry in Entries) + { + writer.WriteStartObject(); writer.WriteString("target", entry.Target); + writer.WriteString("originalKind", entry.OriginalKind.ToString()); writer.WriteString("originalIdentity", entry.OriginalIdentity); + writer.WriteString("candidateIdentity", entry.CandidateIdentity); writer.WriteString("stage", entry.Stage); + writer.WriteString("backup", entry.Backup); writer.WriteEndObject(); + } + writer.WriteEndArray(); writer.WriteStartArray("artifacts"); + foreach (var artifact in Artifacts) + { + writer.WriteStartObject(); writer.WriteString("path", artifact.Path); writer.WriteString("kind", artifact.Kind.ToString()); writer.WriteEndObject(); + } + writer.WriteEndArray(); writer.WriteEndObject(); + } + return stream.ToArray(); + } + + internal static RecoveryDocument Decode(byte[] bytes, string journalPath) + { + using var json = JsonDocument.Parse(bytes, new JsonDocumentOptions { MaxDepth = 16 }); + var root = json.RootElement; + if (!root.GetProperty("version").TryGetInt32(out var version) || version != 2) throw new InvalidDataException("Unknown journal version"); + static string Text(JsonElement value) => value.GetString() ?? throw new InvalidDataException("Missing journal string"); + var id = Text(root.GetProperty("transactionId")); + if (id.Length != 32 || id.Any(c => !Uri.IsHexDigit(c))) throw new InvalidDataException("Invalid journal owner"); + var pendingDirectories = root.TryGetProperty("pendingDirectories", out var pending) + ? pending.EnumerateArray().Select(Text).ToImmutableArray() : []; + var guards = root.GetProperty("guards").EnumerateArray().Select(Text).ToImmutableArray(); + var entries = root.GetProperty("entries").EnumerateArray().Select(e => new RecoveryEntry( + Text(e.GetProperty("target")), Enum.Parse(Text(e.GetProperty("originalKind"))), + Text(e.GetProperty("originalIdentity")), Text(e.GetProperty("candidateIdentity")), + Text(e.GetProperty("stage")), e.GetProperty("backup").ValueKind == JsonValueKind.Null ? null : Text(e.GetProperty("backup")))).ToImmutableArray(); + var artifacts = root.GetProperty("artifacts").EnumerateArray().Select(a => new ConfigurationArtifact( + Text(a.GetProperty("path")), Enum.Parse(Text(a.GetProperty("kind"))), id)).ToImmutableArray(); + static bool Canonical(string path) => ProjectPath.isAbsolute(path) && ProjectPath.normalize(path) == path; + static bool Hash(string value) => value.Length == 64 && value.All(Uri.IsHexDigit); + bool Owned(string path) => Canonical(path) && Path.GetFileName(path).StartsWith(".morphir-edit-" + id + "-", StringComparison.Ordinal) + && guards.Contains(ProjectPath.parent(path), StringComparer.Ordinal); + if (guards.IsDefaultOrEmpty || guards.Any(p => !Canonical(p)) || entries.IsDefaultOrEmpty + || entries.Select(e => e.Target).Distinct(StringComparer.Ordinal).Count() != entries.Length + || entries.Any(e => !Canonical(e.Target) || !Hash(e.CandidateIdentity) + || e.OriginalKind is not (ConfigurationObservationKind.File or ConfigurationObservationKind.Missing) + || e.OriginalKind == ConfigurationObservationKind.File && (!Hash(e.OriginalIdentity) || e.Backup is null) + || e.OriginalKind == ConfigurationObservationKind.Missing && (e.OriginalIdentity.Length != 0 || e.Backup is not null) + || !Owned(e.Stage) || e.Stage != ArtifactPath(e.Target, id, "stage") + || e.Backup is not null && (!Owned(e.Backup) || e.Backup != ArtifactPath(e.Target, id, "backup")) + || !guards.Contains(ProjectPath.parent(e.Target), StringComparer.Ordinal)) + || pendingDirectories.Any(p => !Canonical(p) || !entries.Any(e => e.Target != p && ProjectPath.contains(p, e.Target))) + || artifacts.Any(a => !Enum.IsDefined(a.Kind)) + || artifacts.Any(a => a.Kind == ConfigurationArtifactKind.Directory + ? !Canonical(a.Path) || !guards.Any(g => ProjectPath.contains(a.Path, g)) + : !Owned(a.Path)) + || !Owned(journalPath) || !IsJournal(journalPath)) + throw new InvalidDataException("Invalid journal ownership"); + return new(id, root.GetProperty("committed").GetBoolean(), guards, entries, artifacts) { PendingDirectories = pendingDirectories }; + } +} + +public sealed partial class ConfigurationTransaction +{ + /// Inspect durable metadata without modifying any file. Invalid or linked journals + /// remain explicit recovery blockers. This method never returns raw configuration bytes. + public ConfigurationRecoveryInspection InspectRecovery(string journalPath) + { + if (string.IsNullOrWhiteSpace(journalPath) || !ProjectPath.isAbsolute(journalPath) || ProjectPath.normalize(journalPath) != journalPath) + return new(journalPath ?? "", null, false, false, [], [], [], "config.recovery_invalid_journal"); + try + { + var document = ReadRecovery(journalPath); + document = LatestRecoveryDocument(document); + var artifacts = RecoveryArtifacts(document, journalPath); + var unresolved = RecoveryUnresolved(document, artifacts); + return new(journalPath, document.TransactionId, true, document.Committed, document.Entries.Select(e => e.Target).ToImmutableArray(), + Retained(artifacts), unresolved, unresolved.Length == 0 ? "config.recovery_inspected" : "config.recovery_unresolved") + { UncertainDirectories = document.PendingDirectories.Where(p => host.Observe(p).Kind != ConfigurationObservationKind.Missing).ToImmutableArray() }; + } + catch (Exception error) when (IsRecoveryError(error)) + { + return new(journalPath, null, false, false, [], [new(journalPath, ConfigurationArtifactKind.Journal, "unknown")], + [journalPath], "config.recovery_invalid_journal"); + } + } + + /// Find relevant durable recovery state through the host's bounded journal index. + public ImmutableArray InspectPending(IEnumerable paths) => + host.FindRecoveryJournals(paths.Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal).ToArray()) + .Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal).Select(InspectRecovery).ToImmutableArray(); + + /// Explicitly restore an interrupted operation, or finish cleanup of a durably + /// committed operation. User changes and unverifiable artifacts are preserved and reported. + public ConfigurationTransactionResult Recover(string journalPath) => Recover(journalPath, []); + + /// Explicitly preserve reviewed directories whose creation receipt was interrupted. + /// Such directories are never claimed or removed by transaction cleanup. + public ConfigurationTransactionResult Recover(string journalPath, IEnumerable preserveUncertainDirectories) + { + var preserved = preserveUncertainDirectories.ToHashSet(StringComparer.Ordinal); + var inspection = InspectRecovery(journalPath); + ConfigurationTransactionResult Required(string code, IEnumerable? restored = null) => + new(ConfigurationTransactionStatus.RecoveryRequired, code, inspection.Targets, inspection.Artifacts) + { RestoredPaths = (restored ?? []).ToImmutableArray(), UnresolvedPaths = inspection.UnresolvedPaths.Length > 0 ? inspection.UnresolvedPaths : inspection.Targets }; + if (!inspection.Valid) return Required(inspection.Code); + try + { + var initial = ReadRecovery(journalPath); + using var coordination = host.Acquire(initial.Guards.Concat(initial.Entries.Select(e => e.Target)).Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal).ToArray()); + inspection = InspectRecovery(journalPath); + if (!inspection.Valid) return Required(inspection.Code); + var document = LatestRecoveryDocument(ReadRecovery(journalPath)); + if (preserved.Any(p => !document.PendingDirectories.Contains(p, StringComparer.Ordinal) + || host.Observe(p).Kind != ConfigurationObservationKind.Directory)) return Required("config.recovery_invalid_options"); + if (RecoveryUnresolved(document, RecoveryArtifacts(document, journalPath), preserved).Length > 0) return Required(inspection.Code); + if (!SameOperation(initial, document)) return Required("config.recovery_changed"); + if (document.Guards.Any(g => host.EnumerateTransactionArtifacts(g, document.TransactionId) is null)) + return Required("config.recovery_host_capability"); + var restored = new List(); + var unresolved = new List(); + foreach (var entry in document.Entries.Reverse()) + { + var expected = new ConfigurationObservation(entry.Target, entry.OriginalKind, entry.OriginalIdentity); + var candidate = new ConfigurationObservation(entry.Target, ConfigurationObservationKind.File, entry.CandidateIdentity); + try + { + var current = host.Observe(entry.Target); + if (document.Committed) + { + if (current != candidate) unresolved.Add(entry.Target); + continue; + } + if (current == expected) { restored.Add(entry.Target); continue; } + if (current != candidate) { unresolved.Add(entry.Target); continue; } + fault?.Invoke(ConfigurationTransactionPhase.BeforeRollback, entry.Target); + if (host.Observe(entry.Target) != candidate) { unresolved.Add(entry.Target); continue; } + if (entry.Backup is null) host.DeleteFile(entry.Target); + else + { + if (host.Observe(entry.Backup) != new ConfigurationObservation(entry.Backup, ConfigurationObservationKind.File, entry.OriginalIdentity)) + { unresolved.Add(entry.Target); continue; } + host.Install(entry.Backup, entry.Target, true); + } + fault?.Invoke(ConfigurationTransactionPhase.AfterRollback, entry.Target); + if (host.Observe(entry.Target) == expected) restored.Add(entry.Target); else unresolved.Add(entry.Target); + } + catch (Exception error) when (IsIo(error)) + { + if (host.Observe(entry.Target) == expected) restored.Add(entry.Target); else unresolved.Add(entry.Target); + } + } + var artifacts = RecoveryArtifacts(document, journalPath).ToList(); + if (unresolved.Count > 0 || !TryCleanup(artifacts, document.Committed, document)) + return new(ConfigurationTransactionStatus.RecoveryRequired, "config.recovery_required", inspection.Targets, Retained(artifacts)) + { RestoredPaths = restored.ToImmutableArray(), UnresolvedPaths = unresolved.Count > 0 ? unresolved.ToImmutableArray() : inspection.Targets }; + return new(document.Committed ? ConfigurationTransactionStatus.Written : ConfigurationTransactionStatus.RolledBack, + document.Committed ? "config.recovery_committed" : "config.recovery_rolled_back", inspection.Targets, []) + { RestoredPaths = restored.ToImmutableArray(), PreservedDirectories = preserved.Order(StringComparer.Ordinal).ToImmutableArray() }; + } + catch (Exception error) when (IsRecoveryError(error)) { return Required("config.recovery_io"); } + } + + private RecoveryDocument LatestRecoveryDocument(RecoveryDocument initial) + { + foreach (var role in new[] { "journal-committed", "journal-ready", "journal-directories" }) + { + var path = RecoveryDocument.ArtifactPath(initial.MetadataDestination, initial.TransactionId, role); + if (host.Observe(path).Kind == ConfigurationObservationKind.Missing) continue; + var snapshot = ReadRecovery(path); + if (!SameOperation(initial, snapshot) || (role == "journal-committed") != snapshot.Committed) + throw new InvalidDataException("Invalid recovery snapshot"); + return snapshot; + } + return initial; + } + + private RecoveryDocument ReadRecovery(string path) + { + var observed = host.Observe(path); + if (observed.Kind != ConfigurationObservationKind.File) throw new InvalidDataException("Journal is not a regular file"); + var bytes = host.ReadFile(path); + if (NativeConfigurationTransactionHost.Hash(bytes) != observed.Identity || host.Observe(path) != observed) + throw new InvalidDataException("Journal changed"); + return RecoveryDocument.Decode(bytes, path); + } + + private ImmutableArray RecoveryArtifacts(RecoveryDocument document, string journalPath) + { + var known = document.Artifacts.Append(new(journalPath, ConfigurationArtifactKind.Journal, document.TransactionId)).ToList(); + foreach (var directory in document.Guards) + { + var found = host.EnumerateTransactionArtifacts(directory, document.TransactionId); + if (found is null) continue; + foreach (var path in found) + known.Add(new(path, RecoveryDocument.IsJournal(path) ? ConfigurationArtifactKind.Journal + : path.EndsWith("-backup", StringComparison.Ordinal) ? ConfigurationArtifactKind.Backup : ConfigurationArtifactKind.Stage, document.TransactionId)); + } + return known.Distinct().ToImmutableArray(); + } + + private ImmutableArray RecoveryUnresolved(RecoveryDocument document, IEnumerable artifacts, + IReadOnlySet? preservedDirectories = null) + { + var unresolved = new HashSet(StringComparer.Ordinal); + foreach (var path in document.PendingDirectories) + { + var observed = host.Observe(path); + if (observed.Kind != ConfigurationObservationKind.Missing + && (observed.Kind != ConfigurationObservationKind.Directory || preservedDirectories?.Contains(path) != true)) unresolved.Add(path); + } + foreach (var entry in document.Entries) + { + var current = host.Observe(entry.Target); + var original = new ConfigurationObservation(entry.Target, entry.OriginalKind, entry.OriginalIdentity); + var candidate = new ConfigurationObservation(entry.Target, ConfigurationObservationKind.File, entry.CandidateIdentity); + if (document.Committed ? current != candidate : current != candidate && current != original) unresolved.Add(entry.Target); + if (!document.Committed && current == candidate && entry.Backup is not null + && host.Observe(entry.Backup) != new ConfigurationObservation(entry.Backup, ConfigurationObservationKind.File, entry.OriginalIdentity)) unresolved.Add(entry.Target); + } + foreach (var artifact in artifacts) + { + var observed = host.Observe(artifact.Path); + if (observed.Kind == ConfigurationObservationKind.Missing) continue; + if (artifact.Kind == ConfigurationArtifactKind.Directory) + { + if (observed.Kind != ConfigurationObservationKind.Directory) unresolved.Add(artifact.Path); + continue; + } + if (observed.Kind != ConfigurationObservationKind.File) { unresolved.Add(artifact.Path); continue; } + if (artifact.Kind == ConfigurationArtifactKind.Journal) + { + try { if (!SameOperation(document, ReadRecovery(artifact.Path))) unresolved.Add(artifact.Path); } + catch (Exception error) when (IsRecoveryError(error)) { unresolved.Add(artifact.Path); } + continue; + } + var entry = document.Entries.FirstOrDefault(e => e.Stage == artifact.Path || e.Backup == artifact.Path); + if (entry is null || observed.Identity != (entry.Stage == artifact.Path ? entry.CandidateIdentity : entry.OriginalIdentity)) unresolved.Add(artifact.Path); + } + return unresolved.Order(StringComparer.Ordinal).ToImmutableArray(); + } + + private static bool SameOperation(RecoveryDocument first, RecoveryDocument second) => first.TransactionId == second.TransactionId + && first.Guards.SequenceEqual(second.Guards) + && first.Entries.SequenceEqual(second.Entries); + private static bool IsRecoveryError(Exception error) => IsIo(error) || error is JsonException or InvalidDataException or InvalidOperationException or ArgumentException or KeyNotFoundException or FormatException or OverflowException; +} diff --git a/src/Morphir.Configuration.IO/ConfigurationTransaction.cs b/src/Morphir.Configuration.IO/ConfigurationTransaction.cs index 037ee9c1..2e610326 100644 --- a/src/Morphir.Configuration.IO/ConfigurationTransaction.cs +++ b/src/Morphir.Configuration.IO/ConfigurationTransaction.cs @@ -55,12 +55,20 @@ public sealed record ConfigurationTransactionResult( ConfigurationTransactionStatus Status, string Code, ImmutableArray Targets, - ImmutableArray Artifacts); + ImmutableArray Artifacts) +{ + public ImmutableArray RestoredPaths { get; init; } = []; + public ImmutableArray UnresolvedPaths { get; init; } = []; + public ImmutableArray PreservedDirectories { get; init; } = []; +} /// Explicit host boundary. Acquire must serialize overlapping paths across its writers; /// Install must atomically replace one file, never a group of files. public interface IConfigurationTransactionHost { + /// Explicit opt-in to persistent journal discovery and complete artifact enumeration. + /// Legacy/read-only hosts remain source compatible but cannot perform writes until qualified. + bool SupportsDurableRecovery => false; /// Paths are canonical absolute paths. File identities are uppercase SHA-256 hex of exact bytes. /// Directory identities must be stable inventories excluding only the supplied owned paths. ConfigurationObservation Observe(string path, IReadOnlyCollection? excludedPaths = null); @@ -73,11 +81,16 @@ public interface IConfigurationTransactionHost void Install(string stagedPath, string destination, bool overwrite); void DeleteFile(string path); void DeleteDirectory(string path); + /// Durable hosts enumerate metadata journals without following links. Older custom + /// hosts can still apply plans, but must implement enumeration for automatic overlap guards. + IReadOnlyList FindRecoveryJournals(IReadOnlyList paths) => []; + /// Null means this host cannot prove crash-artifact enumeration completeness. + IReadOnlyList? EnumerateTransactionArtifacts(string directory, string transactionId) => null; } /// Preconditioned configuration-file application. Each installation is atomic; /// a transaction with multiple destinations is not group atomic. -public sealed class ConfigurationTransaction +public sealed partial class ConfigurationTransaction { private readonly IConfigurationTransactionHost host; private readonly Action? fault; @@ -90,10 +103,18 @@ public ConfigurationTransaction(IConfigurationTransactionHost? host = null, Acti public static ConfigurationObservation Observe(string path) => new NativeConfigurationTransactionHost().Observe(path); + /// Stable owned-artifact naming for durable storage adapters. Recovery validates + /// the transaction owner, destination anchor and role rather than trusting a prefix alone. + public static string GetArtifactPath(string destination, string transactionId, string role) => + RecoveryDocument.ArtifactPath(destination, transactionId, role); + public ConfigurationTransactionResult Apply(ConfigurationTransactionPlan plan) { var targets = plan.Changes.Select(c => c.Path).ToImmutableArray(); - if (plan.Changes.Any(c => c.Path != c.Expected.Path + if (!host.SupportsDurableRecovery) + return new(ConfigurationTransactionStatus.IoFailure, "config.recovery_host_capability", targets, []); + if (targets.Distinct(StringComparer.Ordinal).Count() != targets.Length || plan.Changes.Any(c => c.Path != c.Expected.Path + || !ProjectPath.isAbsolute(c.Path) || ProjectPath.normalize(c.Path) != c.Path || c.Expected.Kind is not (ConfigurationObservationKind.File or ConfigurationObservationKind.Missing))) return new(ConfigurationTransactionStatus.IoFailure, "config.edit_invalid_plan", targets, []); var observations = plan.Observations.Concat(plan.Changes.Select(c => c.Expected)).Distinct().ToArray(); @@ -101,7 +122,11 @@ public ConfigurationTransactionResult Apply(ConfigurationTransactionPlan plan) var transactionId = Guid.NewGuid().ToString("N"); var artifacts = new List(); var attempted = new List<(ConfigurationFileChange Change, string? Backup)>(); + var staged = new List<(ConfigurationFileChange Change, string Stage, string? Backup)>(); var ownedDirectories = new Dictionary(StringComparer.Ordinal); + RecoveryDocument? document = null; + var restored = new List(); + var unresolved = new List(); bool ObservationsMatch() { var excluded = artifacts.Select(a => a.Path) @@ -110,21 +135,91 @@ bool ObservationsMatch() (attempted.FirstOrDefault(a => a.Change.Path == o.Path).Change is { } installed ? CandidateObservation(installed) : ownedDirectories.GetValueOrDefault(o.Path, o))); } + bool StagesMatch() => staged.All(s => + (attempted.Any(a => a.Change.Path == s.Change.Path) + || host.Observe(s.Stage) == new ConfigurationObservation(s.Stage, ConfigurationObservationKind.File, CandidateObservation(s.Change).Identity)) + && (s.Backup is null || host.Observe(s.Backup) == new ConfigurationObservation(s.Backup, ConfigurationObservationKind.File, s.Change.Expected.Identity))); IDisposable? coordination = null; try { coordination = host.Acquire(paths); + if (observations.Where(o => o.Kind == ConfigurationObservationKind.Directory).Select(o => o.Path) + .Concat(targets.Select(ProjectPath.parent)).Distinct(StringComparer.Ordinal) + .Any(directory => host.EnumerateTransactionArtifacts(directory, transactionId) is null)) + return new(ConfigurationTransactionStatus.IoFailure, "config.recovery_host_capability", targets, []); + var pending = InspectPending(paths); + if (pending.Length > 0) + return new(ConfigurationTransactionStatus.RecoveryRequired, "config.edit_pending_recovery", targets, + pending.SelectMany(p => p.Artifacts).Distinct().ToImmutableArray()) + { UnresolvedPaths = pending.SelectMany(p => p.UnresolvedPaths.Length > 0 ? p.UnresolvedPaths : p.Targets).Distinct().ToImmutableArray() }; if (!ObservationsMatch()) return new(ConfigurationTransactionStatus.Conflict, "config.edit_conflict", targets, []); - var changed = false; - foreach (var change in plan.Changes) + var changes = plan.Changes.Where(c => c.Expected.Kind != ConfigurationObservationKind.File + || !host.ReadFile(c.Path).AsSpan().SequenceEqual(c.Content.AsSpan())).ToArray(); + if (changes.Length == 0) return new(ConfigurationTransactionStatus.Unchanged, "config.edit_unchanged", targets, []); + var plannedDirectories = new HashSet(StringComparer.Ordinal); + foreach (var change in changes) { - if (change.Expected.Kind == ConfigurationObservationKind.File && host.ReadFile(change.Path).AsSpan().SequenceEqual(change.Content.AsSpan())) continue; - var createdDirectories = host.CreateParentDirectories(change.Path, transactionId); - foreach (var directory in createdDirectories) + for (var parent = ProjectPath.parent(change.Path); ; parent = ProjectPath.parent(parent)) + { + var observed = host.Observe(parent); + if (observed.Kind == ConfigurationObservationKind.Directory) break; + if (observed.Kind != ConfigurationObservationKind.Missing) throw new IOException("A target parent is not a regular directory"); + plannedDirectories.Add(parent); + if (ProjectPath.parent(parent) == parent) throw new IOException("No existing parent directory"); + } + } + var common = ProjectPath.parent(changes[0].Path); + foreach (var change in changes) + while (!ProjectPath.contains(common, change.Path)) common = ProjectPath.parent(common); + while (host.Observe(common).Kind == ConfigurationObservationKind.Missing) common = ProjectPath.parent(common); + if (host.Observe(common).Kind != ConfigurationObservationKind.Directory) throw new IOException("No regular common ancestor"); + var guards = observations.Where(o => o.Kind == ConfigurationObservationKind.Directory + && changes.Any(c => ProjectPath.contains(o.Path, c.Path))).Select(o => o.Path) + .Concat(changes.Select(c => ProjectPath.parent(c.Path))).Append(common) + .Distinct(StringComparer.Ordinal).OrderBy(p => p.Length).ThenBy(p => p, StringComparer.Ordinal).ToImmutableArray(); + var entries = changes.Select(c => new RecoveryEntry(c.Path, c.Expected.Kind, c.Expected.Identity, CandidateObservation(c).Identity, + RecoveryDocument.ArtifactPath(c.Path, transactionId, "stage"), + c.Expected.Kind == ConfigurationObservationKind.File ? RecoveryDocument.ArtifactPath(c.Path, transactionId, "backup") : null)).ToImmutableArray(); + var plannedArtifacts = entries.SelectMany(e => e.Backup is null + ? new[] { new ConfigurationArtifact(e.Stage, ConfigurationArtifactKind.Stage, transactionId) } + : new[] { new ConfigurationArtifact(e.Stage, ConfigurationArtifactKind.Stage, transactionId), new ConfigurationArtifact(e.Backup, ConfigurationArtifactKind.Backup, transactionId) }) + .Concat(guards.Select(g => new ConfigurationArtifact(RecoveryDocument.ArtifactPath(ProjectPath.join(g, ".morphir-transaction"), transactionId, "journal"), ConfigurationArtifactKind.Journal, transactionId))) + .Concat(artifacts).ToImmutableArray(); + document = new(transactionId, false, guards, entries, plannedArtifacts) + { PendingDirectories = plannedDirectories.Order(StringComparer.Ordinal).ToImmutableArray() }; + // Write-ahead metadata is durable before any candidate or original backup is staged. + // The common target ancestor is first, so even an interrupted setup blocks its descendants. + foreach (var directory in guards.OrderBy(g => g == common ? 0 : 1).ThenBy(g => g, StringComparer.Ordinal)) + { + if (host.Observe(directory).Kind == ConfigurationObservationKind.Missing) continue; + var journal = host.Stage(ProjectPath.join(directory, ".morphir-transaction"), document.Encode(), transactionId, "journal"); + artifacts.Add(new(journal, ConfigurationArtifactKind.Journal, transactionId)); + } + foreach (var change in changes) + { + var created = host.CreateParentDirectories(change.Path, transactionId); + foreach (var directory in created) artifacts.Add(new(directory, ConfigurationArtifactKind.Directory, transactionId)); - foreach (var directory in createdDirectories) + foreach (var directory in created) ownedDirectories[directory] = host.Observe(directory, artifacts.Select(a => a.Path).ToArray()); + } + document = document with + { + Artifacts = artifacts.ToImmutableArray(), + PendingDirectories = plannedDirectories.Where(p => !artifacts.Any(a => a.Kind == ConfigurationArtifactKind.Directory && a.Path == p) + && host.Observe(p).Kind != ConfigurationObservationKind.Missing).Order(StringComparer.Ordinal).ToImmutableArray() + }; + if (document.PendingDirectories.Length > 0) throw new IOException("Directory creation ownership is uncertain"); + var directoryReceipt = host.Stage(document.MetadataDestination, document.Encode(), transactionId, "journal-directories"); + artifacts.Add(new(directoryReceipt, ConfigurationArtifactKind.Journal, transactionId)); + foreach (var directory in guards.Where(g => !artifacts.Any(a => a.Kind == ConfigurationArtifactKind.Journal && ProjectPath.parent(a.Path) == g))) + { + var journal = host.Stage(ProjectPath.join(directory, ".morphir-transaction"), document.Encode(), transactionId, "journal"); + artifacts.Add(new(journal, ConfigurationArtifactKind.Journal, transactionId)); + } + foreach (var change in changes) + { fault?.Invoke(ConfigurationTransactionPhase.BeforeStage, change.Path); var stage = host.Stage(change.Path, change.Content.AsMemory(), transactionId, "stage"); artifacts.Add(new(stage, ConfigurationArtifactKind.Stage, transactionId)); @@ -134,21 +229,30 @@ bool ObservationsMatch() backup = host.Stage(change.Path, host.ReadFile(change.Path), transactionId, "backup"); artifacts.Add(new(backup, ConfigurationArtifactKind.Backup, transactionId)); } - var journal = host.Stage(change.Path, RecoveryJournal(change, stage, backup, transactionId), transactionId, "journal"); - artifacts.Add(new(journal, ConfigurationArtifactKind.Journal, transactionId)); + staged.Add((change, stage, backup)); + } + // A ready snapshot records actual artifact paths for non-native storage adapters. + document = document with { Entries = staged.Select(s => new RecoveryEntry(s.Change.Path, s.Change.Expected.Kind, + s.Change.Expected.Identity, CandidateObservation(s.Change).Identity, s.Stage, s.Backup)).ToImmutableArray(), Artifacts = artifacts.ToImmutableArray() }; + var ready = host.Stage(document.MetadataDestination, document.Encode(), transactionId, "journal-ready"); + artifacts.Add(new(ready, ConfigurationArtifactKind.Journal, transactionId)); + if (!StagesMatch() || !ObservationsMatch()) throw new ConflictException(); + foreach (var (change, stage, backup) in staged) + { fault?.Invoke(ConfigurationTransactionPhase.BeforeInstall, change.Path); - if (host.Observe(stage) != new ConfigurationObservation(stage, ConfigurationObservationKind.File, CandidateObservation(change).Identity) - || (backup is not null && host.Observe(backup) != new ConfigurationObservation(backup, ConfigurationObservationKind.File, change.Expected.Identity)) - || !ObservationsMatch()) throw new ConflictException(); + if (!StagesMatch() || !ObservationsMatch()) throw new ConflictException(); attempted.Add((change, backup)); host.Install(stage, change.Path, change.Expected.Kind == ConfigurationObservationKind.File); - changed = true; fault?.Invoke(ConfigurationTransactionPhase.AfterInstall, change.Path); } - if (!TryCleanup(artifacts, true)) - return new(ConfigurationTransactionStatus.RecoveryRequired, "config.edit_cleanup_required", targets, Retained(artifacts)); - return new(changed ? ConfigurationTransactionStatus.Written : ConfigurationTransactionStatus.Unchanged, - changed ? "config.edit_written" : "config.edit_unchanged", targets, []); + if (!StagesMatch() || !ObservationsMatch()) throw new ConflictException(); + document = document with { Committed = true, Artifacts = artifacts.ToImmutableArray() }; + var committed = host.Stage(document.MetadataDestination, document.Encode(), transactionId, "journal-committed"); + artifacts.Add(new(committed, ConfigurationArtifactKind.Journal, transactionId)); + if (!TryCleanup(artifacts, true, document)) + return new(ConfigurationTransactionStatus.RecoveryRequired, "config.edit_cleanup_required", targets, Retained(artifacts)) + { UnresolvedPaths = targets }; + return new(ConfigurationTransactionStatus.Written, "config.edit_written", targets, []); } catch (Exception error) when (IsIo(error)) { @@ -158,95 +262,93 @@ bool ObservationsMatch() artifacts.AddRange(artifactFailure.Artifacts.Where(a => a.TransactionId == transactionId)); foreignArtifacts = artifactFailure.Artifacts.Where(a => a.TransactionId != transactionId).ToImmutableArray(); } - var recovered = foreignArtifacts.Length == 0; + if (document is not null) + foreach (var directory in document.PendingDirectories) + if (!artifacts.Any(a => a.Kind == ConfigurationArtifactKind.Directory && a.Path == directory) + && host.Observe(directory).Kind != ConfigurationObservationKind.Missing) unresolved.Add(directory); foreach (var (change, backup) in attempted.AsEnumerable().Reverse()) { try { var current = host.Observe(change.Path); - if (current == change.Expected) continue; - var candidate = new ConfigurationObservation(change.Path, ConfigurationObservationKind.File, - NativeConfigurationTransactionHost.Hash(change.Content.AsSpan())); - if (current != candidate) { recovered = false; continue; } + if (current == change.Expected) { restored.Add(change.Path); continue; } + var candidate = CandidateObservation(change); + if (current != candidate) { unresolved.Add(change.Path); continue; } fault?.Invoke(ConfigurationTransactionPhase.BeforeRollback, change.Path); - if (host.Observe(change.Path) != candidate || (backup is not null && host.Observe(backup) != - new ConfigurationObservation(backup, ConfigurationObservationKind.File, change.Expected.Identity))) - { recovered = false; continue; } + if (host.Observe(change.Path) != candidate || backup is not null && host.Observe(backup) != + new ConfigurationObservation(backup, ConfigurationObservationKind.File, change.Expected.Identity)) + { unresolved.Add(change.Path); continue; } if (backup is null) host.DeleteFile(change.Path); else host.Install(backup, change.Path, true); fault?.Invoke(ConfigurationTransactionPhase.AfterRollback, change.Path); - if (host.Observe(change.Path) != change.Expected) recovered = false; + if (host.Observe(change.Path) == change.Expected) restored.Add(change.Path); + else unresolved.Add(change.Path); } catch (Exception recoveryError) when (IsIo(recoveryError)) { - try { if (host.Observe(change.Path) != change.Expected) recovered = false; } - catch (Exception observationError) when (IsIo(observationError)) { recovered = false; } + try { if (host.Observe(change.Path) == change.Expected) restored.Add(change.Path); else unresolved.Add(change.Path); } + catch (Exception observationError) when (IsIo(observationError)) { unresolved.Add(change.Path); } } } - if (!recovered) + if (unresolved.Count > 0 || foreignArtifacts.Length > 0) return new(ConfigurationTransactionStatus.RecoveryRequired, "config.edit_recovery_required", targets, - Retained(artifacts).AddRange(foreignArtifacts)); - if (!TryCleanup(artifacts, false)) - return new(ConfigurationTransactionStatus.RecoveryRequired, "config.edit_cleanup_required", targets, Retained(artifacts)); + Retained(artifacts).AddRange(foreignArtifacts)) { RestoredPaths = restored.ToImmutableArray(), UnresolvedPaths = unresolved.ToImmutableArray() }; + if (!TryCleanup(artifacts, false, document)) + return new(ConfigurationTransactionStatus.RecoveryRequired, "config.edit_cleanup_required", targets, Retained(artifacts)) + { RestoredPaths = restored.ToImmutableArray(), UnresolvedPaths = targets }; return new(error is ConflictException ? ConfigurationTransactionStatus.Conflict : attempted.Count > 0 ? ConfigurationTransactionStatus.RolledBack : ConfigurationTransactionStatus.IoFailure, error is ConflictException ? "config.edit_conflict" - : attempted.Count > 0 ? "config.edit_rolled_back" : "config.edit_io", targets, []); + : attempted.Count > 0 ? "config.edit_rolled_back" : "config.edit_io", targets, []) { RestoredPaths = restored.ToImmutableArray() }; } finally { coordination?.Dispose(); } } - private bool TryCleanup(List artifacts, bool committed) + private bool TryCleanup(List artifacts, bool committed, RecoveryDocument? document) { if (artifacts.Count == 0) return true; try { fault?.Invoke(ConfigurationTransactionPhase.BeforeCleanup, artifacts[0].Path); - foreach (var artifact in artifacts.AsEnumerable().Reverse()) - if (artifact.Kind == ConfigurationArtifactKind.Directory) host.DeleteDirectory(artifact.Path); + // Keep metadata outside newly created directories until directory cleanup is proven. + // A committed witness remains last, above all destinations, throughout cleanup. + var directories = artifacts.Where(a => a.Kind == ConfigurationArtifactKind.Directory).Select(a => a.Path).ToHashSet(StringComparer.Ordinal); + int Order(ConfigurationArtifact artifact) => artifact.Kind != ConfigurationArtifactKind.Journal + ? artifact.Kind == ConfigurationArtifactKind.Directory ? 2 : 0 + : directories.Contains(ProjectPath.parent(artifact.Path)) ? 1 + : artifact.Path.EndsWith("-journal-committed", StringComparison.Ordinal) ? 4 : 3; + foreach (var artifact in artifacts.OrderBy(Order).ThenByDescending(a => a.Path.Length)) + { + if (artifact.Kind == ConfigurationArtifactKind.Directory) + { + host.DeleteDirectory(artifact.Path); + if (!committed && host.Observe(artifact.Path).Kind != ConfigurationObservationKind.Missing) + return false; + } else host.DeleteFile(artifact.Path); + } return Retained(artifacts).All(a => committed && a.Kind == ConfigurationArtifactKind.Directory); } catch (Exception error) when (IsIo(error)) { return false; } } private ImmutableArray Retained(IEnumerable artifacts) => - artifacts.Where(a => + artifacts.Distinct().Where(a => { try { return host.Observe(a.Path).Kind != ConfigurationObservationKind.Missing; } catch (Exception error) when (IsIo(error)) { return true; } }).ToImmutableArray(); - private static byte[] RecoveryJournal(ConfigurationFileChange change, string stage, string? backup, string transactionId) - { - using var memory = new MemoryStream(); - using (var writer = new Utf8JsonWriter(memory)) - { - writer.WriteStartObject(); - writer.WriteNumber("version", 1); - writer.WriteString("transactionId", transactionId); - writer.WriteString("target", change.Path); - writer.WriteString("originalKind", change.Expected.Kind.ToString()); - writer.WriteString("originalIdentity", change.Expected.Identity); - writer.WriteString("candidateIdentity", CandidateObservation(change).Identity); - writer.WriteString("stage", stage); - writer.WriteString("backup", backup); - writer.WriteEndObject(); - } - return memory.ToArray(); - } - private static ConfigurationObservation CandidateObservation(ConfigurationFileChange change) => new(change.Path, ConfigurationObservationKind.File, NativeConfigurationTransactionHost.Hash(change.Content.AsSpan())); - private sealed class ConflictException : IOException; - private static bool IsIo(Exception error) => error is IOException or UnauthorizedAccessException or NotSupportedException; } /// Native same-filesystem stages and per-file atomic installation. public sealed class NativeConfigurationTransactionHost : IConfigurationTransactionHost { + public bool SupportsDurableRecovery => true; public ConfigurationObservation Observe(string path, IReadOnlyCollection? excludedPaths = null) { path = Canonical(path); @@ -281,6 +383,37 @@ public ConfigurationObservation Observe(string path, IReadOnlyCollection return new(path, ConfigurationObservationKind.Missing, ""); } + public IReadOnlyList FindRecoveryJournals(IReadOnlyList paths) + { + var directories = new HashSet(StringComparer.Ordinal); + foreach (var path in paths) + { + var current = Observe(path).Kind == ConfigurationObservationKind.Directory ? Canonical(path) : ProjectPath.parent(Canonical(path)); + while (true) + { + directories.Add(current); + var parent = ProjectPath.parent(current); + if (parent == current) break; + current = parent; + } + } + var result = new List(); + foreach (var directory in directories.Order(StringComparer.Ordinal)) + { + if (Observe(directory).Kind != ConfigurationObservationKind.Directory) continue; + result.AddRange(Directory.EnumerateFileSystemEntries(directory, ".morphir-edit-*") + .Select(Canonical).Where(RecoveryDocument.IsJournal)); + } + return result.Order(StringComparer.Ordinal).ToArray(); + } + + public IReadOnlyList? EnumerateTransactionArtifacts(string directory, string transactionId) + { + if (Observe(directory).Kind == ConfigurationObservationKind.Missing) return []; + if (Observe(directory).Kind != ConfigurationObservationKind.Directory) throw new IOException("Recovery directory is not a regular directory"); + return Directory.EnumerateFileSystemEntries(directory, ".morphir-edit-" + transactionId + "-*").Select(Canonical).Order(StringComparer.Ordinal).ToArray(); + } + public byte[] ReadFile(string path) => File.ReadAllBytes(path); public IDisposable Acquire(IReadOnlyList paths) => new NativeLease(); @@ -325,7 +458,7 @@ public IReadOnlyList CreateParentDirectories(string destination, string public string Stage(string destination, ReadOnlyMemory content, string transactionId, string role) { - var path = Path.Combine(Path.GetDirectoryName(destination)!, ".morphir-edit-" + transactionId + "-" + Guid.NewGuid().ToString("N") + "-" + role); + var path = RecoveryDocument.ArtifactPath(Canonical(destination), transactionId, role); var created = false; try { diff --git a/tests/Morphir.Configuration.IO.Tests/ConfigurationBatchRecovery.feature b/tests/Morphir.Configuration.IO.Tests/ConfigurationBatchRecovery.feature new file mode 100644 index 00000000..3ce352d3 --- /dev/null +++ b/tests/Morphir.Configuration.IO.Tests/ConfigurationBatchRecovery.feature @@ -0,0 +1,21 @@ +Feature: Recoverable configuration batches + As a workspace owner + I want interrupted batches to retain inspectable evidence + So that overlapping tools cannot overwrite unfinished changes + + Scenario: Complete batch staging precedes replacements + Given a recoverable configuration batch + When the host stages the complete batch + Then no replacement starts before all candidates are staged + + Scenario: Shared editing and project migration respect pending recovery + Given a recoverable configuration batch + When a batch replacement and its rollback both fail + Then shared configuration editing and project migration refuse overlapping writes + And explicit shared recovery restores the originals and clears the guard + + Scenario: User changes survive explicit recovery + Given a recoverable configuration batch + When a batch replacement and its rollback both fail + And the user changes an installed target + Then shared recovery reports the changed target and preserves its journal diff --git a/tests/Morphir.Configuration.IO.Tests/ConfigurationBatchRecoverySteps.cs b/tests/Morphir.Configuration.IO.Tests/ConfigurationBatchRecoverySteps.cs new file mode 100644 index 00000000..47c8b294 --- /dev/null +++ b/tests/Morphir.Configuration.IO.Tests/ConfigurationBatchRecoverySteps.cs @@ -0,0 +1,90 @@ +using System.Text; +using FluentAssertions; +using Morphir.Configuration.IO; + +namespace Morphir.Configuration.IO.Tests; + +[Binding] +public sealed class ConfigurationBatchRecoverySteps : IDisposable +{ + private readonly string root = ProjectPath.join(Path.GetTempPath(), "batch-recovery-bdd-" + Guid.NewGuid().ToString("N")); + private ConfigurationTransactionResult? failed; + private bool resolved; + private string Target => root + "/morphir.toml"; + private const string Candidate = "[project]\nname='App'\n"; + + [Given("a recoverable configuration batch")] + public void GivenBatch() + { + Directory.CreateDirectory(root); + File.WriteAllText(root + "/morphir.json", "{name:'App',exposedModules:[]}\n"); + } + + [When("the host stages the complete batch")] + public void Stage() => new ConfigurationBatchTransactionTests().EntireBatchStagesBeforeTheFirstReplacement(); + + [Then("no replacement starts before all candidates are staged")] + public void Staged() => resolved = true; + + [When("a batch replacement and its rollback both fail")] + public void FailBatch() + { + failed = new ConfigurationTransaction(fault: (phase, _) => + { + if (phase is ConfigurationTransactionPhase.AfterInstall or ConfigurationTransactionPhase.BeforeRollback) throw new IOException("injected"); + }).Apply(new ConfigurationTransactionPlan([ConfigurationTransaction.Observe(root)], + [new ConfigurationFileChange(Target, ConfigurationTransaction.Observe(Target), Encoding.UTF8.GetBytes(Candidate))])); + failed.Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); + } + + [Then("shared configuration editing and project migration refuse overlapping writes")] + public void SharedGuards() + { + var host = new NativeConfigurationTransactionHost(); + var tree = new FileSystemProjectTree(root); + var editing = new ConfigurationEditService(tree, host); + var preview = editing.Plan(root, new EditRequest(EditOperation.NewSet("true", EditValueType.Boolean), "custom.changed")); + preview.Failure.Should().BeNull(); + editing.Apply(preview).Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); + var migration = new ProjectMigrationService(tree, host); + var project = migration.Plan(root); + project.CanApply.Should().BeTrue(string.Join(",", project.Diagnostics.Select(d => d.Code))); + migration.Apply(project).Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); + File.ReadAllText(Target).Should().Be(Candidate); + } + + [Then("explicit shared recovery restores the originals and clears the guard")] + public void Recover() + { + var journal = failed!.Artifacts.First(a => a.Kind == ConfigurationArtifactKind.Journal).Path; + var transaction = new ConfigurationTransaction(); + transaction.InspectRecovery(journal).Valid.Should().BeTrue(); + transaction.Recover(journal).Status.Should().Be(ConfigurationTransactionStatus.RolledBack); + File.Exists(Target).Should().BeFalse(); + transaction.InspectPending([root]).Should().BeEmpty(); + resolved = true; + } + + [When("the user changes an installed target")] + public void ChangeUser() => File.WriteAllText(Target, "user=true\n"); + + [Then("shared recovery reports the changed target and preserves its journal")] + public void PreserveUser() + { + var journal = failed!.Artifacts.First(a => a.Kind == ConfigurationArtifactKind.Journal).Path; + var result = new ConfigurationTransaction().Recover(journal); + result.Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); + result.UnresolvedPaths.Should().Contain(Target); + File.ReadAllText(Target).Should().Be("user=true\n"); + File.Exists(journal).Should().BeTrue(); + // Restore only the fixture-owned candidate after checking user drift is preserved. + File.WriteAllText(Target, Candidate); + Recover(); + } + + public void Dispose() + { + if (resolved) Directory.Delete(root, true); + else Console.Error.WriteLine("Retained batch recovery fixture: " + root); + } +} diff --git a/tests/Morphir.Configuration.IO.Tests/ConfigurationBatchTransactionTests.cs b/tests/Morphir.Configuration.IO.Tests/ConfigurationBatchTransactionTests.cs new file mode 100644 index 00000000..3681e4d6 --- /dev/null +++ b/tests/Morphir.Configuration.IO.Tests/ConfigurationBatchTransactionTests.cs @@ -0,0 +1,174 @@ +using System.Text; +using FluentAssertions; +using Morphir.Configuration.IO; + +namespace Morphir.Configuration.IO.Tests; + +public sealed class ConfigurationBatchTransactionTests +{ + [Test] + public void EntireBatchStagesBeforeTheFirstReplacement() + { + var root = ProjectPath.join(Path.GetTempPath(), "batch-stage-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + try + { + var first = root + "/first.toml"; + var second = root + "/second.toml"; + File.WriteAllText(first, "original=1\n"); + File.WriteAllText(second, "original=2\n"); + var staged = new List(); + var plan = new ConfigurationTransactionPlan([ConfigurationTransaction.Observe(root)], + new[] { first, second }.Select(path => new ConfigurationFileChange(path, ConfigurationTransaction.Observe(path), Encoding.UTF8.GetBytes("candidate=true\n")))); + var result = new ConfigurationTransaction(fault: (phase, path) => + { + if (phase == ConfigurationTransactionPhase.BeforeStage) staged.Add(path); + if (phase == ConfigurationTransactionPhase.BeforeInstall) + staged.Should().Contain(second, "all destinations must stage before replacing the first destination"); + }).Apply(plan); + result.Status.Should().Be(ConfigurationTransactionStatus.Written); + } + finally { Directory.Delete(root, true); } + } + + [Test] + public void LaterStageFailureDoesNotReplaceAnyDestination() + { + var root = ProjectPath.join(Path.GetTempPath(), "batch-stage-fault-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + try + { + var first = root + "/first.toml"; + var second = root + "/second.toml"; + File.WriteAllText(first, "original=1\n"); + File.WriteAllText(second, "original=2\n"); + var installs = 0; + var plan = new ConfigurationTransactionPlan([ConfigurationTransaction.Observe(root)], + new[] { first, second }.Select(path => new ConfigurationFileChange(path, ConfigurationTransaction.Observe(path), Encoding.UTF8.GetBytes("candidate=true\n")))); + var result = new ConfigurationTransaction(fault: (phase, path) => + { + if (phase == ConfigurationTransactionPhase.BeforeStage && path == second) throw new IOException("stage unavailable"); + if (phase == ConfigurationTransactionPhase.AfterInstall) installs++; + }).Apply(plan); + installs.Should().Be(0); + result.Status.Should().Be(ConfigurationTransactionStatus.IoFailure); + File.ReadAllText(first).Should().Be("original=1\n"); + File.ReadAllText(second).Should().Be("original=2\n"); + } + finally { Directory.Delete(root, true); } + } +} + +public sealed class ConfigurationBatchRecoveryBoundaryTests +{ + [Test] + public void RecoveryMetadataDoesNotWriteIntoObservedExcludedSiblingDirectories() + { + var root = ProjectPath.join(Path.GetTempPath(), "batch-metadata-bound-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root + "/excluded"); + try + { + var target = root + "/morphir.toml"; + var host = new MetadataHost(root + "/excluded"); + var plan = new ConfigurationTransactionPlan([host.Observe(root), host.Observe(root + "/excluded")], + [new ConfigurationFileChange(target, host.Observe(target), Encoding.UTF8.GetBytes("value=true\n"))]); + new ConfigurationTransaction(host).Apply(plan).Status.Should().Be(ConfigurationTransactionStatus.Written, + "read-only inventory evidence does not authorize transaction artifacts in excluded siblings"); + Directory.GetFileSystemEntries(root + "/excluded").Should().BeEmpty(); + } + finally { Directory.Delete(root, true); } + } + + [Test] + public void CleanupInterruptionKeepsTheCommonAncestorGuardForSiblingWriters() + { + var root = ProjectPath.join(Path.GetTempPath(), "batch-cleanup-guard-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root + "/one"); + Directory.CreateDirectory(root + "/two"); + var resolved = false; + try + { + var first = root + "/one/morphir.toml"; + var second = root + "/two/morphir.toml"; + var host = new CleanupHost(root); + var plan = new ConfigurationTransactionPlan([host.Observe(root)], new[] { first, second } + .Select(p => new ConfigurationFileChange(p, host.Observe(p), Encoding.UTF8.GetBytes("value=true\n")))); + var failed = new ConfigurationTransaction(host).Apply(plan); + failed.Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); + var siblingPlan = new ConfigurationTransactionPlan([ConfigurationTransaction.Observe(root)], + [new ConfigurationFileChange(second, ConfigurationTransaction.Observe(second), Encoding.UTF8.GetBytes("new=true\n"))]); + new ConfigurationTransaction().Apply(siblingPlan).Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); + var journal = failed.Artifacts.First(a => a.Kind == ConfigurationArtifactKind.Journal).Path; + new ConfigurationTransaction().Recover(journal).Status.Should().Be(ConfigurationTransactionStatus.Written); + File.ReadAllText(second).Should().Be("value=true\n"); + resolved = true; + } + finally + { + if (resolved) Directory.Delete(root, true); + else Console.Error.WriteLine("Retained cleanup interruption evidence: " + root); + } + } + + private class MetadataHost(string forbidden) : IConfigurationTransactionHost + { + protected readonly NativeConfigurationTransactionHost Native = new(); + public bool SupportsDurableRecovery => Native.SupportsDurableRecovery; + public ConfigurationObservation Observe(string path, IReadOnlyCollection? excludedPaths = null) => Native.Observe(path, excludedPaths); + public byte[] ReadFile(string path) => Native.ReadFile(path); + public IDisposable Acquire(IReadOnlyList paths) => Native.Acquire(paths); + public IReadOnlyList CreateParentDirectories(string destination) => Native.CreateParentDirectories(destination); + public string Stage(string destination, ReadOnlyMemory content, string transactionId, string role) + { + if (ProjectPath.contains(forbidden, destination)) throw new IOException("Excluded sibling is not writable"); + return Native.Stage(destination, content, transactionId, role); + } + public void Install(string stagedPath, string destination, bool overwrite) => Native.Install(stagedPath, destination, overwrite); + public virtual void DeleteFile(string path) => Native.DeleteFile(path); + public void DeleteDirectory(string path) => Native.DeleteDirectory(path); + public IReadOnlyList FindRecoveryJournals(IReadOnlyList paths) => Native.FindRecoveryJournals(paths); + public IReadOnlyList? EnumerateTransactionArtifacts(string directory, string transactionId) => Native.EnumerateTransactionArtifacts(directory, transactionId); + } + + private sealed class CleanupHost(string root) : MetadataHost(root + "/not-used") + { + private bool failed; + public override void DeleteFile(string path) + { + Native.DeleteFile(path); + if (!failed && ProjectPath.parent(path) == root && path.EndsWith("-journal", StringComparison.Ordinal)) + { + failed = true; + throw new IOException("Crash window after root journal deletion"); + } + } + } +} + +public sealed class ConfigurationFinalPreconditionTests +{ + [Test] + public void SourceDriftAfterTheFinalReplacementIsDetectedBeforeCommitIsDeclared() + { + var root = ProjectPath.join(Path.GetTempPath(), "batch-final-precondition-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + try + { + var source = root + "/morphir.json"; + var target = root + "/morphir.toml"; + File.WriteAllText(source, "source-before\n"); + File.WriteAllText(target, "original=true\n"); + var plan = new ConfigurationTransactionPlan([ConfigurationTransaction.Observe(root), ConfigurationTransaction.Observe(source)], + [new ConfigurationFileChange(target, ConfigurationTransaction.Observe(target), Encoding.UTF8.GetBytes("candidate=true\n"))]); + var result = new ConfigurationTransaction(fault: (phase, _) => + { + if (phase == ConfigurationTransactionPhase.AfterInstall) File.WriteAllText(source, "source-after\n"); + }).Apply(plan); + result.Status.Should().Be(ConfigurationTransactionStatus.Conflict); + File.ReadAllText(target).Should().Be("original=true\n"); + File.ReadAllText(source).Should().Be("source-after\n"); + result.Artifacts.Should().BeEmpty(); + } + finally { Directory.Delete(root, true); } + } +} diff --git a/tests/Morphir.Configuration.IO.Tests/ConfigurationEditingTests.cs b/tests/Morphir.Configuration.IO.Tests/ConfigurationEditingTests.cs index fcd5c466..2b1a9517 100644 --- a/tests/Morphir.Configuration.IO.Tests/ConfigurationEditingTests.cs +++ b/tests/Morphir.Configuration.IO.Tests/ConfigurationEditingTests.cs @@ -312,6 +312,9 @@ public void ExplicitAlternateTomlPrimaryCannotIntroduceAmbiguity() private sealed class FaultingSnapshotHost(IConfigurationTransactionHost inner, string mode) : IConfigurationTransactionHost { + public bool SupportsDurableRecovery => inner.SupportsDurableRecovery; + public IReadOnlyList FindRecoveryJournals(IReadOnlyList paths) => inner.FindRecoveryJournals(paths); + public IReadOnlyList? EnumerateTransactionArtifacts(string directory, string transactionId) => inner.EnumerateTransactionArtifacts(directory, transactionId); private int targetObservations; public ConfigurationObservation Observe(string path, IReadOnlyCollection? excludedPaths = null) { diff --git a/tests/Morphir.Configuration.IO.Tests/ConfigurationRecoveryProcessTests.cs b/tests/Morphir.Configuration.IO.Tests/ConfigurationRecoveryProcessTests.cs new file mode 100644 index 00000000..8b5f3f24 --- /dev/null +++ b/tests/Morphir.Configuration.IO.Tests/ConfigurationRecoveryProcessTests.cs @@ -0,0 +1,139 @@ +using System.Diagnostics; +using System.Text; +using FluentAssertions; +using Morphir.Configuration.IO; + +namespace Morphir.Configuration.IO.Tests; + +public sealed class ConfigurationRecoveryProcessTests +{ + [Test] + [Arguments("BeforeStage:first")] + [Arguments("BeforeStage:second")] + [Arguments("BeforeInstall:first")] + [Arguments("AfterInstall:first")] + [Arguments("AfterInstall:second")] + [Arguments("BeforeCleanup:any")] + public async Task ProcessInterruptionRetainsPersistentRecoveryAndBlocksAnotherProcess(string window) + { + var root = ProjectPath.join(Path.GetTempPath(), "recovery-process-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + var resolved = false; + try + { + File.WriteAllText(root + "/first.toml", "original=1\n"); + File.WriteAllText(root + "/second.toml", "original=2\n"); + var crash = await Child(root, window); + crash.ExitCode.Should().Be(79, crash.Output); + var transaction = new ConfigurationTransaction(); + var pending = transaction.InspectPending([root, root + "/first.toml"]); + pending.Should().NotBeEmpty("a process exit cannot erase durable write-ahead state"); + pending.Should().OnlyContain(p => p.Valid); + var blocked = await Child(root, "verify-blocked"); + blocked.ExitCode.Should().Be(0, blocked.Output); + var outcome = transaction.Recover(pending[0].JournalPath); + var committed = window.StartsWith("BeforeCleanup", StringComparison.Ordinal); + outcome.Status.Should().Be(committed ? ConfigurationTransactionStatus.Written : ConfigurationTransactionStatus.RolledBack); + File.ReadAllText(root + "/first.toml").Should().Be(committed ? "candidate=true\n" : "original=1\n"); + File.ReadAllText(root + "/second.toml").Should().Be(committed ? "candidate=true\n" : "original=2\n"); + transaction.InspectPending([root]).Should().BeEmpty(); + Directory.GetFiles(root, ".morphir-edit-*").Should().BeEmpty(); + resolved = true; + } + finally + { + if (resolved) Directory.Delete(root, true); + else Console.Error.WriteLine("Retained process recovery evidence: " + root); + } + } + + [Test] + public async Task ProcessExitAfterDirectoryCreationBeforeHostReturnRetainsAnUncertainOwnershipJournal() + { + var root = ProjectPath.join(Path.GetTempPath(), "recovery-parent-process-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + var resolved = false; + try + { + var child = await Child(root, "parent-create"); + child.ExitCode.Should().Be(79, child.Output); + Directory.Exists(root + "/new").Should().BeTrue(); + var transaction = new ConfigurationTransaction(); + var pending = transaction.InspectPending([root]); + pending.Should().NotBeEmpty("a write-ahead guard must exist before the first mkdir"); + pending[0].UncertainDirectories.Should().Contain(root + "/new"); + // Ownership is uncertain because the interrupted host never returned its creation receipt. + transaction.Recover(pending[0].JournalPath).Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); + Directory.Exists(root + "/new").Should().BeTrue("recovery cannot assume an empty directory belongs to it"); + var recovered = transaction.Recover(pending[0].JournalPath, [root + "/new"]); + recovered.Status.Should().Be(ConfigurationTransactionStatus.RolledBack); + recovered.PreservedDirectories.Should().Equal(root + "/new"); + Directory.Exists(root + "/new").Should().BeTrue(); + transaction.InspectPending([root]).Should().BeEmpty(); + resolved = true; + } + finally + { + if (resolved) Directory.Delete(root, true); + else Console.Error.WriteLine("Retained parent-creation crash evidence: " + root); + } + } + + [Test] + public void RecoveryChildProcess() + { + var root = Environment.GetEnvironmentVariable("MORPHIR_RECOVERY_CHILD_ROOT"); + if (root is null) return; + var window = Environment.GetEnvironmentVariable("MORPHIR_RECOVERY_CHILD_WINDOW")!; + var paths = window == "parent-create" ? new[] { root + "/new/first.toml", root + "/new/second.toml" } + : new[] { root + "/first.toml", root + "/second.toml" }; + var plan = new ConfigurationTransactionPlan([ConfigurationTransaction.Observe(root)], paths.Select(path => + new ConfigurationFileChange(path, ConfigurationTransaction.Observe(path), Encoding.UTF8.GetBytes("candidate=true\n")))); + IConfigurationTransactionHost host = window == "parent-create" ? new ParentExitHost() : new NativeConfigurationTransactionHost(); + var result = new ConfigurationTransaction(host, (phase, path) => + { + var point = phase + ":" + (path == paths[0] ? "first" : path == paths[1] ? "second" : "any"); + if (point == window || phase == ConfigurationTransactionPhase.BeforeCleanup && window == "BeforeCleanup:any") Environment.Exit(79); + }).Apply(plan); + result.Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); + result.Code.Should().Be("config.edit_pending_recovery"); + } + + private sealed class ParentExitHost : IConfigurationTransactionHost + { + private readonly NativeConfigurationTransactionHost native = new(); + public bool SupportsDurableRecovery => true; + public ConfigurationObservation Observe(string path, IReadOnlyCollection? excludedPaths = null) => native.Observe(path, excludedPaths); + public byte[] ReadFile(string path) => native.ReadFile(path); + public IDisposable Acquire(IReadOnlyList paths) => native.Acquire(paths); + public IReadOnlyList CreateParentDirectories(string destination) + { + native.CreateParentDirectories(destination); + Environment.Exit(79); + return []; + } + public string Stage(string destination, ReadOnlyMemory content, string transactionId, string role) => native.Stage(destination, content, transactionId, role); + public void Install(string stagedPath, string destination, bool overwrite) => native.Install(stagedPath, destination, overwrite); + public void DeleteFile(string path) => native.DeleteFile(path); + public void DeleteDirectory(string path) => native.DeleteDirectory(path); + public IReadOnlyList FindRecoveryJournals(IReadOnlyList paths) => native.FindRecoveryJournals(paths); + public IReadOnlyList? EnumerateTransactionArtifacts(string directory, string transactionId) => native.EnumerateTransactionArtifacts(directory, transactionId); + } + + private static async Task<(int ExitCode, string Output)> Child(string root, string window) + { + var start = new ProcessStartInfo("dotnet") { RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false }; + start.ArgumentList.Add(typeof(ConfigurationRecoveryProcessTests).Assembly.Location); + start.ArgumentList.Add("--maximum-parallel-tests"); start.ArgumentList.Add("1"); + start.ArgumentList.Add("--treenode-filter"); start.ArgumentList.Add("/*/*/ConfigurationRecoveryProcessTests/RecoveryChildProcess"); + start.ArgumentList.Add("--results-directory"); start.ArgumentList.Add(root + "/child-results-" + Guid.NewGuid().ToString("N")); + start.Environment["MORPHIR_RECOVERY_CHILD_ROOT"] = root; + start.Environment["MORPHIR_RECOVERY_CHILD_WINDOW"] = window; + using var process = Process.Start(start)!; + var output = process.StandardOutput.ReadToEndAsync(); var errors = process.StandardError.ReadToEndAsync(); + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + try { await process.WaitForExitAsync(timeout.Token); } + catch { process.Kill(true); throw; } + return (process.ExitCode, await output + await errors); + } +} diff --git a/tests/Morphir.Configuration.IO.Tests/ConfigurationRecoveryTests.cs b/tests/Morphir.Configuration.IO.Tests/ConfigurationRecoveryTests.cs new file mode 100644 index 00000000..dd859148 --- /dev/null +++ b/tests/Morphir.Configuration.IO.Tests/ConfigurationRecoveryTests.cs @@ -0,0 +1,237 @@ +using System.Text; +using FluentAssertions; +using Morphir.Configuration.IO; + +namespace Morphir.Configuration.IO.Tests; + +public sealed class ConfigurationRecoveryTests +{ + [Test] + public void PendingRecoveryBlocksAnotherWriterUntilExplicitRecoveryRestoresOriginals() + { + var root = ProjectPath.join(Path.GetTempPath(), "batch-recovery-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + var resolved = false; + try + { + var first = root + "/one/morphir.toml"; + var second = root + "/two/morphir.toml"; + Directory.CreateDirectory(ProjectPath.parent(first)); + Directory.CreateDirectory(ProjectPath.parent(second)); + File.WriteAllText(first, "original=1\n"); + File.WriteAllText(second, "original=2\n"); + var plan = Plan(root, first, second); + var failed = new ConfigurationTransaction(fault: (phase, _) => + { + if (phase is ConfigurationTransactionPhase.AfterInstall or ConfigurationTransactionPhase.BeforeRollback) + throw new IOException("injected interruption and rollback failure"); + }).Apply(plan); + failed.Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); + failed.UnresolvedPaths.Should().Contain(first); + var next = new ConfigurationTransaction().Apply(Plan(root, first, second)); + next.Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); + var journal = failed.Artifacts.First(a => a.Kind == ConfigurationArtifactKind.Journal).Path; + var inspection = new ConfigurationTransaction().InspectRecovery(journal); + inspection.Valid.Should().BeTrue(); + inspection.Targets.Should().BeEquivalentTo([first, second]); + new ConfigurationTransaction().Recover(journal).Status.Should().Be(ConfigurationTransactionStatus.RolledBack); + File.ReadAllText(first).Should().Be("original=1\n"); + File.ReadAllText(second).Should().Be("original=2\n"); + new ConfigurationTransaction().InspectPending([root, first]).Should().BeEmpty(); + resolved = true; + } + finally + { + if (resolved) Directory.Delete(root, true); + else Console.Error.WriteLine("Retained recovery evidence: " + root); + } + } + + [Test] + public void RecoveryPreservesUserDriftAndRetainsItsJournal() + { + var root = ProjectPath.join(Path.GetTempPath(), "batch-recovery-drift-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + var resolved = false; + try + { + var target = root + "/morphir.toml"; + File.WriteAllText(target, "original=1\n"); + var result = new ConfigurationTransaction(fault: (phase, _) => + { + if (phase is ConfigurationTransactionPhase.AfterInstall or ConfigurationTransactionPhase.BeforeRollback) throw new IOException("injected"); + }).Apply(Plan(root, target)); + var journal = result.Artifacts.First(a => a.Kind == ConfigurationArtifactKind.Journal).Path; + File.WriteAllText(target, "user=true\n"); + var recovered = new ConfigurationTransaction().Recover(journal); + recovered.Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); + recovered.UnresolvedPaths.Should().Contain(target); + File.ReadAllText(target).Should().Be("user=true\n"); + File.Exists(journal).Should().BeTrue(); + // The fixture explicitly returns the target to the recorded candidate, then recovers. + File.WriteAllText(target, "candidate=true\n"); + new ConfigurationTransaction().Recover(journal).Status.Should().Be(ConfigurationTransactionStatus.RolledBack); + resolved = true; + } + finally + { + if (resolved) Directory.Delete(root, true); + else Console.Error.WriteLine("Retained recovery drift evidence: " + root); + } + } + + private static ConfigurationTransactionPlan Plan(string root, params string[] targets) => new( + new[] { ConfigurationTransaction.Observe(root) }, + targets.Select(path => new ConfigurationFileChange(path, ConfigurationTransaction.Observe(path), Encoding.UTF8.GetBytes("candidate=true\n")))); +} + +public sealed class ConfigurationRecoveryHostCapabilityTests +{ + [Test] + public void LegacyStorageHostCannotSilentlyWriteWithoutDurableRecoveryEnumeration() + { + var root = ProjectPath.join(Path.GetTempPath(), "recovery-host-capability-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + try + { + var target = root + "/morphir.toml"; + var host = new LegacyHost(); + var result = new ConfigurationTransaction(host).Apply(new ConfigurationTransactionPlan([host.Observe(root)], + [new ConfigurationFileChange(target, host.Observe(target), Encoding.UTF8.GetBytes("value=true\n"))])); + result.Status.Should().Be(ConfigurationTransactionStatus.IoFailure); + result.Code.Should().Be("config.recovery_host_capability"); + File.Exists(target).Should().BeFalse(); + Directory.GetFiles(root, ".morphir-edit-*").Should().BeEmpty(); + } + finally { Directory.Delete(root, true); } + } + + private sealed class LegacyHost : IConfigurationTransactionHost + { + private readonly NativeConfigurationTransactionHost native = new(); + public ConfigurationObservation Observe(string path, IReadOnlyCollection? excludedPaths = null) => native.Observe(path, excludedPaths); + public byte[] ReadFile(string path) => native.ReadFile(path); + public IDisposable Acquire(IReadOnlyList paths) => native.Acquire(paths); + public IReadOnlyList CreateParentDirectories(string destination) => native.CreateParentDirectories(destination); + public string Stage(string destination, ReadOnlyMemory content, string transactionId, string role) => native.Stage(destination, content, transactionId, role); + public void Install(string stagedPath, string destination, bool overwrite) => native.Install(stagedPath, destination, overwrite); + public void DeleteFile(string path) => native.DeleteFile(path); + public void DeleteDirectory(string path) => native.DeleteDirectory(path); + } +} + +public sealed class ConfigurationRecoveryEvidenceTests +{ + [Test] + [Arguments("{\"version\":2,\"transactionId\":null}")] + [Arguments("{\"version\":999999999999999999999999999}")] + [Arguments("{\"version\":2,\"transactionId\":\"11111111111111111111111111111111\",\"guards\":[null],\"entries\":[],\"artifacts\":[],\"committed\":false}")] + [Arguments("{")] + public void MalformedJournalShapesReturnStructuredRecoveryRequiredAndRemainUntouched(string content) + { + var root = ProjectPath.join(Path.GetTempPath(), "recovery-invalid-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + try + { + var journal = root + "/.morphir-edit-11111111111111111111111111111111-1111111111111111-journal"; + File.WriteAllText(journal, content); + var transaction = new ConfigurationTransaction(); + var inspection = transaction.InspectRecovery(journal); + inspection.Valid.Should().BeFalse(); + inspection.Code.Should().Be("config.recovery_invalid_journal"); + transaction.Recover(journal).Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); + var target = root + "/morphir.toml"; + transaction.Apply(new ConfigurationTransactionPlan([ConfigurationTransaction.Observe(root)], + [new ConfigurationFileChange(target, ConfigurationTransaction.Observe(target), Encoding.UTF8.GetBytes("new=true\n"))])) + .Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); + File.ReadAllText(journal).Should().Be(content); + File.Exists(target).Should().BeFalse(); + // This fixture created the invalid text; no real transaction ownership is inferred. + File.Delete(journal); + } + finally { Directory.Delete(root, true); } + } + + [Test] + public void JournalCannotReassignAnArtifactsTargetByEditingItsOwnedPrefix() + { + var root = ProjectPath.join(Path.GetTempPath(), "recovery-artifact-owner-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + var resolved = false; + try + { + var target = root + "/morphir.toml"; + File.WriteAllText(target, "original=true\n"); + var failed = new ConfigurationTransaction(fault: (phase, _) => + { + if (phase is ConfigurationTransactionPhase.AfterInstall or ConfigurationTransactionPhase.BeforeRollback) throw new IOException("injected"); + }).Apply(new ConfigurationTransactionPlan([ConfigurationTransaction.Observe(root)], + [new ConfigurationFileChange(target, ConfigurationTransaction.Observe(target), Encoding.UTF8.GetBytes("candidate=true\n"))])); + var journals = failed.Artifacts.Where(a => a.Kind == ConfigurationArtifactKind.Journal) + .ToDictionary(a => a.Path, a => File.ReadAllBytes(a.Path)); + try + { + foreach (var (path, bytes) in journals) + { + var json = System.Text.Json.Nodes.JsonNode.Parse(bytes)!; + foreach (var entry in json["entries"]!.AsArray()) + entry!["stage"] = root + "/.morphir-edit-" + failed.Artifacts[0].TransactionId + "-forged-stage"; + File.WriteAllText(path, json.ToJsonString()); + } + new ConfigurationTransaction().InspectRecovery(journals.Keys.First()).Valid.Should().BeFalse(); + new ConfigurationTransaction().Recover(journals.Keys.First()).Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); + File.ReadAllText(target).Should().Be("candidate=true\n"); + } + finally + { + foreach (var (path, bytes) in journals) File.WriteAllBytes(path, bytes); + var result = new ConfigurationTransaction().Recover(journals.Keys.First()); + result.Status.Should().Be(ConfigurationTransactionStatus.RolledBack); + resolved = true; + } + } + finally + { + if (resolved) Directory.Delete(root, true); + else Console.Error.WriteLine("Retained artifact ownership evidence: " + root); + } + } + + [Test] + public void ChangedBackupIsPreservedUntilItsExactRecordedBytesAreExplicitlyRestored() + { + var root = ProjectPath.join(Path.GetTempPath(), "recovery-backup-drift-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + var resolved = false; + try + { + var target = root + "/morphir.toml"; + File.WriteAllText(target, "original=true\n"); + var result = new ConfigurationTransaction(fault: (phase, _) => + { + if (phase is ConfigurationTransactionPhase.AfterInstall or ConfigurationTransactionPhase.BeforeRollback) throw new IOException("injected"); + }).Apply(new ConfigurationTransactionPlan([ConfigurationTransaction.Observe(root)], + [new ConfigurationFileChange(target, ConfigurationTransaction.Observe(target), Encoding.UTF8.GetBytes("candidate=true\n"))])); + var backup = result.Artifacts.Single(a => a.Kind == ConfigurationArtifactKind.Backup).Path; + var journal = result.Artifacts.First(a => a.Kind == ConfigurationArtifactKind.Journal).Path; + var original = File.ReadAllBytes(backup); + File.WriteAllText(backup, "user-backup=true\n"); + var transaction = new ConfigurationTransaction(); + transaction.InspectRecovery(journal).UnresolvedPaths.Should().Contain(backup); + var recovered = transaction.Recover(journal); + recovered.Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); + File.ReadAllText(backup).Should().Be("user-backup=true\n"); + File.ReadAllText(target).Should().Be("candidate=true\n"); + File.Exists(journal).Should().BeTrue(); + File.WriteAllBytes(backup, original); + transaction.Recover(journal).Status.Should().Be(ConfigurationTransactionStatus.RolledBack); + File.ReadAllText(target).Should().Be("original=true\n"); + resolved = true; + } + finally + { + if (resolved) Directory.Delete(root, true); + else Console.Error.WriteLine("Retained backup drift evidence: " + root); + } + } +} diff --git a/tests/Morphir.Configuration.IO.Tests/ConfigurationTransactionTests.cs b/tests/Morphir.Configuration.IO.Tests/ConfigurationTransactionTests.cs index 95b1dd9e..3358a665 100644 --- a/tests/Morphir.Configuration.IO.Tests/ConfigurationTransactionTests.cs +++ b/tests/Morphir.Configuration.IO.Tests/ConfigurationTransactionTests.cs @@ -210,13 +210,15 @@ public void IncompleteRollbackRetainsTypedOwnedBackupAndRecoveryJournal() File.ReadAllText(target).Should().Be("value = 2\n"); var backup = result.Artifacts.Single(a => a.Kind == ConfigurationArtifactKind.Backup); File.ReadAllText(backup.Path).Should().Be("value = 1\n"); - var journal = result.Artifacts.Single(a => a.Kind == ConfigurationArtifactKind.Journal); + var journal = result.Artifacts.First(a => a.Kind == ConfigurationArtifactKind.Journal); journal.TransactionId.Should().Be(backup.TransactionId); - using var metadata = System.Text.Json.JsonDocument.Parse(File.ReadAllText(journal.Path)); - metadata.RootElement.GetProperty("transactionId").GetString().Should().Be(journal.TransactionId); - metadata.RootElement.GetProperty("target").GetString().Should().Be(target); - metadata.RootElement.GetProperty("backup").GetString().Should().Be(backup.Path); - metadata.RootElement.GetProperty("originalIdentity").GetString().Should().Be(observed.Identity); + var inspection = new ConfigurationTransaction().InspectRecovery(journal.Path); + inspection.Valid.Should().BeTrue(); + inspection.TransactionId.Should().Be(backup.TransactionId); + inspection.Targets.Should().Equal(target); + inspection.Artifacts.Should().Contain(backup); + new ConfigurationTransaction().Recover(journal.Path).Status.Should().Be(ConfigurationTransactionStatus.RolledBack); + File.ReadAllText(target).Should().Be("value = 1\n"); } [Test] @@ -471,10 +473,14 @@ public void PartialParentCreationReportsOwnedDirectories(bool blockCleanup) File.Exists(target).Should().BeFalse(); if (blockCleanup) { - var artifact = result.Artifacts.Single(); + var artifact = result.Artifacts.Single(a => a.Kind == ConfigurationArtifactKind.Directory); + result.Artifacts.Should().Contain(a => a.Kind == ConfigurationArtifactKind.Journal); artifact.Kind.Should().Be(ConfigurationArtifactKind.Directory); artifact.Path.Should().Be(parent); artifact.TransactionId.Should().Be(host.Owner); + var journal = result.Artifacts.First(a => a.Kind == ConfigurationArtifactKind.Journal).Path; + new ConfigurationTransaction().Recover(journal, [parent]).Status.Should().Be(ConfigurationTransactionStatus.RolledBack); + Directory.Exists(parent).Should().BeTrue("explicitly reviewed retention never deletes the directory"); } else Directory.Exists(parent).Should().BeFalse(); } @@ -482,6 +488,9 @@ public void PartialParentCreationReportsOwnedDirectories(bool blockCleanup) private sealed class ParentFaultHost(bool blockCleanup) : IConfigurationTransactionHost { private readonly NativeConfigurationTransactionHost native = new(); + public bool SupportsDurableRecovery => native.SupportsDurableRecovery; + public IReadOnlyList FindRecoveryJournals(IReadOnlyList paths) => native.FindRecoveryJournals(paths); + public IReadOnlyList? EnumerateTransactionArtifacts(string directory, string transactionId) => native.EnumerateTransactionArtifacts(directory, transactionId); public string? Owner { get; private set; } public ConfigurationObservation Observe(string path, IReadOnlyCollection? excludedPaths = null) => native.Observe(path, excludedPaths); public byte[] ReadFile(string path) => native.ReadFile(path); @@ -574,11 +583,36 @@ public void AFileAtTheParentLocationReturnsIoFailureWithoutChangingIt() result.Artifacts.Should().BeEmpty(); } + [Test] + public void MountedHostRecoversFromSerializedEvidenceInANewTransactionInstance() + { + var host = new MemoryHost(); + const string target = "/memory/morphir.toml"; + host.Files[target] = Encoding.UTF8.GetBytes("original=true\n"); + var plan = new ConfigurationTransactionPlan([host.Observe("/memory")], + [new ConfigurationFileChange(target, host.Observe(target), Encoding.UTF8.GetBytes("candidate=true\n"))]); + var interrupted = () => new ConfigurationTransaction(host, (phase, _) => + { + if (phase == ConfigurationTransactionPhase.AfterInstall) throw new InvalidOperationException("simulated host interruption"); + }).Apply(plan); + interrupted.Should().Throw(); + var restarted = new ConfigurationTransaction(host); + var pending = restarted.InspectPending(["/memory"]); + pending.Should().NotBeEmpty(); + pending.Should().OnlyContain(p => p.Valid); + restarted.Apply(plan).Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); + restarted.Recover(pending[0].JournalPath).Status.Should().Be(ConfigurationTransactionStatus.RolledBack); + Encoding.UTF8.GetString(host.Files[target]).Should().Be("original=true\n"); + host.Files.Keys.Should().Equal(target); + } + private sealed class MemoryHost : IConfigurationTransactionHost { public Dictionary Files { get; } = new(StringComparer.Ordinal); + public bool SupportsDurableRecovery => true; + public IReadOnlyList FindRecoveryJournals(IReadOnlyList paths) => Files.Keys.Where(p => Path.GetFileName(p).Contains("-journal", StringComparison.Ordinal)).ToArray(); + public IReadOnlyList? EnumerateTransactionArtifacts(string directory, string transactionId) => Files.Keys.Where(p => ProjectPath.parent(p) == directory && Path.GetFileName(p).StartsWith(".morphir-edit-" + transactionId + "-", StringComparison.Ordinal)).ToArray(); private readonly object coordination = new(); - private int sequence; private static string Hash(byte[] value) => Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(value)); public ConfigurationObservation Observe(string path, IReadOnlyCollection? excludedPaths = null) { @@ -596,7 +630,7 @@ public IDisposable Acquire(IReadOnlyList paths) public IReadOnlyList CreateParentDirectories(string destination) => []; public string Stage(string destination, ReadOnlyMemory content, string transactionId, string role) { - var path = "/memory/.stage-" + transactionId + "-" + role + "-" + sequence++; + var path = ConfigurationTransaction.GetArtifactPath(destination, transactionId, role); Files[path] = content.ToArray(); return path; } @@ -614,6 +648,9 @@ private sealed class MemoryLease(object coordination) : IDisposable { public voi private sealed class PartialStageHost(bool blockCleanup, bool foreignOwner = false) : IConfigurationTransactionHost { private readonly NativeConfigurationTransactionHost native = new(); + public bool SupportsDurableRecovery => native.SupportsDurableRecovery; + public IReadOnlyList FindRecoveryJournals(IReadOnlyList paths) => native.FindRecoveryJournals(paths); + public IReadOnlyList? EnumerateTransactionArtifacts(string directory, string transactionId) => native.EnumerateTransactionArtifacts(directory, transactionId); public string? Owner { get; private set; } public ConfigurationObservation Observe(string path, IReadOnlyCollection? excludedPaths = null) => native.Observe(path, excludedPaths); public byte[] ReadFile(string path) => native.ReadFile(path); diff --git a/tests/Morphir.Configuration.IO.Tests/ProjectMigrationTests.cs b/tests/Morphir.Configuration.IO.Tests/ProjectMigrationTests.cs index d2e351c4..3bcf8ec5 100644 --- a/tests/Morphir.Configuration.IO.Tests/ProjectMigrationTests.cs +++ b/tests/Morphir.Configuration.IO.Tests/ProjectMigrationTests.cs @@ -276,6 +276,9 @@ public void WriteFaultsReportRollbackOrExplicitOwnedRecovery(bool refuseRollback private sealed class FaultHost(bool refuseRollback) : IConfigurationTransactionHost { private readonly NativeConfigurationTransactionHost inner = new(); + public bool SupportsDurableRecovery => inner.SupportsDurableRecovery; + public IReadOnlyList FindRecoveryJournals(IReadOnlyList paths) => inner.FindRecoveryJournals(paths); + public IReadOnlyList? EnumerateTransactionArtifacts(string directory, string transactionId) => inner.EnumerateTransactionArtifacts(directory, transactionId); private int installs; public ConfigurationObservation Observe(string path, IReadOnlyCollection? excludedPaths = null) => inner.Observe(path, excludedPaths); public byte[] ReadFile(string path) => inner.ReadFile(path); @@ -376,6 +379,9 @@ public ProjectEntry Observe(string path) private sealed class EvidenceHost(bool unstable) : IConfigurationTransactionHost { private readonly NativeConfigurationTransactionHost inner = new(); + public bool SupportsDurableRecovery => inner.SupportsDurableRecovery; + public IReadOnlyList FindRecoveryJournals(IReadOnlyList paths) => inner.FindRecoveryJournals(paths); + public IReadOnlyList? EnumerateTransactionArtifacts(string directory, string transactionId) => inner.EnumerateTransactionArtifacts(directory, transactionId); private int observations; public ConfigurationObservation Observe(string path, IReadOnlyCollection? excludedPaths = null) { From 2f66cebe70b4266e3f96707dd87326d3dd2aaafa Mon Sep 17 00:00:00 2001 From: Damian Reeves <957246+DamianReeves@users.noreply.github.com> Date: Tue, 3 Nov 2026 14:21:07 -0600 Subject: [PATCH 03/14] feat: plan bounded workspace adoption Reuse project loss and equivalence rules in one proposed workspace. Verify membership, selection and non-adoption values before aggregate writes. Refs #479. --- README.md | 4 +- .../docs/configuration/workspace-migration.md | 102 ++++ .../prds/workspace-migration-delivery.md | 22 + .../changes/add-workspace-migration/tasks.md | 2 +- .../ProjectMigrationService.cs | 22 +- .../WorkspaceMigrationService.cs | 457 ++++++++++++++++++ src/Morphir.Configuration/Migration.fs | 4 + src/Morphir.Configuration/ProjectDiscovery.fs | 4 + .../WorkspaceMigration.feature | 18 + .../WorkspaceMigrationSteps.cs | 28 ++ .../WorkspaceMigrationTests.cs | 347 +++++++++++++ 11 files changed, 1007 insertions(+), 3 deletions(-) create mode 100644 docs/content/docs/configuration/workspace-migration.md create mode 100644 src/Morphir.Configuration.IO/WorkspaceMigrationService.cs create mode 100644 tests/Morphir.Configuration.IO.Tests/WorkspaceMigration.feature create mode 100644 tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationSteps.cs create mode 100644 tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationTests.cs diff --git a/README.md b/README.md index 7c7417e9..33e1e8e3 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,9 @@ and `morphir config set/unset` provide typed, scoped TOML previews and writes, source-preserving changes, optimistic conflicts and explicit recovery ownership. Shared [classic project migration](docs/content/docs/configuration/project-migration.md) and `morphir project migrate` provide loss-checked previews and guarded TOML -adoption while retaining classic bytes. Workspace migration remains planned. See the +adoption while retaining classic bytes. Shared [workspace migration](docs/content/docs/configuration/workspace-migration.md) +adds bounded inventory, verified membership and recoverable batch application through +`morphir workspace migrate [directory]`. See the [reader profile](src/Morphir.Configuration.IO/README.md) for syntax and provenance. ## SDK libraries diff --git a/docs/content/docs/configuration/workspace-migration.md b/docs/content/docs/configuration/workspace-migration.md new file mode 100644 index 00000000..f27d12ef --- /dev/null +++ b/docs/content/docs/configuration/workspace-migration.md @@ -0,0 +1,102 @@ +--- +title: "Workspace migration API and inventory" +linkTitle: "Workspace migration" +weight: 65 +--- + +# Adopt a bounded repository + +`Morphir.Configuration.IO.WorkspaceMigrationService` inventories, plans and applies +workspace adoption without constructing the CLI. It reuses `ProjectMigration`'s +mapping, raw loss ledger and independent equivalence verification. It does not +compile sources, move generated outputs, remove classic manifests or invoke +publication services. + +```csharp +var root = ProjectPath.normalize(Path.GetFullPath(directory)); +var service = new WorkspaceMigrationService( + new FileSystemProjectTree(root), new NativeConfigurationTransactionHost()); +var preview = service.Plan(root, includes: ["apps/**"], excludes: ["**/experimental"]); +if (preview.CanApply) +{ + var outcome = service.Apply(preview); + // Inspect outcome.Status, Artifacts, RestoredPaths and UnresolvedPaths. +} +``` + +Omit storage for a read-only host. Supply `InMemoryProjectTree` or implement +`IProjectTree` for a mounted tree. Both use canonical absolute paths with `/` +separators. A mounted directory returns only immediate child directories or +links. A child outside that directory or the selected operation root blocks the +plan. Application requires storage-backed evidence and the service that created +the verified preview. Read-only and foreign-service plans cannot be applied. + +## Inventory rules + +The absolute directory passed to `Plan` is the operation boundary even if the +host's authorization boundary is larger. No ancestor is searched for projects +or workspace adoption. Inventory orders paths ordinally. It inspects shared +primary layouts and formats, including supplemental workspace JSON, without +reading personal, machine, environment or active profile overlays. + +Include and exclude filters are repeated root-relative segment globs. `*` and +`?` match within one segment; `**` matches zero or more segments. Absolute paths, +empty filters and `..` segments are invalid. Excluded directories are pruned. +Existing `workspace.exclude` patterns retain their declared workspace-base +anchor and cannot be overridden by an include filter. + +Version control directories `.git`, `.hg`, `.svn`, dependency directories +`node_modules`, `elm-stuff`, `vendor`, `.nuget`, `packages`, and output directories +`bin`, `obj`, `dist`, `build`, `target` are skipped. Primary layout directories +`.morphir` and `.config` are inspected as layouts at their owner; they are not +separate projects. The declared workspace output directory is also skipped. Discovered links are reported and never followed, including +linked ancestors. A nested declared workspace is reported as a boundary and +its children are not inventoried for outer adoption. The maximum inventory is +100,000 directories; exceeding it blocks the plan instead of returning an +incomplete applicable result. + +`Inventory` explains candidate, excluded, boundary, already migrated and skipped +link paths. Duplicate names remain distinct paths and produce a diagnostic; +explicit path selection remains unambiguous. + +## Reviewed membership and verification + +`Members` preserves existing declarations, including globs, order and defaults, +and appends explicit root-relative paths for additions. A root project remains a +root project and never becomes its own glob member. An otherwise projectless +root becomes virtual. Already migrated members retain their bytes and settings. +An existing valid workspace with no conversions succeeds unchanged. An +undeclared empty root reports that adoption is impossible. + +`Projects` contains the single-project plans and their loss ledgers. Use each +project's `Presentation` for logs. The canonical `Plan.Candidate` and `Plan.Edits` +contain application values, including secrets; do not log them. Secret references +remain opaque leaves. Unsupported values cannot be dropped to make a batch pass. + +Every conversion receives explicit proposed base workspace context. Verification +disables activation and compares declared profile controls separately. The +aggregate candidate excludes every converted classic source together and reruns +shared discovery and distinct-path selection. It must preserve project source +paths and non-adoption settings. Intended ownership, membership, default selection +and shared output-root changes appear separately in `AdoptionEffects`. They do +not authorize other value changes. Existing generated files stay where they are. + +## Application and recovery + +Application validates source bytes, target bytes or absence, relevant directory +listings and primary topology. It stages the entire batch before replacing any +destination. Cooperating writers coordinate through the shared transaction +adapter. Each file installation is atomic; the batch is not crash-atomic. + +Storage-backed previews inspect pending journals without writing and expose +`PendingRecovery`; a pending operation makes the preview inapplicable. On failure +inspect the structured status, restored paths, unresolved paths and +owned artifacts. Pending recovery blocks overlapping shared configuration edits +and project migrations as well as subsequent workspace writes. Use the explicit +shared journal inspection and recovery APIs described in +[transaction recovery](transaction-recovery.md). Do not remove recovery journals +or backups to bypass a guard. Preserve user changes and unresolved evidence. + +CLI reports use the same plans and outcomes. See the +[workspace command reference](../cli/workspace-migrate.md), walkthrough and +troubleshooting for exit codes, previews and scriptable JSON. diff --git a/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md b/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md index 72658c16..b5790443 100644 --- a/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md +++ b/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md @@ -80,3 +80,25 @@ Setup completed before runtime changes. Proposal now links actual issue #479. Ta found in the helper's final inventory. - Native coordination is machine-local, and file replacement is per-file atomic. No cross-file crash-atomicity or distributed-lock guarantee is claimed. + +### Common-context verification and source policy + +`transactions/lead-declared-controls-red.log` passes one and fails one declared +control test. `transactions/lead-layout-red.log` fails both mixed-layout cases. +The aggregate now compares declared controls with activation disabled through the +same shared control policy as single-project verification. It supplies an explicit +existing supported TOML target to the shared planner, preserving owner anchors +and all existing target/ambiguity guards. Both root and member layouts pass. + +`source-policy-red-run.log` reproduces a normalized legacy JSON5 primary being +misreported as already migrated. Inventory now sends every normalized legacy +candidate to the project planner, which retains its established `.json` classic +source contract and returns `migration.classic_required` for `.json5` primaries. +Full JSON5 syntax in supported `morphir.json` remains accepted; no reader policy +is changed or loosened. Canonical JSON5 projects remain valid canonical projects. + +The first final planner build tried to call an internal F# helper from C# and +failed compilation. Its subsequently invoked stale-DLL tests are not a current +qualification gate. `planner-final-build-02.log` is the successful rebuilt source, +followed by `planner-final-io-02.log`, 371/371 current tests. Task 2.3 was marked +only after that qualification, at the refreshed source location. diff --git a/openspec/changes/add-workspace-migration/tasks.md b/openspec/changes/add-workspace-migration/tasks.md index 078cb6c3..51394135 100644 --- a/openspec/changes/add-workspace-migration/tasks.md +++ b/openspec/changes/add-workspace-migration/tasks.md @@ -11,7 +11,7 @@ Read [the design](design.html) and change-local specs before starting. Existing - [x] 2.1 Write failing BDD/API cases for roots with no workspace, include/exclude patterns, default skipped directories, nested workspace boundaries, discovered links, existing migrated members, virtual/root projects, and stable ordering; confirm red before inventory implementation. - [x] 2.2 Implement bounded shared inventory and aggregate plans by composing the project migrator, preserving existing membership/defaults/settings and using explicit member paths; verify the inventory/planning suite and document the adoption command's scan rules. -- [ ] 2.3 Add failing then passing verification of project conversion in a common proposed workspace context, with separately reported ownership/membership/selection/output-root effects; verify source paths/non-adoption settings stay equivalent and document that generated files are not moved. +- [x] 2.3 Add failing then passing verification of project conversion in a common proposed workspace context, with separately reported ownership/membership/selection/output-root effects; verify source paths/non-adoption settings stay equivalent and document that generated files are not moved. ## 3. Coordinated application diff --git a/src/Morphir.Configuration.IO/ProjectMigrationService.cs b/src/Morphir.Configuration.IO/ProjectMigrationService.cs index c75a06b3..919ce19b 100644 --- a/src/Morphir.Configuration.IO/ProjectMigrationService.cs +++ b/src/Morphir.Configuration.IO/ProjectMigrationService.cs @@ -35,11 +35,26 @@ internal ProjectMigrationPreview(object owner, string? source, string? target, s } } +/// Explicit proposed base workspace shared by aggregate conversion hosts. +/// Contains only declared base primaries; no ambient overlays are loaded. +public sealed class ProjectMigrationContext(string rootPath, FSharpOption? workspace, IEnumerable rootSources) +{ + public string RootPath { get; } = rootPath; + public FSharpOption? Workspace { get; } = workspace; + public ImmutableArray RootSources { get; } = rootSources.ToImmutableArray(); +} + /// Plans one classic project over an explicitly bounded tree. A storage adapter is optional /// for read-only hosts; only an adapter-backed verified plan can be applied by its creating service. public sealed class ProjectMigrationService(IProjectTree tree, IConfigurationTransactionHost? storage = null) { public ProjectMigrationPreview Plan(string startPath, string? inputPath = null, string? outputPath = null) + => PlanCore(startPath, inputPath, outputPath, null); + + public ProjectMigrationPreview PlanInWorkspace(string startPath, ProjectMigrationContext context, string? inputPath = null, string? outputPath = null) + => PlanCore(startPath, inputPath, outputPath, context); + + private ProjectMigrationPreview PlanCore(string startPath, string? inputPath, string? outputPath, ProjectMigrationContext? context) { string? sourcePath = null, targetPath = null, projectPath = null; var diagnostics = new List(); @@ -95,7 +110,12 @@ public ProjectMigrationPreview Plan(string startPath, string? inputPath = null, var role = project.Path == found.Discovery.Location ? SourceRole.RootPrimary : SourceRole.MemberPrimary; // Only declared base primary documents participate. No loader call can read personal, // machine or activated-profile sources during migration. - var sources = found.Discovery.Root.Sources.ToList(); + if (context is not null && (context.RootPath != found.Discovery.Location + || !Equals(context.Workspace, found.Discovery.Workspace) + || context.RootSources.Any(s => s.Source.Role != SourceRole.RootPrimary + || !DiscoveryPolicy.candidates(context.RootPath).Contains(s.Source.Path)))) + return Fail("migration.context", "Proposed context must belong to the discovered workspace base primaries."); + var sources = context is null ? found.Discovery.Root.Sources.ToList() : context.RootSources.ToList(); if (role == SourceRole.MemberPrimary) sources.AddRange(project.Location.Sources.Select(s => Rebind(s, role))); var classics = project.Location.Sources.Where(s => s.Source.Path.EndsWith(".json", StringComparison.OrdinalIgnoreCase) diff --git a/src/Morphir.Configuration.IO/WorkspaceMigrationService.cs b/src/Morphir.Configuration.IO/WorkspaceMigrationService.cs new file mode 100644 index 00000000..6d33c319 --- /dev/null +++ b/src/Morphir.Configuration.IO/WorkspaceMigrationService.cs @@ -0,0 +1,457 @@ +// Copyright 2026 FINOS +// SPDX-License-Identifier: Apache-2.0 +using System.Collections.Immutable; +using System.Text; +using System.Text.RegularExpressions; +using Microsoft.FSharp.Collections; +using Microsoft.FSharp.Core; + +namespace Morphir.Configuration.IO; + +public enum WorkspaceInventoryKind { Candidate, Excluded, Boundary, AlreadyMigrated, LinkSkipped } +public sealed record WorkspaceInventoryEntry(string Path, string RelativePath, WorkspaceInventoryKind Kind, string Reason, string? Name); +public sealed record WorkspaceAdoptionEffect(string ProjectPath, string Kind, string? Before, string? After); + +/// An immutable aggregate. Project Presentation is safe for logs; project canonical plans +/// and internal transaction buffers are application data and must not be logged. +public sealed class WorkspaceMigrationPreview +{ + public string RootPath { get; } + public bool RootIsProject { get; } + public ImmutableArray Inventory { get; } + public ImmutableArray Projects { get; } + public ImmutableArray Members { get; } + public string? WorkspaceTarget { get; } + public ImmutableArray WorkspaceEdits { get; } + public ImmutableArray AdoptionEffects { get; } + public ImmutableArray Diagnostics { get; } + public ImmutableArray PendingRecovery { get; } + public bool Verified { get; } + public bool Changed { get; } + public bool CanApply => Verified && !Diagnostics.Any(d => d.Severity == DiagnosticSeverity.Error); + internal object Owner { get; } + internal ConfigurationTransactionPlan? Transaction { get; } + internal WorkspaceMigrationPreview(object owner, string root, bool rootIsProject, + IEnumerable inventory, IEnumerable projects, + IEnumerable members, string? target, IEnumerable edits, + IEnumerable effects, IEnumerable diagnostics, + bool verified = false, bool changed = false, ConfigurationTransactionPlan? transaction = null, + IEnumerable? pendingRecovery = null) + { + Owner = owner; RootPath = root; RootIsProject = rootIsProject; + Inventory = inventory.OrderBy(i => i.Path, StringComparer.Ordinal).ToImmutableArray(); + Projects = projects.OrderBy(p => p.ProjectPath, StringComparer.Ordinal).ToImmutableArray(); + Members = members.ToImmutableArray(); WorkspaceTarget = target; WorkspaceEdits = edits.ToImmutableArray(); + AdoptionEffects = effects.ToImmutableArray(); Diagnostics = Inspection.safeDiagnostics(ReaderValues.List(diagnostics)).ToImmutableArray(); + PendingRecovery = (pendingRecovery ?? []).ToImmutableArray(); + Verified = verified; Changed = changed; Transaction = transaction; + } +} + +/// Bounded inventory, adoption planning and verified aggregate application for any host. +/// Does not load ancestors, personal, machine, environment or activated profile overlays. +public sealed class WorkspaceMigrationService(IProjectTree tree, IConfigurationTransactionHost? storage = null) +{ + public WorkspaceMigrationPreview Plan(string root, IEnumerable? includes = null, IEnumerable? excludes = null) + { + var diagnostics = new List(); + var inventory = new List(); + var projects = new List(); + var members = new List(); + var edits = new List(); + var effects = new List(); + var pendingRecovery = new List(); + string? target = null; + bool rootProject = false; + WorkspaceMigrationPreview Result(bool verified = false, bool changed = false, ConfigurationTransactionPlan? transaction = null) => + new(this, root, rootProject, inventory, projects, members, target, edits, effects, diagnostics, verified, changed, transaction, pendingRecovery); + void Error(string path, string code, string message) => diagnostics.Add(Diagnostic(path, code, message)); + try + { + // Validate before normalization so empty/relative invocations cannot acquire ambient scope. + if (string.IsNullOrWhiteSpace(root) || !ProjectPath.isAbsolute(root)) + { Error(root, "workspace.migrate_boundary", "Select an absolute existing directory inside the authorized tree."); return Result(); } + root = ProjectPath.normalize(root); + if (!ProjectPath.contains(tree.Boundary, root)) + { Error(root, "workspace.migrate_boundary", "Selected root is outside the authorized tree."); return Result(); } + var include = (includes ?? []).ToArray(); + var exclude = (excludes ?? []).ToArray(); + if (include.Concat(exclude).Any(p => !ValidPattern(p))) + { Error(root, "workspace.migrate_filter", "Filters must be nonempty root-relative patterns without parent traversal."); return Result(); } + var observed = new ObservedTree(tree, root, storage); + if (observed.Observe(root) is not ProjectEntry.Directory) + { Error(root, "workspace.migrate_root", "Selected root must be a readable directory without linked ancestors."); return Result(); } + var locations = new SortedDictionary(StringComparer.Ordinal); + var rootLocation = Inspect(observed, root); + diagnostics.AddRange(rootLocation.Diagnostics); + rootProject = Has(rootLocation.Values, "project"); + var declared = Has(rootLocation.Values, "workspace"); + var workspaceBase = ProjectPath.join(root, Text(rootLocation.Values, ".", "workspace", "root")); + if (!ProjectPath.contains(root, workspaceBase)) + { Error(root, "workspace.migrate_boundary", "Declared workspace root escapes the selected operation boundary."); return Result(); } + // Existing exclusions are relative to the declared workspace base. Supplied filters are operation-root relative. + var declaredExcludes = Strings(rootLocation.Values, "workspace", "exclude").ToArray(); + var outputDirectory = declared ? ProjectPath.join(workspaceBase, Text(rootLocation.Values, ".morphir/out", "workspace", "out_dir")) : null; + var availableDirectories = new HashSet(StringComparer.Ordinal); + var queue = new Stack(); queue.Push(root); + const int maxDirectories = 100000; + var visited = new HashSet(StringComparer.Ordinal); + while (queue.TryPop(out var path)) + { + if (!visited.Add(path)) continue; + if (visited.Count > maxDirectories) + { Error(root, "workspace.migrate_inventory_limit", "Inventory directory limit exceeded."); break; } + var relative = ProjectPath.relative(root, path); + if (path != root && (Skipped(Path.GetFileName(path)) || outputDirectory is not null && ProjectPath.contains(outputDirectory, path) || Excluded(relative, exclude) + || ProjectPath.contains(workspaceBase, path) && Excluded(ProjectPath.relative(workspaceBase, path), declaredExcludes))) + { inventory.Add(new(path, relative, WorkspaceInventoryKind.Excluded, "Directory excluded by default, supplied filter or declared workspace exclusion.", null)); continue; } + var entry = observed.Observe(path); + if (entry.IsLink) { inventory.Add(new(path, relative, WorkspaceInventoryKind.LinkSkipped, "Discovered link is not followed.", null)); continue; } + if (entry is ProjectEntry.Failure) + { Error(path, "workspace.migrate_inventory", "Unable to inspect a directory."); continue; } + if (entry is not ProjectEntry.Directory directory) continue; + var location = path == root ? rootLocation : Inspect(observed, path); + if (path != root && Has(location.Values, "workspace")) + { inventory.Add(new(path, relative, WorkspaceInventoryKind.Boundary, "Nested declared workspace is independent.", Name(location))); continue; } + availableDirectories.Add(path); + diagnostics.AddRange(path == root ? [] : location.Diagnostics); + if (Has(location.Values, "project")) + { + if (include.Length > 0 && !include.Any(p => Glob(p, relative))) + inventory.Add(new(path, relative, WorkspaceInventoryKind.Excluded, "Project does not match include filters.", Name(location))); + else + { + var classics = Classics(location).ToArray(); + locations[path] = location; + inventory.Add(new(path, relative, classics.Length == 0 ? WorkspaceInventoryKind.AlreadyMigrated : WorkspaceInventoryKind.Candidate, + classics.Length == 0 ? "Canonical project retained without rewriting." : "Classic project selected for verified conversion.", Name(location))); + } + } + foreach (var child in directory.children.Reverse()) + { + if (!ProjectPath.isAbsolute(child) || ProjectPath.parent(child) != path || !ProjectPath.contains(root, child)) + { Error(path, "workspace.migrate_listing", "Host returned a child outside the bounded immediate directory."); continue; } + // Primary layout directories belong to this location, never separate projects. + if (Path.GetFileName(child) is ".morphir" or ".config") continue; + queue.Push(child); + } + } + if (storage is { SupportsDurableRecovery: true }) + { + pendingRecovery.AddRange(new ConfigurationTransaction(storage).InspectPending(observed.Evidence.Keys)); + if (pendingRecovery.Count > 0) + { + Error(root, "workspace.migrate_recovery_required", "Pending recovery blocks this operation. Inspect and explicitly recover its journal before another write."); + return Result(); + } + } + if (diagnostics.Any(d => d.Severity == DiagnosticSeverity.Error)) return Result(); + if (!declared && locations.Count == 0) + { Error(root, "workspace.migrate_no_adoption", "No eligible projects exist within the selected root; no adoption can be planned."); return Result(); } + // Existing declarations are preserved verbatim. Only append explicit paths not already represented. + var originalMembers = Strings(rootLocation.Values, "workspace", "members").ToArray(); + members.AddRange(originalMembers); + var bounded = new InventoryTree(observed, root, availableDirectories); + var existingDiscovery = declared ? ProjectSystem.Discover(bounded, root).Discovery : null; + if (existingDiscovery is not null) + { + diagnostics.AddRange(existingDiscovery.Diagnostics); + // Never absorb a pre-existing declaration outside the operation root or through a nested boundary. + foreach (var old in existingDiscovery.Projects.Where(p => p.Path != root)) + { + if (!ProjectPath.contains(root, old.Path) || inventory.Any(i => i.Kind == WorkspaceInventoryKind.Boundary && ProjectPath.contains(i.Path, old.Path))) + Error(old.Path, "workspace.migrate_membership", "Existing membership crosses the operation or nested-workspace boundary."); + if (!inventory.Any(i => i.Path == old.Path)) + inventory.Add(new(old.Path, ProjectPath.relative(root, old.Path), WorkspaceInventoryKind.AlreadyMigrated, "Existing declared member retained.", old.Name?.Value)); + } + } + var existingPaths = existingDiscovery?.Projects.Select(p => p.Path).ToHashSet(StringComparer.Ordinal) ?? []; + foreach (var path in locations.Keys.Where(p => p != root)) + { + if (!ProjectPath.contains(workspaceBase, path)) + { Error(path, "workspace.migrate_membership", "Candidate lies outside the declared workspace base."); continue; } + if (!existingPaths.Contains(path)) + { + var member = ProjectPath.relative(workspaceBase, path); + if (member.IndexOfAny(['*', '?']) >= 0) + { Error(path, "workspace.migrate_member_path", "A project path containing glob characters cannot be declared as an explicit literal member."); continue; } + if (!members.Contains(member, StringComparer.Ordinal)) members.Add(member); + } + } + var rootTomls = rootLocation.Sources.Where(s => s.Source.Format == DocumentFormat.Toml).ToArray(); + var rootClassic = Classics(rootLocation).SingleOrDefault(); + target = rootTomls.Length == 1 ? rootTomls[0].Source.Path : rootClassic is not null + ? Path.ChangeExtension(rootClassic.Source.Path, ".toml") : ProjectPath.join(root, "morphir.toml"); + if (!declared || !originalMembers.SequenceEqual(members)) + edits.Add(new(ReaderValues.List(new[] { "workspace", "members" }), ArrayValue(members))); + var targetRaw = rootTomls.FirstOrDefault()?.Content is SourceContent.Available current ? current.Item.Raw : DocumentDefaults.emptyValue; + var rootText = BufferFor(observed, target, targetRaw); + var workspaceText = rootText.Text; + foreach (var edit in edits) + { + var rendered = TomlEditor.Edit(workspaceText, edit.KeyPath.ToArray(), edit.Value); + if (!rendered.Success) { Error(target, "workspace.migrate_render", "Workspace edits cannot be represented without loss."); return Result(); } + workspaceText = rendered.Text; + } + var overlay = new CandidateTree(bounded, root); + overlay.Set(target, workspaceText); + // Convert root first so all members resolve against its final proposed canonical base. + if (locations.TryGetValue(root, out var selectedRoot) && Classics(selectedRoot).Any()) + { + var service = new ProjectMigrationService(overlay); + var rootPreview = service.PlanInWorkspace(root, Context(overlay, root, Classics(selectedRoot).Select(s => s.Source.Path)), Classics(selectedRoot).Single().Source.Path, target); + projects.Add(rootPreview); diagnostics.AddRange(rootPreview.Diagnostics); + if (rootPreview.CanApply) + { + workspaceText = Render(workspaceText, rootPreview.Plan!.Edits); + overlay.Set(target, workspaceText); + } + } + var common = Context(overlay, root, projects.Select(p => p.SourcePath!)); + foreach (var (path, location) in locations.Where(p => p.Key != root)) + { + var classics = Classics(location).ToArray(); + if (classics.Length == 0) continue; + if (classics.Length != 1) { Error(path, "workspace.migrate_classic", "Expected one unambiguous classic source."); continue; } + var service = new ProjectMigrationService(overlay); + var existingTarget = location.Sources.FirstOrDefault(s => s.Source.Format == DocumentFormat.Toml)?.Source.Path; + var preview = service.PlanInWorkspace(path, common, classics[0].Source.Path, existingTarget); + projects.Add(preview); diagnostics.AddRange(preview.Diagnostics); + if (preview.CanApply) + { + var raw = location.Sources.FirstOrDefault(s => s.Source.Path == preview.TargetPath)?.Content is SourceContent.Available a ? a.Item.Raw : DocumentDefaults.emptyValue; + var original = BufferFor(observed, preview.TargetPath!, raw); + overlay.Set(preview.TargetPath!, Render(original.Text, preview.Plan!.Edits)); + } + } + foreach (var project in projects.Where(p => p.CanApply && !p.Changed)) + { + var index = inventory.FindIndex(i => i.Path == project.ProjectPath); + if (index >= 0) inventory[index] = inventory[index] with { Kind = WorkspaceInventoryKind.AlreadyMigrated, Reason = "Equivalent migrated project retained without rewriting." }; + } + if (projects.Any(p => !p.CanApply) || diagnostics.Any(d => d.Severity == DiagnosticSeverity.Error)) return Result(); + if (!declared) + { + // Adoption may change topology/output roots, but never unrelated values. + foreach (var (path, location) in locations) + { + var original = BaseResolution(location.Sources, ProjectScope.Root, null); + var adoptedSources = path == root ? common.RootSources.Concat(location.Sources.Where(s => Classics(location).Any(c => c.Source.Path == s.Source.Path))) + : common.RootSources.Concat(location.Sources.Select(s => Rebind(s, SourceRole.MemberPrimary))); + var adopted = BaseResolution(adoptedSources, path == root ? ProjectScope.Root : ProjectScope.Member, common.Workspace); + var beforeValues = ComparableValues(original, location.Sources); + var afterValues = ComparableValues(adopted, adoptedSources); + if (!ProjectMigration.same(beforeValues, afterValues)) + Error(path, "workspace.migrate_adoption_value", "Adoption changes a non-adoption setting; review the project and proposed shared base settings."); + } + } + // Remove every converted classic input together. Rerun actual discovery/selection over the whole candidate. + foreach (var project in projects) overlay.Hide(project.SourcePath!); + var candidate = ProjectSystem.Discover(overlay, root).Discovery; + diagnostics.AddRange(candidate.Diagnostics); + if (candidate.Location != root || candidate.Workspace is null || candidate.Projects.Any(p => !p.IsValid)) + Error(root, "workspace.migrate_verification", "Proposed workspace discovery is invalid."); + var expectedPaths = locations.Keys.Concat(existingPaths).Concat(rootProject ? new[] { root } : []).Where(p => p == root || ProjectPath.contains(workspaceBase, p)).Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal).ToArray(); + if (!candidate.Projects.Select(p => p.Path).Order(StringComparer.Ordinal).SequenceEqual(expectedPaths)) + Error(root, "workspace.migrate_membership", "Proposed discovery differs from the reviewed project paths."); + foreach (var project in candidate.Projects) + { + var selected = DiscoveryPolicy.select(candidate, FSharpOption.Some(project.Path)); + if (selected.Projects.Count() != 1 || selected.Projects.Head.Path != project.Path) + Error(project.Path, "workspace.migrate_selection", "Proposed member cannot be selected by its distinct path."); + } + var beforeSelection = existingDiscovery is null ? null : DiscoveryPolicy.select(existingDiscovery, null); + var afterSelection = DiscoveryPolicy.select(candidate, null); + if (Has(rootLocation.Values, "workspace", "default_member") && afterSelection.Diagnostics.Any(d => d.Severity == DiagnosticSeverity.Error)) + Error(root, "workspace.migrate_default", "The declared default member is not a valid unambiguous project in the proposed workspace."); + var beforeSelected = beforeSelection is null ? "" : string.Join(",", beforeSelection.Projects.Select(p => p.Path)); + var afterSelected = string.Join(",", afterSelection.Projects.Select(p => p.Path)); + if (declared && (beforeSelected != afterSelected || !beforeSelection!.Diagnostics.Select(d => d.Code).SequenceEqual(afterSelection.Diagnostics.Select(d => d.Code)))) + { + // An undeclared default can become ambiguous through new membership; this is an explicit adoption effect. + effects.Add(new(root, "default-selection", beforeSelected, afterSelected)); + } + else if (!declared) effects.Add(new(root, "default-selection", null, afterSelected)); + var output = candidate.Workspace?.Value.OutputDirectory?.Value; + foreach (var path in locations.Keys) + { + var oldOutput = declared ? existingDiscovery!.Workspace?.Value.OutputDirectory?.Value : ProjectPath.join(path, ".morphir/out"); + effects.Add(new(path, "ownership", declared ? existingDiscovery!.Workspace?.Value.Root : path, candidate.Workspace?.Value.Root)); + effects.Add(new(path, "membership", existingPaths.Contains(path) || path == root && declared ? "retained" : "standalone", path == root ? "root-project" : ProjectPath.relative(workspaceBase, path))); + effects.Add(new(path, "shared-output-root", oldOutput, output)); + } + var verified = !diagnostics.Any(d => d.Severity == DiagnosticSeverity.Error); + var changes = new List(); + if (storage is not null && verified) + { + foreach (var (path, text) in overlay.Texts.OrderBy(p => p.Key, StringComparer.Ordinal)) + { + var oldRaw = path == target ? targetRaw : DocumentDefaults.emptyValue; + var buffer = BufferFor(observed, path, oldRaw); + var bytes = buffer.Encode(text); + var expected = observed.Evidence[path]; + if (expected.Kind == ConfigurationObservationKind.File && storage.ReadFile(path).AsSpan().SequenceEqual(bytes)) continue; + changes.Add(new(path, expected, bytes)); + } + } + var changed = edits.Count > 0 || projects.Any(p => p.Changed); + var transaction = storage is null ? null : new ConfigurationTransactionPlan(observed.Evidence.Values, changes); + return Result(verified, changed, transaction); + } + catch (ObservationConflictException) { Error(root, "workspace.migrate_stale", "Inventory or source changed while planning."); return Result(); } + catch (Exception error) when (error is IOException or UnauthorizedAccessException or ArgumentException or NotSupportedException or InvalidOperationException) + { Error(root, "workspace.migrate_io", "Unable to inventory, parse or compose workspace migration. Check paths and permissions."); return Result(); } + } + + public ConfigurationTransactionResult Apply(WorkspaceMigrationPreview preview) => ReferenceEquals(preview.Owner, this) + && preview.CanApply && storage is not null && preview.Transaction is { } transaction + ? new ConfigurationTransaction(storage).Apply(transaction) + : new(ConfigurationTransactionStatus.IoFailure, "workspace.migrate_unverified", [], []); + + private static SourceInput Rebind(SourceInput source, SourceRole role) => new( + new SourceIdentity(source.Source.Path, source.Source.Format, role, source.Source.Order, null), source.Content, source.Required); + private static DocumentResolution BaseResolution(IEnumerable sources, ProjectScope scope, FSharpOption? workspace) => + Resolver.resolve(new ResolutionInput(ReaderValues.List(sources), MapModule.Empty(), DocumentDefaults.emptyValue, + new ProfileOptions(ProfileSelection.Disabled, DocumentDefaults.profileOptions.MaxDepth, DocumentDefaults.profileOptions.MaxExpansions), scope, workspace, FSharpList.Empty)); + + private static ConfigValue ComparableValues(DocumentResolution resolution, IEnumerable sources) + { + var values = ConfigTree.remove("workspace", resolution.Effective); + var controls = ProjectMigration.declaredControls(ReaderValues.List(sources)); + return controls is null ? values : ConfigTree.set(ReaderValues.List(new[] { "profiles" }), controls.Value, values); + } + + private static ProjectLocation Inspect(IProjectTree tree, string path) + { + var entries = DiscoveryPolicy.candidates(path).Select(p => Tuple.Create(p, tree.Observe(p))); + return DiscoveryPolicy.inspectLocation(path, new ProjectSnapshot(MapModule.OfSeq(entries))); + } + private static ProjectMigrationContext Context(IProjectTree tree, string root, IEnumerable convertedSources) + { + var found = ProjectSystem.Discover(tree, root).Discovery; + var converted = convertedSources.ToHashSet(StringComparer.Ordinal); + return new(root, found.Workspace, found.Root.Sources.Where(s => !converted.Contains(s.Source.Path))); + } + private static IEnumerable Classics(ProjectLocation location) => location.Sources.Where(s => + s.Content is SourceContent.Available a + && a.Item.LegacyFields.Any(f => f.RawPath.Any() && f.Disposition == "normalized")); + private static bool Has(ConfigValue value, params string[] path) => ConfigTree.tryFind(ReaderValues.List(path), value) is not null; + private static string? Name(ProjectLocation location) => (ConfigTree.tryFind(ReaderValues.List(new[] { "project", "name" }), location.Values)?.Value as ConfigValue.String)?.Item; + private static string Text(ConfigValue value, string fallback, params string[] path) => (ConfigTree.tryFind(ReaderValues.List(path), value)?.Value as ConfigValue.String)?.Item ?? fallback; + private static IEnumerable Strings(ConfigValue value, params string[] path) => ConfigTree.tryFind(ReaderValues.List(path), value)?.Value is ConfigValue.Array array + ? array.Item.OfType().Select(s => s.Item) : []; + private static ConfigValue ArrayValue(IEnumerable values) => ConfigValue.NewArray(ReaderValues.List(values.Select(ConfigValue.NewString))); + private static bool Skipped(string name) => name is ".git" or ".hg" or ".svn" or "node_modules" or "elm-stuff" or "bin" or "obj" or "dist" or "build" or "target" or "vendor" or ".nuget" or "packages"; + private static bool ValidPattern(string pattern) => !string.IsNullOrWhiteSpace(pattern) && !ProjectPath.isAbsolute(pattern) + && !pattern.Replace('\\','/').Split('/').Contains("..", StringComparer.Ordinal) && !pattern.Contains(':'); + internal static bool Glob(string pattern, string relative) + { + var parts = pattern.Replace('\\','/').Split('/', StringSplitOptions.RemoveEmptyEntries).Where(s => s != ".").ToArray(); + var path = relative.Split('/', StringSplitOptions.RemoveEmptyEntries).Where(s => s != ".").ToArray(); + var memo = new Dictionary<(int,int),bool>(); + bool Match(int p,int v) + { + if (memo.TryGetValue((p,v),out var found)) return found; + var result = p == parts.Length ? v == path.Length : parts[p] == "**" + ? Match(p+1,v) || v < path.Length && Match(p,v+1) + : v < path.Length && Regex.IsMatch(path[v], "\\A" + Regex.Escape(parts[p]).Replace("\\*", ".*").Replace("\\?", ".") + "\\z", RegexOptions.CultureInvariant, TimeSpan.FromSeconds(1)) && Match(p+1,v+1); + memo[(p,v)] = result; return result; + } + return Match(0,0); + } + private static bool Excluded(string relative, IEnumerable patterns) => patterns.Any(p => Glob(p, relative)); + private static ConfigDiagnostic Diagnostic(string path, string code, string message) => new(code, message, DiagnosticSeverity.Error, + new SourceIdentity(path, DocumentFormat.Memory, SourceRole.RootPrimary, 0, null), FSharpList.Empty, null); + private static string Render(string text, IEnumerable edits) + { + foreach (var edit in edits) + { + var rendered = TomlEditor.Edit(text, edit.KeyPath.ToArray(), edit.Value); + if (!rendered.Success) throw new IOException("Mapped value cannot be represented."); + text = rendered.Text; + } + return text; + } + private static TextBuffer BufferFor(ObservedTree tree, string path, ConfigValue raw) + { + tree.Observe(path); + if (tree.Storage is { } storage) + { + var expected = tree.Evidence[path]; + var bytes = expected.Kind == ConfigurationObservationKind.File ? storage.ReadFile(path) : []; + if (storage.Observe(path) != expected) throw new ObservationConflictException(); + return TextBuffer.Decode(bytes); + } + var text = ""; + if (raw is ConfigValue.Object fields) + foreach (var field in fields.Item) text = Render(text, [new(ReaderValues.List(new[] { field.Key }),field.Value)]); + return TextBuffer.Decode(Encoding.UTF8.GetBytes(text)); + } + private sealed class ObservationConflictException : IOException; + private sealed class ObservedTree(IProjectTree inner, string root, IConfigurationTransactionHost? storage) : IProjectTree + { + public string Boundary => root; + public IConfigurationTransactionHost? Storage => storage; + public Dictionary Evidence { get; } = new(StringComparer.Ordinal); + private readonly Dictionary entries = new(StringComparer.Ordinal); + public ProjectEntry Observe(string path) + { + if (!ProjectPath.contains(root,path)) return ProjectEntry.NewFailure("Outside operation boundary"); + var before = storage?.Observe(path); + if (before is not null && Evidence.TryGetValue(path,out var prior) && prior != before) throw new ObservationConflictException(); + var entry=inner.Observe(path); + if (entry is ProjectEntry.File && before?.Kind == ConfigurationObservationKind.File) + { + var bytes=storage!.ReadFile(path); + if (NativeConfigurationTransactionHost.Hash(bytes)!=before.Identity) throw new ObservationConflictException(); + entry=ProjectEntry.NewFile(ConfigurationReader.ReadText(path,TextBuffer.Decode(bytes).Text).Content); + } + if (storage is not null) + { + if (storage.Observe(path)!=before) throw new ObservationConflictException(); + Evidence[path]=before!; + } + if (entries.TryGetValue(path,out var old) && !old.Equals(entry)) throw new ObservationConflictException(); + entries[path]=entry; + return entry; + } + } + // Reuse discovery over exactly the inventoried directories. Globs must not reopen + // excluded output trees or nested workspace boundaries during aggregate verification. + private sealed class InventoryTree(IProjectTree inner, string root, HashSet directories) : IProjectTree + { + public string Boundary => root; + private readonly HashSet primaries = directories.SelectMany(DiscoveryPolicy.candidates).ToHashSet(StringComparer.Ordinal); + public ProjectEntry Observe(string path) + { + if (!directories.Contains(path) && !primaries.Contains(path)) return ProjectEntry.Missing; + var entry = inner.Observe(path); + return entry is ProjectEntry.Directory directory + ? ProjectEntry.NewDirectory(ReaderValues.List(directory.children.Where(directories.Contains))) : entry; + } + } + private sealed class CandidateTree(IProjectTree inner,string root) : IProjectTree + { + public string Boundary => root; + public Dictionary Texts { get; }=new(StringComparer.Ordinal); + private readonly HashSet hidden = new(StringComparer.Ordinal); + public void Set(string path,string text) => Texts[path]=text; + public void Hide(string path) => hidden.Add(path); + public ProjectEntry Observe(string path) => hidden.Contains(path) ? ProjectEntry.Missing + : Texts.TryGetValue(path,out var text) ? ProjectEntry.NewFile(ConfigurationReader.ReadText(path,text).Content) : inner.Observe(path); + } + private sealed record TextBuffer(string Text, Encoding Encoding, byte[] Preamble) + { + public byte[] Encode(string text) => [.. Preamble,.. Encoding.GetBytes(text)]; + public static TextBuffer Decode(byte[] bytes) + { + Encoding[] encodings=[new UTF32Encoding(false,true,true),new UTF32Encoding(true,true,true),new UTF8Encoding(true,true),new UnicodeEncoding(false,true,true),new UnicodeEncoding(true,true,true)]; + foreach(var encoding in encodings) + { + var preamble=encoding.GetPreamble(); + if(bytes.AsSpan().StartsWith(preamble)) return new(encoding.GetString(bytes.AsSpan(preamble.Length)),encoding,preamble); + } + var utf8=new UTF8Encoding(false,true);return new(utf8.GetString(bytes),utf8,[]); + } + } +} diff --git a/src/Morphir.Configuration/Migration.fs b/src/Morphir.Configuration/Migration.fs index 75857786..c5602aa9 100644 --- a/src/Morphir.Configuration/Migration.fs +++ b/src/Morphir.Configuration/Migration.fs @@ -64,6 +64,10 @@ type MigrationVerification = { /// Single-project configuration conversion. No filesystem or ambient state is consulted. module ProjectMigration = + /// Declared profile controls remain observable while profile activation is disabled. + /// Aggregate hosts use the same control merge policy as independent verification. + let declaredControls sources = Profiles.declaredControls sources + let rec same left right = match left, right with | ConfigValue.Number left, ConfigValue.Number right -> diff --git a/src/Morphir.Configuration/ProjectDiscovery.fs b/src/Morphir.Configuration/ProjectDiscovery.fs index b6445f4f..dfbce42c 100644 --- a/src/Morphir.Configuration/ProjectDiscovery.fs +++ b/src/Morphir.Configuration/ProjectDiscovery.fs @@ -190,6 +190,10 @@ module DiscoveryPolicy = ) } + /// Inspect only one supplied location using the shared primary/format policy. + /// No upward search or membership expansion is performed. + let inspectLocation path snapshot = location SourceRole.RootPrimary path snapshot + let private reports role path (snapshot: ProjectSnapshot) = candidates path |> List.choose (fun candidate -> diff --git a/tests/Morphir.Configuration.IO.Tests/WorkspaceMigration.feature b/tests/Morphir.Configuration.IO.Tests/WorkspaceMigration.feature new file mode 100644 index 00000000..d1ccc2f2 --- /dev/null +++ b/tests/Morphir.Configuration.IO.Tests/WorkspaceMigration.feature @@ -0,0 +1,18 @@ +Feature: Bounded workspace migration + Repository adoption uses shared inventory and verified project migration before writes. + + Scenario: Adopt projects without a declared workspace + Given a repository with two classic workspace candidates + When a host previews bounded workspace migration + Then both conversions share one verified virtual workspace + And workspace preview changes no files + + Scenario: Root classic project retains its identity + Given a repository with a root classic project and one child + When a host previews bounded workspace migration + Then the root remains a project outside the member list + + Scenario: Exclusions and nested workspace boundaries explain omissions + Given a repository with excluded and nested workspace candidates + When a host previews bounded workspace migration + Then excluded and boundary paths have no conversions diff --git a/tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationSteps.cs b/tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationSteps.cs new file mode 100644 index 00000000..7c5b3d8e --- /dev/null +++ b/tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationSteps.cs @@ -0,0 +1,28 @@ +// Copyright 2026 FINOS +// SPDX-License-Identifier: Apache-2.0 +using FluentAssertions; +using Morphir.Configuration.IO; +namespace Morphir.Configuration.IO.Tests; + +[Binding] +public sealed class WorkspaceMigrationSteps +{ + private Dictionary files = []; + private WorkspaceMigrationPreview? preview; + [Given("a repository with two classic workspace candidates")] + public void Two() => files = new() { ["/repo/a/morphir.json"]="{name:'A',exposedModules:[]}", ["/repo/b/morphir.json"]="{name:'B',exposedModules:[]}" }; + [Given("a repository with a root classic project and one child")] + public void Root() { Two(); files.Remove("/repo/b/morphir.json"); files["/repo/morphir.json"]="{name:'Root',exposedModules:[]}"; } + [Given("a repository with excluded and nested workspace candidates")] + public void Boundaries() { Two(); files["/repo/morphir.toml"]="[workspace]\nmembers=[]\nexclude=['b']\n"; files["/repo/nested/morphir.toml"]="[workspace]\nmembers=['inner']\n"; files["/repo/nested/inner/morphir.json"]="{name:'Inner',exposedModules:[]}"; } + [When("a host previews bounded workspace migration")] + public void Preview() => preview=new WorkspaceMigrationService(new InMemoryProjectTree("/repo", files)).Plan("/repo"); + [Then("both conversions share one verified virtual workspace")] + public void Virtual() { preview!.CanApply.Should().BeTrue(string.Join(",",preview.Diagnostics.Select(d=>d.Code))); preview.Verified.Should().BeTrue(); preview.RootIsProject.Should().BeFalse(); preview.Members.Should().Equal("a","b"); preview.Projects.Should().HaveCount(2).And.OnlyContain(p=>p.Verification!.BaseWorkspace!.Value.Root=="/repo"); } + [Then("workspace preview changes no files")] + public void Unchanged() => files.Keys.Should().OnlyContain(p=>p.EndsWith(".json",StringComparison.Ordinal)); + [Then("the root remains a project outside the member list")] + public void RootIdentity() { preview!.CanApply.Should().BeTrue(); preview.RootIsProject.Should().BeTrue(); preview.Members.Should().Equal("a"); } + [Then("excluded and boundary paths have no conversions")] + public void Omitted() { preview!.CanApply.Should().BeTrue(); preview.Inventory.Should().Contain(i=>i.RelativePath=="b"&&i.Kind==WorkspaceInventoryKind.Excluded).And.Contain(i=>i.RelativePath=="nested"&&i.Kind==WorkspaceInventoryKind.Boundary); preview.Projects.Should().ContainSingle(); } +} diff --git a/tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationTests.cs b/tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationTests.cs new file mode 100644 index 00000000..2ffae581 --- /dev/null +++ b/tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationTests.cs @@ -0,0 +1,347 @@ +// Copyright 2026 FINOS +// SPDX-License-Identifier: Apache-2.0 +using FluentAssertions; +using Morphir.Configuration.IO; +namespace Morphir.Configuration.IO.Tests; + +public sealed class WorkspaceMigrationInventoryTests +{ + [Test] + public void InventoryIsStableBoundedAndExplainsDefaultSkips() + { + var files=new Dictionary { + ["/repo/z/morphir.json"]="{name:'Same',exposedModules:[]}", + ["/repo/a/.morphir/morphir.json"]="{name:'Same',exposedModules:[]}", + ["/repo/node_modules/pkg/morphir.json"]="{name:'Ignored',exposedModules:[]}", + ["/outside/morphir.json"]="{name:'Outside',exposedModules:[]}" + }; + var tree=new InMemoryProjectTree("/",files); + var plan=new WorkspaceMigrationService(tree).Plan("/repo"); + plan.CanApply.Should().BeTrue(string.Join(",",plan.Diagnostics.Select(d=>d.Code))); + plan.Members.Should().Equal("a","z"); + plan.Projects.Select(p=>p.ProjectPath).Should().Equal("/repo/a","/repo/z"); + plan.Inventory.Should().Contain(i=>i.RelativePath=="node_modules"&&i.Kind==WorkspaceInventoryKind.Excluded); + plan.Diagnostics.Should().Contain(d=>d.Code=="discovery.duplicate_name"); + tree.ObservedPaths.Should().OnlyContain(p=>ProjectPath.contains("/repo",p)); + } + [Test] + public void FiltersAreRootRelativeAndExistingExclusionsRemainEffective() + { + var files=new Dictionary { + ["/repo/morphir.toml"]="[workspace]\nmembers=[]\nexclude=['apps/b']\n", + ["/repo/apps/a/morphir.json"]="{name:'A',exposedModules:[]}", + ["/repo/apps/b/morphir.json"]="{name:'B',exposedModules:[]}", + ["/repo/other/morphir.json"]="{name:'Other',exposedModules:[]}" + }; + var plan=new WorkspaceMigrationService(new InMemoryProjectTree("/repo",files)).Plan("/repo",["apps/**"],["**/ignored"]); + plan.CanApply.Should().BeTrue(); plan.Members.Should().Equal("apps/a"); + plan.Inventory.Should().Contain(i=>i.RelativePath=="apps/b"&&i.Kind==WorkspaceInventoryKind.Excluded).And.Contain(i=>i.RelativePath=="other"&&i.Kind==WorkspaceInventoryKind.Excluded); + } +} + +public sealed class WorkspaceMigrationPlanningTests +{ + private const string Classic="{name:'A',sourceDirectory:'../source',exposedModules:[],profiles:{default:['dev']}}"; + [Test] + public void ExistingWorkspaceSettingsDefaultsAndMigratedMembersRemainEffective() + { + var tree=new InMemoryProjectTree("/repo",new Dictionary { + ["/repo/morphir.toml"]="# root\n[workspace]\nmembers=['old']\ndefault_member='old'\nout_dir='shared-output'\n[pipeline]\nbackend='shared'\n", + ["/repo/old/morphir.toml"]="# exact old bytes\n[project]\nname='Old'\n", + ["/repo/new/.morphir/morphir.json"]=Classic, + ["/repo/new/morphir.user.toml"]="[project]\nsource_directory='WRONG'\n", + ["/repo/new/morphir.dev.toml"]="[project]\nsource_directory='WRONG'\n" + }); + var plan=new WorkspaceMigrationService(tree).Plan("/repo"); + plan.CanApply.Should().BeTrue(string.Join(",",plan.Diagnostics.Select(d=>d.Code))); + plan.Members.Should().Equal("old","new"); + plan.Projects.Should().ContainSingle().Which.ProjectPath.Should().Be("/repo/new"); + var verification=plan.Projects[0].Verification!; + verification.Verified.Should().BeTrue(); + verification.BaseWorkspace!.Value.OutputDirectory!.Value.Should().Be("/repo/shared-output"); + ConfigTree.tryFind(Microsoft.FSharp.Collections.ListModule.OfSeq(new[]{"pipeline","backend"}),verification.After)!.Value.Should().Be(ConfigValue.NewString("shared")); + tree.ObservedPaths.Should().NotContain(p=>p.Contains(".user.",StringComparison.Ordinal)||p.Contains(".dev.",StringComparison.Ordinal)); + plan.Inventory.Should().Contain(i=>i.RelativePath=="old"&&i.Kind==WorkspaceInventoryKind.AlreadyMigrated); + } + [Test] + public void RootAndChildConversionsShareProposedBaseWithEveryClassicExcluded() + { + var tree=new InMemoryProjectTree("/repo",new Dictionary{ + ["/repo/.morphir/morphir.json"]="{name:'Root',exposedModules:[],custom:{same:1}}", + ["/repo/app/.config/morphir/config.json"]="{name:'App',exposedModules:[],custom:{same:1}}" + }); + var plan=new WorkspaceMigrationService(tree).Plan("/repo"); + plan.CanApply.Should().BeTrue(string.Join(",",plan.Diagnostics.Select(d=>d.Code))); + plan.RootIsProject.Should().BeTrue(); plan.Members.Should().Equal("app"); + plan.Projects.Select(p=>p.ProjectPath).Should().Equal("/repo","/repo/app"); + plan.Projects.Should().OnlyContain(p=>p.Verification!.BaseWorkspace!.Value.Root=="/repo"); + plan.AdoptionEffects.Should().Contain(e=>e.Kind=="shared-output-root"&&e.ProjectPath=="/repo/app"&&e.Before=="/repo/app/.morphir/out"&&e.After=="/repo/.morphir/out"); + } + [Test] + public void AdoptionCannotExcuseUnrelatedInheritedSettingChanges() + { + var tree=new InMemoryProjectTree("/repo",new Dictionary{ + ["/repo/morphir.json"]="{name:'Root',exposedModules:[],pipeline:{backend:'root-specific'}}", + ["/repo/app/morphir.json"]="{name:'App',exposedModules:[]}" + }); + var plan=new WorkspaceMigrationService(tree).Plan("/repo"); + plan.CanApply.Should().BeFalse("a new shared output root cannot excuse an unrelated backend change"); + plan.Diagnostics.Should().Contain(d=>d.Code=="workspace.migrate_adoption_value"); + } + [Test] + public void DeclaredGlobsDoNotReadSkippedOutputsOrNestedIndependentMembers() + { + var tree=new InMemoryProjectTree("/repo",new Dictionary{ + ["/repo/morphir.toml"]="[workspace]\nmembers=['apps/**']\n", + ["/repo/apps/a/morphir.json"]="{name:'A',exposedModules:[]}", + ["/repo/apps/node_modules/pkg/morphir.json"]="{name:'Ignored',exposedModules:[]}", + ["/repo/apps/nested/morphir.toml"]="[workspace]\nmembers=['inside']\n", + ["/repo/apps/nested/inside/morphir.json"]="{name:'Inside',exposedModules:[]}" + }); + var plan=new WorkspaceMigrationService(tree).Plan("/repo"); + plan.CanApply.Should().BeTrue(string.Join(",",plan.Diagnostics.Select(d=>d.Code))); + plan.Projects.Select(p=>p.ProjectPath).Should().Equal("/repo/apps/a"); + tree.ObservedPaths.Should().NotContain(p=>p.StartsWith("/repo/apps/node_modules/",StringComparison.Ordinal)||p.StartsWith("/repo/apps/nested/inside",StringComparison.Ordinal)); + } + [Test] + public void SuppliedFiltersDoNotRemoveExistingRootProjectIdentity() + { + var plan=new WorkspaceMigrationService(new InMemoryProjectTree("/repo",new Dictionary{ + ["/repo/morphir.json"]="{name:'Root',exposedModules:[]}", + ["/repo/apps/a/morphir.json"]="{name:'A',exposedModules:[]}" + })).Plan("/repo",["apps/**"]); + plan.CanApply.Should().BeTrue(); plan.RootIsProject.Should().BeTrue(); plan.Members.Should().Equal("apps/a"); + } +} + +public sealed class WorkspaceMigrationApplicationTests +{ + [Test] + public void BatchPreservesSourcesAndEquivalentRepeatBytesTimestampsAndInventory() + { + using var fixture=new WorkspaceFixture(); + fixture.Write("a/.morphir/morphir.json","{name:'A',sourceDirectory:'../source',exposedModules:[]}"); + fixture.Write("b/morphir.json","{name:'B',exposedModules:[]}"); + var sources=Directory.GetFiles(fixture.Root,"*.json",SearchOption.AllDirectories).ToDictionary(p=>p,File.ReadAllBytes); + var plan=fixture.Service.Plan(fixture.Root); plan.CanApply.Should().BeTrue(string.Join(",",plan.Diagnostics.Select(d=>d.Code))); + fixture.Service.Apply(plan).Status.Should().Be(ConfigurationTransactionStatus.Written); + var targets=Directory.GetFiles(fixture.Root,"*.toml",SearchOption.AllDirectories); + var original=targets.ToDictionary(p=>p,File.ReadAllBytes); + var time=new DateTime(2020,1,1,0,0,0,DateTimeKind.Utc); + foreach(var path in targets)File.SetLastWriteTimeUtc(path,time); + var repeated=fixture.Service.Plan(fixture.Root); repeated.CanApply.Should().BeTrue(); repeated.Changed.Should().BeFalse(); + repeated.Inventory.Where(i=>i.Path.EndsWith("/a",StringComparison.Ordinal)||i.Path.EndsWith("/b",StringComparison.Ordinal)).Should().OnlyContain(i=>i.Kind==WorkspaceInventoryKind.AlreadyMigrated); + fixture.Service.Apply(repeated).Status.Should().Be(ConfigurationTransactionStatus.Unchanged); + foreach(var path in targets){File.ReadAllBytes(path).Should().Equal(original[path]);File.GetLastWriteTimeUtc(path).Should().Be(time);} + foreach(var source in sources)File.ReadAllBytes(source.Key).Should().Equal(source.Value); + } + [Test] + [Arguments("source")] + [Arguments("target")] + [Arguments("listing")] + [Arguments("primary")] + public void AggregateRechecksEveryRelevantPreconditionBeforeFirstReplacement(string drift) + { + using var fixture=new WorkspaceFixture(); + fixture.Write("a/morphir.json","{name:'A',exposedModules:[]}"); + fixture.Write("b/morphir.json","{name:'B',exposedModules:[]}"); + var preview=fixture.Service.Plan(fixture.Root); preview.CanApply.Should().BeTrue(); + switch(drift){case "source": fixture.Write("b/morphir.json","{name:'Changed',exposedModules:[]}");break; + case "target":fixture.Write("a/morphir.toml","[custom]\nuser=true\n");break; + case "listing":fixture.Write("new/morphir.json","{name:'New',exposedModules:[]}");break; + default:fixture.Write("b/.config/morphir/config.toml","[custom]\nuser=true\n");break;} + fixture.Service.Apply(preview).Status.Should().Be(ConfigurationTransactionStatus.Conflict); + File.Exists(fixture.Root+"/morphir.toml").Should().BeFalse(); + File.Exists(fixture.Root+"/b/morphir.toml").Should().BeFalse(); + } + [Test] + public void BlockedMemberPreventsEveryWriteAndReadonlyPlansCannotBeTransferred() + { + using var fixture=new WorkspaceFixture(); + fixture.Write("a/morphir.json","{name:'A',exposedModules:[]}"); + fixture.Write("b/morphir.json","{name:'B',exposedModules:[],dependencies:[{env:'secret-reference'}]}"); + var blocked=fixture.Service.Plan(fixture.Root);blocked.CanApply.Should().BeFalse();fixture.Service.Apply(blocked).Status.Should().Be(ConfigurationTransactionStatus.IoFailure); + Directory.GetFiles(fixture.Root,"*.toml",SearchOption.AllDirectories).Should().BeEmpty(); + fixture.Write("b/morphir.json","{name:'B',exposedModules:[]}"); + var readOnly=new WorkspaceMigrationService(new FileSystemProjectTree(fixture.Root)); + var preview=readOnly.Plan(fixture.Root);preview.CanApply.Should().BeTrue(); + readOnly.Apply(preview).Code.Should().Be("workspace.migrate_unverified");fixture.Service.Apply(preview).Code.Should().Be("workspace.migrate_unverified"); + } + [Test] + public void ExistingEmptyVirtualWorkspaceIsUnchangedButUndeclaredEmptyRootCannotAdopt() + { + using var fixture=new WorkspaceFixture(); + fixture.Service.Plan(fixture.Root).CanApply.Should().BeFalse(); + fixture.Write("morphir.toml","# keep\n[workspace]\nmembers=[]\n"); + var preview=fixture.Service.Plan(fixture.Root);preview.CanApply.Should().BeTrue();preview.Changed.Should().BeFalse(); + fixture.Service.Apply(preview).Status.Should().Be(ConfigurationTransactionStatus.Unchanged); + File.ReadAllText(fixture.Root+"/morphir.toml").Should().Be("# keep\n[workspace]\nmembers=[]\n"); + } +} + +internal sealed class WorkspaceFixture:IDisposable +{ + public string Root {get;}=ProjectPath.join(Path.GetTempPath(),"workspace-migration-"+Guid.NewGuid().ToString("N")); + public WorkspaceMigrationService Service {get;} + public WorkspaceFixture(){Directory.CreateDirectory(Root);Service=new(new FileSystemProjectTree(Root),new NativeConfigurationTransactionHost());} + public void Write(string path,string text){path=ProjectPath.join(Root,path);Directory.CreateDirectory(ProjectPath.parent(path));File.WriteAllText(path,text);} + public void Dispose() + { + if(Directory.EnumerateFileSystemEntries(Root,".morphir-edit-*",SearchOption.AllDirectories).Any())Console.Error.WriteLine("Retained workspace recovery evidence: "+Root); + else Directory.Delete(Root,true); + } +} + +public sealed class WorkspaceMigrationBoundaryTests +{ + [Test] + public void DiscoveredLinksAndReversedHostOrderDoNotChangeInventoryOrEscapeRoot() + { + var files=new Dictionary{["/repo/a/morphir.json"]="{name:'A',exposedModules:[]}",["/repo/z/morphir.json"]="{name:'Z',exposedModules:[]}"}; + var first=new WorkspaceMigrationService(new OrderAndLinkTree(new InMemoryProjectTree("/",files),false)).Plan("/repo"); + var second=new WorkspaceMigrationService(new OrderAndLinkTree(new InMemoryProjectTree("/",files),true)).Plan("/repo"); + first.CanApply.Should().BeTrue();second.CanApply.Should().BeTrue();second.Inventory.Should().Equal(first.Inventory);second.Members.Should().Equal(first.Members); + first.Inventory.Should().Contain(i=>i.RelativePath=="linked"&&i.Kind==WorkspaceInventoryKind.LinkSkipped); + } + [Test] + [Arguments("../outside")] + [Arguments("/outside")] + [Arguments("")] + public void InvalidFiltersAreStructuredAndCannotBroadenTheBoundary(string filter) + { + var plan=new WorkspaceMigrationService(new InMemoryProjectTree("/repo",new Dictionary{["/repo/a/morphir.json"]="{name:'A',exposedModules:[]}"})).Plan("/repo",[filter]); + plan.CanApply.Should().BeFalse();plan.Diagnostics.Should().Contain(d=>d.Code=="workspace.migrate_filter"); + } + [Test] + public void ExcludedRootClassicStillSuppliesActualBaseSettingsForAdoptedMembers() + { + var plan=new WorkspaceMigrationService(new InMemoryProjectTree("/repo",new Dictionary{ + ["/repo/morphir.json"]="{name:'Root',exposedModules:[],pipeline:{backend:'root-specific'}}", + ["/repo/app/morphir.json"]="{name:'App',exposedModules:[]}" + })).Plan("/repo",["app"]); + plan.CanApply.Should().BeFalse("retained root settings will inherit after adoption even if its conversion is filtered out"); + plan.Diagnostics.Should().Contain(d=>d.Code=="workspace.migrate_adoption_value"); + } + [Test] + public void ConfiguredOutputDirectoriesAreExcludedWithoutReadingTheirContents() + { + var tree=new InMemoryProjectTree("/repo",new Dictionary{ + ["/repo/morphir.toml"]="[workspace]\nmembers=[]\nout_dir='generated-artifacts'\n", + ["/repo/app/morphir.json"]="{name:'App',exposedModules:[]}", + ["/repo/generated-artifacts/fake/morphir.json"]="{name:'Generated',exposedModules:[]}" + }); + var plan=new WorkspaceMigrationService(tree).Plan("/repo"); + plan.CanApply.Should().BeTrue();plan.Members.Should().Equal("app"); + tree.ObservedPaths.Should().NotContain(p=>p.StartsWith("/repo/generated-artifacts/",StringComparison.Ordinal)); + plan.Inventory.Should().Contain(i=>i.RelativePath=="generated-artifacts"&&i.Kind==WorkspaceInventoryKind.Excluded); + } + [Test] + public void MissingDeclaredDefaultIsInvalidRatherThanSuccessfulNoop() + { + var plan=new WorkspaceMigrationService(new InMemoryProjectTree("/repo",new Dictionary{ + ["/repo/morphir.toml"]="[workspace]\nmembers=[]\ndefault_member='absent'\n" + })).Plan("/repo"); + plan.CanApply.Should().BeFalse();plan.Diagnostics.Should().Contain(d=>d.Code=="workspace.migrate_default"); + } + [Test] + public void GlobCharactersCannotPretendToBeAnExplicitLiteralMemberPath() + { + var plan=new WorkspaceMigrationService(new InMemoryProjectTree("/repo",new Dictionary{["/repo/a?/morphir.json"]="{name:'A',exposedModules:[]}"})).Plan("/repo"); + plan.CanApply.Should().BeFalse();plan.Diagnostics.Should().Contain(d=>d.Code=="workspace.migrate_member_path"); + } + private sealed class OrderAndLinkTree(IProjectTree inner,bool reverse):IProjectTree + { + public string Boundary=>inner.Boundary; + public ProjectEntry Observe(string path) + { + if(!ProjectPath.contains("/repo",path))throw new InvalidOperationException("Scanned outside supplied root"); + if(path=="/repo/linked")return ProjectEntry.Link; + var entry=inner.Observe(path); + if(entry is not ProjectEntry.Directory directory)return entry; + var children=directory.children.AsEnumerable();if(path=="/repo")children=children.Append("/repo/linked"); + if(reverse)children=children.Reverse();return ProjectEntry.NewDirectory(Microsoft.FSharp.Collections.ListModule.OfSeq(children)); + } + } +} + +public sealed class WorkspaceMigrationRecoveryPreviewTests +{ + [Test] + public void PendingJournalMakesPreviewRecoveryRequiredBeforeAnyNewWrite() + { + using var fixture=new WorkspaceFixture(); + fixture.Write("a/morphir.json","{name:'A',exposedModules:[]}"); + var target=fixture.Root+"/a/morphir.toml"; + var initial=new ConfigurationTransactionPlan([ConfigurationTransaction.Observe(fixture.Root)], + [new ConfigurationFileChange(target,ConfigurationTransaction.Observe(target),System.Text.Encoding.UTF8.GetBytes("[project]\nname='A'\n"))]); + var failed=new ConfigurationTransaction(fault:(phase,_)=>{if(phase is ConfigurationTransactionPhase.AfterInstall or ConfigurationTransactionPhase.BeforeRollback)throw new IOException("injected interrupted batch");}).Apply(initial); + failed.Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); + var journal=failed.Artifacts.First(a=>a.Kind==ConfigurationArtifactKind.Journal).Path; + try + { + var preview=fixture.Service.Plan(fixture.Root); + preview.CanApply.Should().BeFalse("pending recovery makes the plan unavailable even in dry-run"); + preview.Diagnostics.Should().Contain(d=>d.Code=="workspace.migrate_recovery_required"); + File.Exists(fixture.Root+"/morphir.toml").Should().BeFalse(); + } + finally + { + new ConfigurationTransaction().Recover(journal).Status.Should().Be(ConfigurationTransactionStatus.RolledBack); + } + } +} + +public sealed class WorkspaceMigrationDeclaredControlTests +{ + [Test] + public void AdoptionCannotHideChangedProfileDeclarationsBehindDisabledActivation() + { + var plan=new WorkspaceMigrationService(new InMemoryProjectTree("/repo",new Dictionary{ + ["/repo/morphir.json"]="{name:'Root',exposedModules:[],profiles:{default:['root-only']}}", + ["/repo/app/morphir.json"]="{name:'App',exposedModules:[]}" + })).Plan("/repo"); + plan.CanApply.Should().BeFalse("disabled activation must not hide newly inherited declaration controls"); + plan.Diagnostics.Should().Contain(d=>d.Code=="workspace.migrate_adoption_value"); + } + [Test] + public void EveryUnicodeKeySegmentStillUsesTheProjectLossLedger() + { + var plan=new WorkspaceMigrationService(new InMemoryProjectTree("/repo",new Dictionary{ + ["/repo/a/morphir.json"]="{name:'A',exposedModules:[],custom:{'\\ud800':{nested:'retained'}}}" + })).Plan("/repo"); + plan.CanApply.Should().BeFalse();plan.Projects.Should().ContainSingle(); + plan.Projects[0].Presentation!.Ledger.ToArray().Should().Contain(f=>f.Disposition=="blocking"); + } +} + +public sealed class WorkspaceMigrationLayoutTests +{ + [Test] + [Arguments(true)] + [Arguments(false)] + public void AggregateReusesExistingSupportedTomlLayoutRatherThanCreatingAmbiguity(bool rootProject) + { + var prefix=rootProject?"":"app/"; + var files=new Dictionary{ + ["/repo/"+prefix+".morphir/morphir.json"]="{name:'A',exposedModules:[]}", + ["/repo/"+prefix+"morphir.toml"]="# keep\n[project]\nname='A'\n" + }; + if(!rootProject)files["/repo/morphir.toml"]="[workspace]\nmembers=['app']\n"; + var plan=new WorkspaceMigrationService(new InMemoryProjectTree("/repo",files)).Plan("/repo"); + plan.CanApply.Should().BeTrue(string.Join(",",plan.Diagnostics.Select(d=>d.Code))); + plan.Projects.Should().ContainSingle().Which.TargetPath.Should().Be("/repo/"+prefix+"morphir.toml"); + plan.Projects[0].ProjectPath.Should().Be(rootProject ? "/repo" : "/repo/app"); + } +} + +public sealed class WorkspaceMigrationClassicSourcePolicyTests +{ + [Test] + public void LegacyJson5PrimaryCannotBeMisreportedAsAlreadyMigrated() + { + var plan=new WorkspaceMigrationService(new InMemoryProjectTree("/repo",new Dictionary{["/repo/a/morphir.json5"]="{name:'A',exposedModules:[]}"})).Plan("/repo"); + plan.CanApply.Should().BeFalse("batch cannot bypass the single-project source contract by calling legacy JSON5 migrated"); + plan.Diagnostics.Should().Contain(d=>d.Code=="migration.classic_required"); + plan.Inventory.Should().Contain(i=>i.RelativePath=="a"&&i.Kind==WorkspaceInventoryKind.Candidate); + } +} From db2e629f3aca172e3f9ceacac55d2d500fae7468 Mon Sep 17 00:00:00 2001 From: Damian Reeves <957246+DamianReeves@users.noreply.github.com> Date: Tue, 3 Nov 2026 14:32:39 -0600 Subject: [PATCH 04/14] feat: expose verified workspace migrate command Use shared plans and recovery outcomes for CLI and JSON reports. Reject ambiguous missing filter values before invoking a writer. Refs #479. --- docs/content/docs/cli/troubleshooting.md | 39 + docs/content/docs/cli/workspace-migrate.md | 190 +++++ docs/content/docs/cli/workspace.md | 8 +- .../prds/workspace-migration-delivery.md | 31 + .../docs/getting-started/configuration.md | 31 +- .../workspace-migration-result.schema.json | 674 ++++++++++++++++++ .../changes/add-workspace-migration/tasks.md | 2 +- .../WorkspaceMigrationCommand.cs | 113 +++ .../WorkspaceMigrationHost.cs | 52 ++ .../WorkspaceMigrationReport.cs | 56 ++ src/Morphir.Tooling/MorphirCli.cs | 2 +- .../Scenarios/WorkspaceScenario.cs | 11 + src/Morphir/Program.cs | 1 + .../WorkspaceMigration.feature | 178 +++++ .../WorkspaceMigrationRecoveryWorkerTests.cs | 47 ++ .../WorkspaceMigrationSteps.cs | 371 ++++++++++ 16 files changed, 1799 insertions(+), 7 deletions(-) create mode 100644 docs/content/docs/cli/workspace-migrate.md create mode 100644 docs/spec/schemas/workspace-migration-result.schema.json create mode 100644 src/Morphir.Tooling/Features/WorkspaceMigration/WorkspaceMigrationCommand.cs create mode 100644 src/Morphir.Tooling/Features/WorkspaceMigration/WorkspaceMigrationHost.cs create mode 100644 src/Morphir.Tooling/Features/WorkspaceMigration/WorkspaceMigrationReport.cs create mode 100644 src/Morphir.Tooling/Scenarios/WorkspaceScenario.cs create mode 100644 tests/Morphir.Tooling.Tests/Features/WorkspaceMigration/WorkspaceMigration.feature create mode 100644 tests/Morphir.Tooling.Tests/Features/WorkspaceMigration/WorkspaceMigrationRecoveryWorkerTests.cs create mode 100644 tests/Morphir.Tooling.Tests/Features/WorkspaceMigration/WorkspaceMigrationSteps.cs diff --git a/docs/content/docs/cli/troubleshooting.md b/docs/content/docs/cli/troubleshooting.md index 3c548287..db8b5cae 100644 --- a/docs/content/docs/cli/troubleshooting.md +++ b/docs/content/docs/cli/troubleshooting.md @@ -582,3 +582,42 @@ write failures. A `rolled-back` outcome restored the original target; a `recovery-required` outcome names owned journal, stage and backup artifacts. Preserve them and later user edits until manual recovery is resolved. Migration retains the original classic source and never edits workspace membership. + + +## Workspace migration blocks or needs recovery + +Run `morphir workspace migrate --dry-run --json` and inspect +`inventory`, `projects`, `diagnostics`, and `verified`. The selected directory is +the boundary even if it has no workspace. A child invocation does not adopt its +parent or siblings. Excluded candidates, default skipped directories, links, +and nested workspace boundaries appear in the inventory. Include/exclude controls +are root relative and repeatable; existing workspace exclusions stay effective. + +One unsupported classic field or conflicting TOML value blocks the entire batch. +Resolve the reported project blocker and create a fresh preview. Migration retains +classic JSON bytes, uses the single-project loss ledger, and excludes ambient +personal/machine/profile/environment settings from equivalence checks. An adoption +effect for shared output roots does not permit source-directory or setting drift. +Generated artifacts remain where they are. + +A conflict means input, target, listing, or topology evidence changed after +planning. Replan after reviewing that change. Do not weaken filesystem boundary or +link guards. On macOS use physical `/private/tmp` rather than its linked aliases. + +`recovery-required` is unfinished work. Preserve all listed journals, stages, +backups, and user-modified targets. Overlapping configuration edits, project +migration, and workspace migration remain blocked until explicit recovery resolves +the journal. `uncertainDirectories` identifies directories whose creation receipt +was interrupted. Do not delete them even if they are empty. Explicit recovery can +preserve reviewed uncertain directories and reports them in `preservedDirectories`. +Inspect the shared transaction recovery report before choosing a recovery action. It identifies restored and unresolved paths; unresolved user +changes are preserved. See [workspace API and recovery]({{< relref +"/docs/configuration/workspace-migration" >}}) for the supported API. There is no +claim of cross-file crash atomicity, and silently deleting artifacts is not +recovery. + +Every JSON result, including parse or host-initialization failure, follows the +[workspace result schema](https://github.com/finos/morphir-dotnet/blob/main/docs/spec/schemas/workspace-migration-result.schema.json). +Exit 0 means applicable preview, success, or valid-workspace no-op. Exit 2 means +blocked, invalid, failed, or recovery required. Diagnostics go to stderr and stdout +contains one JSON document when `--json` is supplied. diff --git a/docs/content/docs/cli/workspace-migrate.md b/docs/content/docs/cli/workspace-migrate.md new file mode 100644 index 00000000..40450676 --- /dev/null +++ b/docs/content/docs/cli/workspace-migrate.md @@ -0,0 +1,190 @@ +--- +title: "workspace migrate command" +linkTitle: "workspace migrate" +weight: 51 +description: "Preview and apply bounded workspace adoption with per-project loss checks and recoverable writes" +--- + +# workspace migrate + +`morphir workspace migrate` adopts eligible projects within one selected directory. +It uses the reusable inventory, aggregate planner/verifier, and transaction APIs. +The same operation is available to embedded hosts without invoking this command. + +## Synopsis + +```text +morphir workspace migrate [] + [--include ]... [--exclude ]... + [--dry-run] [--json] +``` + +`` defaults to the current directory. It must name an existing directory. +That directory bounds inventory, discovery, and all proposed destinations, including +when it contains no workspace declaration. Migration does not search ancestors for +projects to adopt. Select the intended repository root explicitly when invoking +from a member or source directory. + +## Arguments and options + +| Control | Meaning | +| --- | --- | +| `` | Bounded root; omitted means current directory | +| `--include ` | Include candidate project directories matching a root-relative pattern; repeatable | +| `--exclude ` | Exclude candidate project directories; repeatable | +| `--dry-run` | Return the complete verified plan and write nothing | +| `--json` | Write one redacted JSON result to stdout; diagnostics remain on stderr | +| `--help` | Show usage without discovery or migration | + +Patterns are relative to the selected root. `*` and `?` match within a path +segment; `**` matches across directory segments. Absolute patterns and `..` +segments are invalid. Exclusions prune matching subtrees. Shell-quote wildcards +so the shell does not expand them before the CLI receives them. Prefix a literal +leading-dash pattern with `./`, for example `--include './--experimental'`, so it +cannot be mistaken for a missing value followed by an option. Existing `workspace.exclude` +settings remain effective alongside the supplied exclusions. Explicit filters do +not override nested workspace, link, version-control, dependency, or build-output +boundaries. Invalid controls block before application. + +```bash +morphir workspace migrate . --dry-run +morphir workspace migrate ./repo --dry-run --json +morphir workspace migrate . --include 'packages/*' --include 'services/*' \ + --exclude 'packages/experimental-*' --exclude 'services/retired' --dry-run --json +``` + +## Inventory and membership + +Inventory reports project candidates, excluded paths, skipped links, nested declared +workspace boundaries, and already migrated projects in deterministic path order. +It remains bounded even without an existing workspace. It skips generated output, +version-control, and dependency directories and does not follow discovered links. + +The plan uses explicit root-relative paths for new members. Existing membership, +exclusions, settings, and valid `workspace.default_member` remain in place. Already +migrated members are retained without rewriting their primaries. A root project +keeps its identity. A root without a project becomes a virtual workspace. Duplicate +project names produce diagnostics while their distinct paths remain separate. + +For example, a new virtual workspace containing `packages/orders` and +`packages/payments` receives: + +```toml +[workspace] +members = ["packages/orders", "packages/payments"] +``` + +Each classic project receives its own supported TOML primary. Hidden primary +layouts keep the owning project directory as the source-path anchor. + +## Conversion and verification + +Every selected classic project uses the [single-project migration +contract]({{< relref "/docs/cli/project-migrate" >}}), including the complete raw +loss ledger, unsupported-field blockers, conflict checks, and materialized legacy +defaults. A blocked member blocks the aggregate before any destination replacement. + +Migration compares each classic source and its TOML conversion inside the same +proposed base workspace context. It excludes converted classic sources from the +candidate and ambient personal, machine, environment, explicit CLI, and activated +profile overlays from both sides. Declared profile controls are still checked. +The comparison preserves typed values and source-path meaning and checks the +resulting workspace discovery, membership, and selection. + +Adoption effects appear separately in the plan: ownership, membership, default +selection, and shared output roots. Standalone projects may acquire a shared +`/.morphir/out` output context. That effect cannot excuse unrelated +setting or source-path changes. Migration moves no generated artifacts and keeps +every classic source byte-identical. + +## Output + +Human output names the outcome, inventory entries, member paths, project edits, +workspace edits, adoption effects, and verification. Diagnostics go to stderr. + +JSON follows the [workspace migration result +schema](https://github.com/finos/morphir-dotnet/blob/main/docs/spec/schemas/workspace-migration-result.schema.json). +Its fields include: + +| Field | Content | +| --- | --- | +| `operation`, `outcome`, `exitCode` | Operation identity, result state, and exit code | +| `rootPath`, `rootIsProject` | Bounded root and root versus virtual identity | +| `changed`, `dryRun`, `verified` | Planned changes, preview mode, and aggregate verification | +| `inventory` | Path, relative path, kind, reason, and project name | +| `projects` | Per-project redacted ledger, edits, conflicts, and independent verification | +| `members`, `workspaceTarget`, `workspaceEdits` | Reviewed membership and workspace primary edits | +| `adoptionEffects` | Intentional ownership, membership, default-selection, and output-root effects | +| `diagnostics` | Structured warnings and blockers also sent to stderr | +| `recoveryArtifacts`, `restoredPaths`, `unresolvedPaths` | Retained owned evidence and recovery state | +| `uncertainDirectories`, `preservedDirectories` | Directory paths whose ownership is uncertain, or that explicit recovery was instructed to preserve | +| `valueTypes` | Typed encoding metadata for nonfinite and temporal values in workspace edits | + +Each nested project report has its own `valueTypes` array. Its JSON pointers are +relative to that project report. Sensitive settings and opaque secret references +are redacted throughout the reports; reference targets are never resolved. + +```bash +morphir workspace migrate . --dry-run --json > workspace-preview.json +morphir workspace migrate . --json > workspace-result.json +morphir workspace migrate . --json # unchanged after successful adoption +``` + +## Exit codes + +| Code | Meaning | +| --- | --- | +| `0` | Applicable preview, successful application, or valid existing-workspace no-op | +| `2` | Invalid input, blocked conversion, failed verification, stale evidence, failed application, or recovery required | + +An undeclared directory with no eligible projects reports no adoption plan and +exits 2. A valid existing workspace requiring no conversions succeeds unchanged. +Equivalent repeats preserve target bytes and timestamps. + +## Application and recovery + +Application validates and stages the entire aggregate before any destination is +replaced. It coordinates overlapping cooperating writers, rechecks source, +target, listing, and topology preconditions, and records durable recovery evidence. +A later replacement failure triggers rollback attempts. The result identifies +restored and unresolved paths. + +Per-file atomic replacement does not make the batch crash-atomic. Incomplete +rollback or interrupted state retains journals, stages, and backups and produces +`recovery-required`. A read-only preview also reports pending recovery with exit +2 and retains the journal metadata. Overlapping configuration editing and +project/workspace migration writes remain blocked until explicit recovery resolves +that state. +Successful application and proven complete rollback remove only owned transaction +artifacts. User changes and unresolved evidence are preserved. If interruption +occurs after directory creation but before its ownership receipt returns, +`uncertainDirectories` names those paths explicitly. Recovery does not claim or +delete them. The shared recovery API can explicitly preserve reviewed uncertain +directories and reports them in `preservedDirectories`. + +Use the shared [inspection and recovery APIs]({{< relref +"/docs/configuration/workspace-migration" >}}). There is no automatic cleanup of +uncertain recovery state. Do not delete a journal or backup to force a repeat +migration through. Classic sources are never removed by migration or recovery. + +## Troubleshooting and automation + +For a blocker, inspect `projects[*].ledger`, `projects[*].conflicts`, and +`diagnostics` before changing source or target settings. A fresh preview is needed +after source, target, relevant listing, or topology changes. A shared-output-root +adoption effect does not permit a different source directory. + +CI should inspect the exit code and retain a failed JSON result as an artifact: + +```bash +morphir workspace migrate "$REPOSITORY" --dry-run --json > workspace-plan.json +``` + +A preview still requires independent review before choosing to apply it. No +release, deployment, source deletion, or generated-output publication is part of +this operation. Migration is owned by `project` and `workspace`; there is no +`config migrate` alias. + +See the [walkthrough]({{< relref "/docs/getting-started/configuration#adopt-a-repository-of-classic-projects" >}}), +[workspace overview]({{< relref "/docs/cli/workspace" >}}), +and [troubleshooting]({{< relref "/docs/cli/troubleshooting#workspace-migration-blocks-or-needs-recovery" >}}). diff --git a/docs/content/docs/cli/workspace.md b/docs/content/docs/cli/workspace.md index 3428a1dc..28542351 100644 --- a/docs/content/docs/cli/workspace.md +++ b/docs/content/docs/cli/workspace.md @@ -2,12 +2,12 @@ title: "Workspace migration" linkTitle: "Workspace" weight: 50 -description: "Planned bounded adoption of classic projects into a Morphir workspace" +description: "Bounded adoption of classic projects into a Morphir workspace" --- # Workspace migration -Status: planned. `morphir workspace migrate` is not yet implemented in .NET. It will coordinate the shared [project migrator]({{< relref "/docs/cli/project" >}}) and workspace membership planning. Other hosts will use the same library services. +`morphir workspace migrate` coordinates the shared [project migrator]({{< relref "/docs/cli/project" >}}), bounded inventory, aggregate verification, and recoverable application. Other hosts use the same library services. The [complete command reference]({{< relref "/docs/cli/workspace-migrate" >}}) describes the JSON contract and recovery outcomes. Use it when a repository contains several classic projects, including when the repository does not yet declare a Morphir workspace. For one project without workspace membership changes, use `morphir project migrate`. @@ -53,7 +53,9 @@ Check the resulting workspace as a whole against those declared effects. Do not Validate and stage the entire plan before replacing any destination. Recheck relevant source and target preconditions, coordinate overlapping writers, and journal replacements so a commit failure can attempt rollback. -Multiple file replacements are not crash-atomic. Report restored and unresolved paths explicitly if rollback fails. Retain recovery data and block overlapping writes until recovery is resolved. Successful application and complete rollback clean their temporary artifacts. Every classic `morphir.json` remains unchanged. +Multiple file replacements are not crash-atomic. Report restored and unresolved paths explicitly if rollback fails. Retain recovery data and block overlapping writes until recovery is resolved. +A preview encountering pending recovery reports `recovery-required` with exit 2 +and the retained metadata without modifying it. Successful application and complete rollback clean their temporary artifacts. Every classic `morphir.json` remains unchanged. An equivalent repeat reports unchanged files without rewriting them. A valid existing workspace needing no further conversion is a successful no-op. An undeclared root with no eligible projects reports that no adoption can be planned. diff --git a/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md b/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md index b5790443..53ef7190 100644 --- a/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md +++ b/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md @@ -102,3 +102,34 @@ failed compilation. Its subsequently invoked stale-DLL tests are not a current qualification gate. `planner-final-build-02.log` is the successful rebuilt source, followed by `planner-final-io-02.log`, 371/371 current tests. Task 2.3 was marked only after that qualification, at the refreshed source location. + +### CLI adapter and subprocess contracts + +The actual baseline CLI and tooling test assembly were built before CLI runtime +implementation. `cli/cli-red-run-02.log` contains 27 intended missing-command +failures; two early alias-test assertions were corrected to the existing generic +usage contract and are not claimed behavioral evidence. The first integrated +29-case run passed 27 and exposed a missing-filter token being consumed as text +and an unstructured workspace host-initialization crash. Both were reproduced +and fixed. Its mutable log was later overwritten by a script; terminal evidence +and `cli/parse-probe.log` remain, so that overwritten file is not claimed as the +original failure receipt. Later ordinal checkpoint receipts are immutable. + +`cli/cli-green-02-run.log` passes 29/29 after actual CLI rebuilding. An isolated +existing-DLL probe then reproduces a pending-journal dry-run incorrectly returning +applicable preview. `cli/pending-cli-red.log` and the associated JSON receipt +preserve that failure, followed by explicit shared rollback. The final adapter +surfaces pending recovery, uncertain directories and preserved directories through +safe JSON and human outcomes. `cli/cli-green-03-run.log` passes all 31 subprocess +BDD cases after rebuilding the actual CLI against final shared sources. Every +operation report validates against the checked-in schema using a fresh registry. + +The cases include current/supplied bounded roots, repeated include/exclude, +root/virtual/existing workspaces, no adoption/no-op, loss blockers, raw nested +secret redaction, depth 120, parse/initialization failures, source preservation, +no config migrate alias, durable pending recovery and a real process interrupted +after mkdir. Explicit recovery resolves every real CLI fixture. The actual CLI +build retains two existing package warnings; tooling retains three existing +package/generated warnings. `cli/format-03.log` passes own C# formatter verification. +`cli/final-receipt.json` and `cli/final-source-sha256.json` record exact commands, +counts and source identity. Task 4.1 was checked at its refreshed returned location. diff --git a/docs/content/docs/getting-started/configuration.md b/docs/content/docs/getting-started/configuration.md index f9f26c54..c8f0ba94 100644 --- a/docs/content/docs/getting-started/configuration.md +++ b/docs/content/docs/getting-started/configuration.md @@ -176,8 +176,6 @@ See the [migration command reference]({{< relref "/docs/cli/project-migrate" >}} ## Adopt a repository of classic projects -This command remains planned. - If the repository has several classic projects and no new workspace yet, preview the bounded workspace migration from its root: ```bash @@ -189,6 +187,35 @@ The plan lists projects, exclusions, nested workspace boundaries, already migrat Adopting a shared workspace can change standalone output defaults to `/.morphir/out`. The preview reports that change separately from serialization equivalence; it does not move previously generated files. +For two eligible projects under `packages/orders` and `packages/payments`, a new +virtual workspace contains explicit member paths: + +```toml +[workspace] +members = ["packages/orders", "packages/payments"] +``` + +Each converted member receives its own TOML primary with the same loss checks as +single-project migration. If the repository root itself has a project, migration +keeps it as the root project rather than adding `"."` to the member list. Existing +membership patterns, exclusions, valid default selection, comments, and settings +remain in place. Preview the operation from the directory you intend to adopt; +invoking from a child deliberately limits the operation to that child. + +Use JSON in automation and retain the result when application fails: + +```bash +morphir workspace migrate . --dry-run --json > workspace-preview.json +morphir workspace migrate . --json > workspace-result.json +morphir workspace migrate . --json # unchanged repeat +``` + +Exit 2 means blocked, invalid, failed, or recovery required. A recovery-required +result identifies the transaction artifacts and unresolved paths. Stop overlapping +writes and use the shared explicit inspection/recovery APIs described in the +[recovery guide]({{< relref "/docs/configuration/workspace-migration" >}}). +Do not delete journals or backups to make a rerun pass. + Read the [workspace migration reference]({{< relref "/docs/cli/workspace" >}}) for batch recovery and supported controls. ## Inspect configuration in CI diff --git a/docs/spec/schemas/workspace-migration-result.schema.json b/docs/spec/schemas/workspace-migration-result.schema.json new file mode 100644 index 00000000..aa71a679 --- /dev/null +++ b/docs/spec/schemas/workspace-migration-result.schema.json @@ -0,0 +1,674 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://morphir.finos.org/schemas/workspace-migration-result.schema.json", + "title": "Bounded workspace migration result", + "type": "object", + "additionalProperties": false, + "required": [ + "operation", + "outcome", + "rootPath", + "rootIsProject", + "changed", + "dryRun", + "verified", + "inventory", + "projects", + "members", + "workspaceTarget", + "workspaceEdits", + "adoptionEffects", + "diagnostics", + "recoveryArtifacts", + "restoredPaths", + "unresolvedPaths", + "exitCode", + "valueTypes", + "uncertainDirectories", + "preservedDirectories" + ], + "properties": { + "operation": { + "const": "workspace-migrate" + }, + "outcome": { + "enum": [ + "preview", + "written", + "unchanged", + "blocked", + "conflict", + "rolled-back", + "recovery-required", + "io-failure" + ] + }, + "rootPath": { + "type": [ + "string", + "null" + ] + }, + "rootIsProject": { + "type": "boolean" + }, + "changed": { + "type": "boolean" + }, + "dryRun": { + "type": "boolean" + }, + "verified": { + "type": "boolean" + }, + "inventory": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "relativePath", + "kind", + "reason", + "name" + ], + "properties": { + "path": { + "type": "string" + }, + "relativePath": { + "type": "string" + }, + "kind": { + "enum": [ + "Candidate", + "Excluded", + "Boundary", + "AlreadyMigrated", + "LinkSkipped" + ] + }, + "reason": { + "type": [ + "string", + "null" + ] + }, + "name": { + "type": [ + "string", + "null" + ] + } + } + } + }, + "projects": { + "type": "array", + "items": { + "$ref": "#/$defs/projectReport" + } + }, + "members": { + "type": "array", + "items": { + "type": "string" + } + }, + "workspaceTarget": { + "type": [ + "string", + "null" + ] + }, + "workspaceEdits": { + "type": "array", + "items": { + "$ref": "#/$defs/projectReport/$defs/edit" + } + }, + "adoptionEffects": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "projectPath", + "kind", + "before", + "after" + ], + "properties": { + "projectPath": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "before": { + "type": [ + "string", + "null" + ] + }, + "after": { + "type": [ + "string", + "null" + ] + } + } + } + }, + "diagnostics": { + "type": "array", + "items": { + "$ref": "#/$defs/projectReport/$defs/diagnostic" + } + }, + "recoveryArtifacts": { + "type": "array", + "items": { + "$ref": "#/$defs/projectReport/$defs/artifact" + } + }, + "restoredPaths": { + "type": "array", + "items": { + "type": "string" + } + }, + "unresolvedPaths": { + "type": "array", + "items": { + "type": "string" + } + }, + "exitCode": { + "enum": [ + 0, + 2 + ] + }, + "valueTypes": { + "type": "array", + "items": { + "$ref": "#/$defs/projectReport/$defs/valueType" + } + }, + "uncertainDirectories": { + "type": "array", + "items": { + "type": "string" + } + }, + "preservedDirectories": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "$defs": { + "projectReport": { + "title": "Single project migration result", + "type": "object", + "additionalProperties": false, + "required": [ + "operation", + "outcome", + "projectPath", + "source", + "target", + "changed", + "dryRun", + "ledger", + "edits", + "verification", + "diagnostics", + "recoveryArtifacts", + "exitCode", + "valueTypes", + "conflicts" + ], + "properties": { + "operation": { + "const": "migrate" + }, + "outcome": { + "enum": [ + "preview", + "written", + "unchanged", + "blocked", + "conflict", + "rolled-back", + "recovery-required", + "io-failure" + ] + }, + "projectPath": { + "type": [ + "string", + "null" + ] + }, + "source": { + "anyOf": [ + { + "$ref": "#/$defs/projectReport/$defs/source" + }, + { + "type": "null" + } + ] + }, + "target": { + "anyOf": [ + { + "$ref": "#/$defs/projectReport/$defs/source" + }, + { + "type": "null" + } + ] + }, + "changed": { + "type": "boolean" + }, + "dryRun": { + "type": "boolean" + }, + "ledger": { + "type": "array", + "items": { + "$ref": "#/$defs/projectReport/$defs/field" + } + }, + "edits": { + "type": "array", + "items": { + "$ref": "#/$defs/projectReport/$defs/edit" + } + }, + "verification": { + "anyOf": [ + { + "$ref": "#/$defs/projectReport/$defs/verification" + }, + { + "type": "null" + } + ] + }, + "diagnostics": { + "type": "array", + "items": { + "$ref": "#/$defs/projectReport/$defs/diagnostic" + } + }, + "recoveryArtifacts": { + "type": "array", + "items": { + "$ref": "#/$defs/projectReport/$defs/artifact" + } + }, + "exitCode": { + "enum": [ + 0, + 2 + ] + }, + "valueTypes": { + "type": "array", + "items": { + "$ref": "#/$defs/projectReport/$defs/valueType" + } + }, + "conflicts": { + "type": "array", + "items": { + "type": "object", + "required": [ + "key", + "sourceValue", + "targetValue", + "source", + "target" + ], + "properties": { + "key": { + "type": "string" + }, + "sourceValue": {}, + "targetValue": {}, + "source": { + "$ref": "#/$defs/projectReport/$defs/source" + }, + "target": { + "$ref": "#/$defs/projectReport/$defs/source" + } + }, + "additionalProperties": false + } + } + }, + "$defs": { + "source": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "format", + "role", + "order", + "profile" + ], + "properties": { + "path": { + "type": "string" + }, + "format": { + "type": "string" + }, + "role": { + "type": "string" + }, + "order": { + "type": "integer" + }, + "profile": { + "type": [ + "string", + "null" + ] + } + } + }, + "field": { + "type": "object", + "additionalProperties": false, + "required": [ + "rawPath", + "canonicalPath", + "value", + "disposition" + ], + "properties": { + "rawPath": { + "type": "string" + }, + "canonicalPath": { + "type": [ + "string", + "null" + ] + }, + "value": {}, + "disposition": { + "enum": [ + "mapped", + "retained", + "blocking", + "derived" + ] + } + } + }, + "edit": { + "type": "object", + "additionalProperties": false, + "required": [ + "key", + "value" + ], + "properties": { + "key": { + "type": "string" + }, + "value": {} + } + }, + "verification": { + "type": "object", + "additionalProperties": false, + "required": [ + "verified", + "before", + "after", + "differences", + "originChanges", + "baseWorkspace" + ], + "properties": { + "verified": { + "type": "boolean" + }, + "before": {}, + "after": {}, + "differences": { + "type": "array", + "items": { + "$ref": "#/$defs/projectReport/$defs/difference" + } + }, + "originChanges": { + "type": "array", + "items": { + "$ref": "#/$defs/projectReport/$defs/originChange" + } + }, + "baseWorkspace": { + "type": [ + "object", + "null" + ], + "additionalProperties": false, + "required": [ + "root", + "outputDirectory" + ], + "properties": { + "root": { + "type": "string" + }, + "outputDirectory": { + "type": [ + "string", + "null" + ] + } + } + } + } + }, + "difference": { + "type": "object", + "additionalProperties": false, + "required": [ + "key", + "before", + "after", + "beforePath", + "afterPath" + ], + "properties": { + "key": { + "type": "string" + }, + "before": {}, + "after": {}, + "beforePath": { + "type": [ + "string", + "null" + ] + }, + "afterPath": { + "type": [ + "string", + "null" + ] + } + } + }, + "origin": { + "type": "object", + "additionalProperties": false, + "required": [ + "key", + "source", + "line", + "column" + ], + "properties": { + "key": { + "type": "string" + }, + "source": { + "$ref": "#/$defs/projectReport/$defs/source" + }, + "line": { + "type": [ + "integer", + "null" + ] + }, + "column": { + "type": [ + "integer", + "null" + ] + } + } + }, + "originChange": { + "type": "object", + "additionalProperties": false, + "required": [ + "key", + "before", + "after" + ], + "properties": { + "key": { + "type": "string" + }, + "before": { + "anyOf": [ + { + "$ref": "#/$defs/projectReport/$defs/origin" + }, + { + "type": "null" + } + ] + }, + "after": { + "anyOf": [ + { + "$ref": "#/$defs/projectReport/$defs/origin" + }, + { + "type": "null" + } + ] + } + } + }, + "diagnostic": { + "type": "object", + "additionalProperties": false, + "required": [ + "code", + "severity", + "message", + "path", + "key", + "line", + "column" + ], + "properties": { + "code": { + "type": "string" + }, + "severity": { + "enum": [ + "error", + "warning" + ] + }, + "message": { + "type": "string" + }, + "path": { + "type": [ + "string", + "null" + ] + }, + "key": { + "type": [ + "string", + "null" + ] + }, + "line": { + "type": [ + "integer", + "null" + ] + }, + "column": { + "type": [ + "integer", + "null" + ] + } + } + }, + "artifact": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "kind", + "transactionId" + ], + "properties": { + "path": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "transactionId": { + "type": "string" + } + } + }, + "valueType": { + "type": "object", + "additionalProperties": false, + "required": [ + "pointer", + "kind", + "encoding" + ], + "properties": { + "pointer": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "encoding": { + "type": "string" + } + } + } + } + } + } +} diff --git a/openspec/changes/add-workspace-migration/tasks.md b/openspec/changes/add-workspace-migration/tasks.md index 51394135..a855dd90 100644 --- a/openspec/changes/add-workspace-migration/tasks.md +++ b/openspec/changes/add-workspace-migration/tasks.md @@ -21,7 +21,7 @@ Read [the design](design.html) and change-local specs before starting. Existing ## 4. Workspace CLI and integration -- [ ] 4.1 Add red-then-green workspace migrate subprocess scenarios for bounded directory/current roots, repeated include/exclude controls, --dry-run/--json, blocked/no-op results, and absence of config migrate; verify thin command registration and the workspace reference. +- [x] 4.1 Add red-then-green workspace migrate subprocess scenarios for bounded directory/current roots, repeated include/exclude controls, --dry-run/--json, blocked/no-op results, and absence of config migrate; verify thin command registration and the workspace reference. - [ ] 4.2 Run format/build/analyzers, shared/project/workspace/tooling suites, aggregate fault/concurrent-process and contract/coverage checks, and a non-CLI batch preview; verify repeat runs preserve bytes/timestamps and sources, then record results. ## Workflow follow-up diff --git a/src/Morphir.Tooling/Features/WorkspaceMigration/WorkspaceMigrationCommand.cs b/src/Morphir.Tooling/Features/WorkspaceMigration/WorkspaceMigrationCommand.cs new file mode 100644 index 00000000..a48ee6d8 --- /dev/null +++ b/src/Morphir.Tooling/Features/WorkspaceMigration/WorkspaceMigrationCommand.cs @@ -0,0 +1,113 @@ +// Copyright 2026 FINOS +// SPDX-License-Identifier: Apache-2.0 +using System.CommandLine; +using Microsoft.Extensions.DependencyInjection; +using Morphir.Tooling.Infrastructure; + +namespace Morphir.Tooling.Features.WorkspaceMigration; + +/// Registers workspace operations without discovery or file access. +public sealed class WorkspaceCommandProvider : ICommandProvider, IMorphirHostInitializationErrorHandler +{ + private readonly IWorkspaceMigrationHost _host; + private bool _initializationFailed; + + public WorkspaceCommandProvider() : this(new NativeWorkspaceMigrationHost()) { } + public WorkspaceCommandProvider(IWorkspaceMigrationHost host) => _host = host; + void IMorphirHostInitializationErrorHandler.HandleHostInitializationFailure() => _initializationFailed = true; + + public void RegisterCommands(RootCommand rootCommand, IServiceCollection services) + { + var workspace = new Command("workspace", "Manage a bounded Morphir workspace"); + workspace.Subcommands.Add(new WorkspaceMigrationCommand(_host, _initializationFailed)); + rootCommand.Subcommands.Add(workspace); + } +} + +/// Thin argument binding and safe presentation over shared aggregate migration. +public sealed class WorkspaceMigrationCommand : Command, IMorphirParseErrorHandler +{ + private readonly IWorkspaceMigrationHost _host; + private readonly bool _initializationFailed; + private readonly Argument _directory = new("directory") { Arity = ArgumentArity.ZeroOrOne, Description = "Bounded root; defaults to the current directory" }; + private readonly Option _include = new("--include") { Description = "Root-relative candidate pattern; repeatable", Arity = ArgumentArity.OneOrMore, AllowMultipleArgumentsPerToken = false }; + private readonly Option _exclude = new("--exclude") { Description = "Root-relative exclusion pattern; repeatable", Arity = ArgumentArity.OneOrMore, AllowMultipleArgumentsPerToken = false }; + private readonly Option _dryRun = new("--dry-run") { Description = "Preview the complete aggregate without writing" }; + private readonly Option _json = new("--json") { Description = "Emit one redacted structured JSON result" }; + + public WorkspaceMigrationCommand(IWorkspaceMigrationHost host, bool initializationFailed = false) : base("migrate", "Adopt classic projects within one bounded workspace") + { + _host = host; + _initializationFailed = initializationFailed; + Arguments.Add(_directory); + foreach (var option in new Option[] { _include, _exclude, _dryRun, _json }) Options.Add(option); + SetAction(Execute); + } + + public int HandleParseErrors(ParseResult parsed) => Render(WorkspaceMigrationReport.Failure("workspace.migrate_usage", + "Expected one bounded directory and supported workspace migration options."), WantsJson(parsed)); + + private static bool WantsJson(ParseResult parsed) => parsed.Tokens.Any(token => token.Value is "--json" or "--json=true"); + + private int Execute(ParseResult parsed) + { + var json = parsed.GetValue(_json) || WantsJson(parsed); + var dryRun = parsed.GetValue(_dryRun); + var directory = parsed.GetValue(_directory); + var includes = parsed.GetValue(_include) ?? []; + var excludes = parsed.GetValue(_exclude) ?? []; + // The parser accepts an option-looking token as a required string-array value. + // Reject that ambiguity before any host call; './--name' expresses a literal leading dash. + if (directory is not null && string.IsNullOrWhiteSpace(directory) + || includes.Concat(excludes).Any(pattern => string.IsNullOrWhiteSpace(pattern) || pattern.StartsWith('-'))) + return Render(WorkspaceMigrationReport.Failure("workspace.migrate_usage", "Directory and filter patterns must be nonempty. Prefix a literal leading-dash pattern with './'.", dryRun), json); + if (_initializationFailed) + return Render(WorkspaceMigrationReport.Failure("workspace.migrate_initialization", "Host configuration initialization failed. Check appsettings and access permissions.", dryRun), json); + try { return Render(_host.Execute(new(directory, includes, excludes, dryRun)), json); } + catch (Exception error) when (error is IOException or UnauthorizedAccessException or ArgumentException or NotSupportedException) + { return Render(WorkspaceMigrationReport.Failure("workspace.migrate_io", "Workspace migration could not read or write the bounded root. Check paths and permissions.", dryRun), json); } + } + + private static int Render(WorkspaceMigrationReport report, bool json) + { + foreach (var diagnostic in report.Diagnostics) + Console.Error.WriteLine($"{diagnostic.Path ?? "workspace"}: {diagnostic.Code}: {diagnostic.Message}"); + foreach (var artifact in report.RecoveryArtifacts) + Console.Error.WriteLine($"recovery\t{artifact.Kind}\t{artifact.Path}\t{artifact.TransactionId}"); + foreach (var path in report.UnresolvedPaths) Console.Error.WriteLine($"unresolved\t{path}"); + foreach (var directory in report.UncertainDirectories) Console.Error.WriteLine($"uncertain-directory\t{directory}"); + if (json) ConsoleOutputHelper.WriteLineToStdout(report.ToJson()); + else + { + ConsoleOutputHelper.WriteLineToStdout($"{report.Outcome}\t{report.RootPath ?? "workspace"}\t{report.WorkspaceTarget ?? ""}"); + foreach (var entry in report.Inventory) + ConsoleOutputHelper.WriteLineToStdout($"{entry.Kind}\t{entry.RelativePath}\t{entry.Reason ?? ""}"); + foreach (var member in report.Members) ConsoleOutputHelper.WriteLineToStdout($"member\t{member}"); + foreach (var project in report.Projects) + { + foreach (var field in project.Ledger) + ConsoleOutputHelper.WriteLineToStdout($"{field.Disposition}\t{project.ProjectPath}\t{field.RawPath}\t{field.CanonicalPath ?? ""}\t{field.Value.GetRawText()}"); + foreach (var edit in project.Edits) + ConsoleOutputHelper.WriteLineToStdout($"set\t{project.Target?.Path ?? ""}\t{edit.Key}\t{edit.Value.GetRawText()}"); + foreach (var conflict in project.Conflicts) + ConsoleOutputHelper.WriteLineToStdout($"conflict\t{project.ProjectPath}\t{conflict.Key}\t{conflict.SourceValue.GetRawText()}\t{conflict.TargetValue.GetRawText()}"); + ConsoleOutputHelper.WriteLineToStdout($"verification\t{project.ProjectPath}\t{(project.Verification?.Verified == true ? "verified" : "unverified")}"); + if (project.Verification is { } verification) + { + foreach (var difference in verification.Differences) + ConsoleOutputHelper.WriteLineToStdout($"difference\t{project.ProjectPath}\t{difference.Key}\t{difference.Before?.GetRawText() ?? ""}\t{difference.After?.GetRawText() ?? ""}\t{difference.BeforePath ?? ""}\t{difference.AfterPath ?? ""}"); + foreach (var origin in verification.OriginChanges) + ConsoleOutputHelper.WriteLineToStdout($"origin\t{project.ProjectPath}\t{origin.Key}\t{origin.Before?.Source.Path ?? ""}\t{origin.After?.Source.Path ?? ""}"); + } + } + foreach (var edit in report.WorkspaceEdits) + ConsoleOutputHelper.WriteLineToStdout($"set\t{report.WorkspaceTarget ?? ""}\t{edit.Key}\t{edit.Value.GetRawText()}"); + foreach (var effect in report.AdoptionEffects) + ConsoleOutputHelper.WriteLineToStdout($"adoption\t{effect.ProjectPath}\t{effect.Kind}\t{effect.Before ?? ""}\t{effect.After ?? ""}"); + foreach (var path in report.RestoredPaths) ConsoleOutputHelper.WriteLineToStdout($"restored\t{path}"); + foreach (var directory in report.PreservedDirectories) ConsoleOutputHelper.WriteLineToStdout($"preserved-directory\t{directory}"); + ConsoleOutputHelper.WriteLineToStdout($"verification\t{(report.Verified ? "verified" : "unverified")}"); + } + return report.ExitCode; + } +} diff --git a/src/Morphir.Tooling/Features/WorkspaceMigration/WorkspaceMigrationHost.cs b/src/Morphir.Tooling/Features/WorkspaceMigration/WorkspaceMigrationHost.cs new file mode 100644 index 00000000..351e0070 --- /dev/null +++ b/src/Morphir.Tooling/Features/WorkspaceMigration/WorkspaceMigrationHost.cs @@ -0,0 +1,52 @@ +// Copyright 2026 FINOS +// SPDX-License-Identifier: Apache-2.0 +using Morphir.Configuration; +using Morphir.Configuration.IO; +using Morphir.Tooling.Features.Configuration; + +namespace Morphir.Tooling.Features.WorkspaceMigration; + +public sealed record WorkspaceMigrationInvocation(string? Directory, string[] Includes, string[] Excludes, bool DryRun); + +public interface IWorkspaceMigrationHost +{ + WorkspaceMigrationReport Execute(WorkspaceMigrationInvocation invocation); +} + +/// The supplied/current directory is the native authorization boundary. +public sealed class NativeWorkspaceMigrationHost : IWorkspaceMigrationHost +{ + public WorkspaceMigrationReport Execute(WorkspaceMigrationInvocation invocation) + { + var root = ProjectPath.normalize(Path.GetFullPath(invocation.Directory ?? Environment.CurrentDirectory)); + var service = new WorkspaceMigrationService(new FileSystemProjectTree(root), new NativeConfigurationTransactionHost()); + var preview = service.Plan(root, invocation.Includes, invocation.Excludes); + if (preview.PendingRecovery.Length > 0) + return WorkspaceMigrationReport.From(preview, "recovery-required", invocation.DryRun, 2, + artifacts: preview.PendingRecovery.SelectMany(inspection => inspection.Artifacts).Distinct() + .OrderBy(artifact => artifact.Path, StringComparer.Ordinal) + .Select(artifact => new ConfigurationEditArtifactDto(artifact.Path, artifact.Kind.ToString(), artifact.TransactionId)).ToArray(), + unresolvedPaths: preview.PendingRecovery.SelectMany(inspection => inspection.UnresolvedPaths.Length > 0 + ? inspection.UnresolvedPaths : inspection.Targets).Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal).ToArray()); + if (!preview.CanApply) return WorkspaceMigrationReport.From(preview, "blocked", invocation.DryRun, 2); + if (invocation.DryRun) return WorkspaceMigrationReport.From(preview, "preview", true); + var applied = service.Apply(preview); + var outcome = applied.Status switch + { + ConfigurationTransactionStatus.Written => "written", + ConfigurationTransactionStatus.Unchanged => "unchanged", + ConfigurationTransactionStatus.Conflict => "conflict", + ConfigurationTransactionStatus.RolledBack => "rolled-back", + ConfigurationTransactionStatus.RecoveryRequired => "recovery-required", + _ => "io-failure" + }; + var success = applied.Status is ConfigurationTransactionStatus.Written or ConfigurationTransactionStatus.Unchanged; + var inspection = new ConfigurationTransaction(); + var uncertain = applied.Artifacts.Where(artifact => artifact.Kind == ConfigurationArtifactKind.Journal) + .Select(artifact => inspection.InspectRecovery(artifact.Path)).SelectMany(recovery => recovery.UncertainDirectories) + .Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal).ToArray(); + return WorkspaceMigrationReport.From(preview, outcome, false, success ? 0 : 2, success ? null : applied.Code, + applied.Artifacts.Select(artifact => new ConfigurationEditArtifactDto(artifact.Path, artifact.Kind.ToString(), artifact.TransactionId)).ToArray(), + applied.RestoredPaths.ToArray(), applied.UnresolvedPaths.ToArray(), uncertain, applied.PreservedDirectories.ToArray()); + } +} diff --git a/src/Morphir.Tooling/Features/WorkspaceMigration/WorkspaceMigrationReport.cs b/src/Morphir.Tooling/Features/WorkspaceMigration/WorkspaceMigrationReport.cs new file mode 100644 index 00000000..ef4c421d --- /dev/null +++ b/src/Morphir.Tooling/Features/WorkspaceMigration/WorkspaceMigrationReport.cs @@ -0,0 +1,56 @@ +// Copyright 2026 FINOS +// SPDX-License-Identifier: Apache-2.0 +using System.Text.Json; +using System.Text.Json.Serialization; +using Morphir.Configuration; +using Morphir.Configuration.IO; +using Morphir.Tooling.Features.Configuration; +using Morphir.Tooling.Features.ProjectMigration; + +namespace Morphir.Tooling.Features.WorkspaceMigration; + +public sealed record WorkspaceInventoryDto(string Path, string RelativePath, string Kind, string? Reason, string? Name); +public sealed record WorkspaceAdoptionEffectDto(string ProjectPath, string Kind, string? Before, string? After); + +/// Safe aggregate presentation. Canonical candidate bytes are never serialized. +public sealed record WorkspaceMigrationReport( + string Operation, string Outcome, string? RootPath, bool RootIsProject, bool Changed, bool DryRun, bool Verified, + WorkspaceInventoryDto[] Inventory, ProjectMigrationReport[] Projects, string[] Members, string? WorkspaceTarget, + MigrationEditDto[] WorkspaceEdits, WorkspaceAdoptionEffectDto[] AdoptionEffects, ConfigurationDiagnosticDto[] Diagnostics, + ConfigurationEditArtifactDto[] RecoveryArtifacts, string[] RestoredPaths, string[] UnresolvedPaths, + string[] UncertainDirectories, string[] PreservedDirectories, int ExitCode, + ConfigurationValueTypeDto[] ValueTypes) +{ + public string ToJson() => JsonSerializer.Serialize(this, WorkspaceMigrationJsonContext.Default.WorkspaceMigrationReport); + + internal static WorkspaceMigrationReport From(WorkspaceMigrationPreview preview, string outcome, bool dryRun, + int exitCode = 0, string? errorCode = null, ConfigurationEditArtifactDto[]? artifacts = null, + string[]? restoredPaths = null, string[]? unresolvedPaths = null, + string[]? uncertainDirectories = null, string[]? preservedDirectories = null) + { + var types = new List(); + var projects = preview.Projects.Select(project => ProjectMigrationReport.From(project, + !project.CanApply ? "blocked" : !project.Changed ? "unchanged" : outcome == "blocked" ? "preview" : outcome, + dryRun, project.CanApply ? exitCode : 2)).ToArray(); + var edits = preview.WorkspaceEdits.Select((edit, index) => new MigrationEditDto(KeyPath.format(edit.KeyPath), + ConfigurationReport.JsonValue(Editing.redactValue(edit.KeyPath, edit.Value), $"/workspaceEdits/{index}/value", types))).ToArray(); + var diagnostics = preview.Diagnostics.Select(ConfigurationReport.Diagnostic).ToList(); + if (errorCode is not null) + diagnostics.Add(new(errorCode, "error", "Workspace application failed. Inspect the outcome and retained recovery evidence.", preview.WorkspaceTarget, null, null, null)); + return new("workspace-migrate", outcome, preview.RootPath, preview.RootIsProject, preview.Changed, dryRun, preview.Verified, + preview.Inventory.Select(entry => new WorkspaceInventoryDto(entry.Path, entry.RelativePath, entry.Kind.ToString(), entry.Reason, entry.Name)).ToArray(), + projects, preview.Members.ToArray(), preview.WorkspaceTarget, edits, + preview.AdoptionEffects.Select(effect => new WorkspaceAdoptionEffectDto(effect.ProjectPath, effect.Kind, effect.Before, effect.After)).ToArray(), + diagnostics.ToArray(), artifacts ?? [], restoredPaths ?? [], unresolvedPaths ?? [], + uncertainDirectories ?? preview.PendingRecovery.SelectMany(inspection => inspection.UncertainDirectories) + .Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal).ToArray(), preservedDirectories ?? [], exitCode, types.ToArray()); + } + + internal static WorkspaceMigrationReport Failure(string code, string message, bool dryRun = false) => new("workspace-migrate", "blocked", + null, false, false, dryRun, false, [], [], [], null, [], [], [new(code, "error", message, null, null, null, null)], [], [], [], [], [], 2, []); +} + +[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, WriteIndented = true, + MaxDepth = global::Morphir.Tooling.Features.Configuration.ConfigurationReport.MaxJsonDepth)] +[JsonSerializable(typeof(WorkspaceMigrationReport))] +internal partial class WorkspaceMigrationJsonContext : JsonSerializerContext; diff --git a/src/Morphir.Tooling/MorphirCli.cs b/src/Morphir.Tooling/MorphirCli.cs index 92e9a8cc..cf56b387 100644 --- a/src/Morphir.Tooling/MorphirCli.cs +++ b/src/Morphir.Tooling/MorphirCli.cs @@ -69,7 +69,7 @@ internal MorphirCliBuilder(string[] args, string description = "Morphir CLI") _hostBuilder = Host.CreateApplicationBuilder(new HostApplicationBuilderSettings { DisableDefaults = help }); } catch (Exception error) when ((args.FirstOrDefault() == "config" - || args.Length >= 2 && args[0] == "project" && args[1] == "migrate") + || args.Length >= 2 && args[0] is "project" or "workspace" && args[1] == "migrate") && error is InvalidDataException or IOException or UnauthorizedAccessException or ArgumentException or FormatException) { // Retain the failure for an opted-in command's structured renderer. diff --git a/src/Morphir.Tooling/Scenarios/WorkspaceScenario.cs b/src/Morphir.Tooling/Scenarios/WorkspaceScenario.cs new file mode 100644 index 00000000..88d65c79 --- /dev/null +++ b/src/Morphir.Tooling/Scenarios/WorkspaceScenario.cs @@ -0,0 +1,11 @@ +// Copyright 2026 FINOS +// SPDX-License-Identifier: Apache-2.0 +using Morphir.Tooling.Features.WorkspaceMigration; + +namespace Morphir.Tooling.Scenarios; + +/// Workspace registration performs no inventory or migration. +public sealed class WorkspaceScenario : IMorphirCliScenario +{ + public void Configure(MorphirCliBuilder builder) => builder.AddCommandProvider(); +} diff --git a/src/Morphir/Program.cs b/src/Morphir/Program.cs index 6ef93ce6..850b787e 100644 --- a/src/Morphir/Program.cs +++ b/src/Morphir/Program.cs @@ -40,6 +40,7 @@ public static int Main(string[] args) .AddScenario() .AddScenario() .AddScenario() + .AddScenario() .AddCommand(CreateInfoCommand()) .Build(); diff --git a/tests/Morphir.Tooling.Tests/Features/WorkspaceMigration/WorkspaceMigration.feature b/tests/Morphir.Tooling.Tests/Features/WorkspaceMigration/WorkspaceMigration.feature new file mode 100644 index 00000000..a3760612 --- /dev/null +++ b/tests/Morphir.Tooling.Tests/Features/WorkspaceMigration/WorkspaceMigration.feature @@ -0,0 +1,178 @@ +@WorkspaceMigration @CLI +Feature: Adopt classic projects through one bounded workspace migration + Workspace adoption reuses independently verified project conversion and retains sources. + + Scenario Outline: A directory without a workspace supports current and explicit bounded adoption + Given a workspace migration repository + When I migrate the workspace with "" + Then workspace migration exits 0 with outcome "" + And workspace migration retains every classic source + And workspace migration verifies the aggregate and every conversion + And workspace migration reports members "apps/a,apps/b" + + Examples: + | command | outcome | + | --dry-run | preview | + | . --dry-run | preview | + | . | written | + + Scenario: Preview writes no destination + Given a workspace migration repository + When I migrate the workspace with "--dry-run" + Then workspace migration exits 0 with outcome "preview" + And workspace migration writes no destinations + + Scenario: Repeated include and exclude filters are root relative + Given a workspace migration repository + And workspace migration project "tools/c" + When I migrate the workspace with "--include apps/* --include tools/* --exclude apps/b --exclude tools/c --dry-run" + Then workspace migration exits 0 with outcome "preview" + And workspace migration reports members "apps/a" + And workspace migration reports excluded "apps/b,tools/c" + + Scenario: Default skips and nested declared workspaces bound adoption + Given a workspace migration repository + And workspace migration skipped and nested projects + When I migrate the workspace with "--dry-run" + Then workspace migration exits 0 with outcome "preview" + And workspace migration reports members "apps/a,apps/b" + And workspace migration reports boundary "nested" + + Scenario: Existing default and already migrated member remain unchanged + Given a workspace migration repository + And an existing workspace migration member and default + When I migrate the workspace with "" + Then workspace migration exits 0 with outcome "written" + And workspace migration retains the existing member and default + And workspace migration retains every classic source + + Scenario: Root projects retain root identity + Given a workspace migration repository + And a workspace migration root project + When I migrate the workspace with "--dry-run" + Then workspace migration exits 0 with outcome "preview" + And workspace migration retains root project identity + + Scenario: Hidden primary layouts keep the member directory anchor + Given a workspace migration repository + And a hidden workspace migration member primary + When I migrate the workspace with "" + Then workspace migration exits 0 with outcome "written" + And workspace migration retains every classic source + And workspace migration keeps the hidden member anchor + + Scenario: An equivalent repeat preserves every target byte and timestamp + Given a workspace migration repository + When I migrate the workspace with "" + Then workspace migration exits 0 with outcome "written" + When I migrate the workspace with "" + Then workspace migration exits 0 with outcome "unchanged" + And workspace migration preserves every target byte and timestamp + + Scenario Outline: A blocked member prevents every destination + Given a workspace migration repository + And workspace migration fixture "" + When I migrate the workspace with "--dry-run" + Then workspace migration exits 2 with outcome "blocked" + And workspace migration reports diagnostics on stderr + And workspace migration writes no destinations + And workspace migration retains every classic source + + Examples: + | fixture | + | dependencies | + | decorations | + | canonical-conflict | + | invalid-json5 | + + Scenario: An undeclared root with no eligible project cannot be adopted + Given an empty workspace migration repository + When I migrate the workspace with "--dry-run" + Then workspace migration exits 2 with outcome "blocked" + And workspace migration reports diagnostics on stderr + And workspace migration writes no destinations + + Scenario: A valid workspace without conversions succeeds unchanged + Given an empty workspace migration repository + And a declared empty migration workspace + When I migrate the workspace with "" + Then workspace migration exits 0 with outcome "unchanged" + + Scenario: Ancestor and sibling projects are never adopted + Given a workspace migration repository + And workspace migration invoked from a bounded child + When I migrate the workspace with "--dry-run" + Then workspace migration exits 0 with outcome "preview" + And workspace migration reports members "" + And workspace migration retains root project identity + + Scenario: A pending durable transaction blocks a workspace preview with recovery evidence + Given a workspace migration repository + And workspace migration has pending durable recovery + When I migrate the workspace with "--dry-run" + Then workspace migration exits 2 with outcome "recovery-required" + And workspace migration preserves and reports pending recovery + + Scenario: A process interrupted after directory creation reports uncertain ownership + Given a workspace migration repository + And workspace migration has interrupted directory recovery + When I migrate the workspace with "--dry-run" + Then workspace migration exits 2 with outcome "recovery-required" + And workspace migration reports uncertain directory ownership + + Scenario: Secret values stay redacted throughout aggregate reports + Given a workspace migration repository + And workspace migration fixture "secrets" + When I migrate the workspace with "--dry-run" + Then workspace migration exits 0 with outcome "preview" + And workspace migration streams redact secret values + + Scenario: Unsupported raw secret references remain in a redacted loss ledger + Given a workspace migration repository + And workspace migration fixture "unsupported-secrets" + When I migrate the workspace with "--dry-run" + Then workspace migration exits 2 with outcome "blocked" + And workspace migration accounts for unsupported raw fields + And workspace migration streams redact secret values + And workspace migration writes no destinations + + Scenario: Deep raw values fit the aggregate JSON envelope + Given a workspace migration repository + And workspace migration fixture "deep" + When I migrate the workspace with "--dry-run" + Then workspace migration exits 0 with outcome "preview" + And workspace migration verifies the aggregate and every conversion + + Scenario: Human output enumerates verified adoption + Given a workspace migration repository + When I migrate the workspace with human output + Then workspace migration human output reports verified preview + + Scenario: Initialization failures retain JSON stdout + Given a workspace migration repository + And malformed workspace migration appsettings + When I migrate the workspace with "--dry-run" + Then workspace migration exits 2 with outcome "blocked" + And workspace migration reports diagnostics on stderr + And workspace migration writes no destinations + + Scenario Outline: Invalid controls retain the JSON result contract + Given a workspace migration repository + When I migrate the workspace with "" + Then workspace migration exits 2 with outcome "blocked" + And workspace migration reports diagnostics on stderr + And workspace migration writes no destinations + + Examples: + | command | + | --unknown | + | --include | + | --exclude | + | first second | + | morphir.json | + | --include ../outside | + + Scenario: Configuration does not expose a migration alias + Given a workspace migration repository + When I request the configuration migration alias + Then configuration migration remains unavailable diff --git a/tests/Morphir.Tooling.Tests/Features/WorkspaceMigration/WorkspaceMigrationRecoveryWorkerTests.cs b/tests/Morphir.Tooling.Tests/Features/WorkspaceMigration/WorkspaceMigrationRecoveryWorkerTests.cs new file mode 100644 index 00000000..95a5c381 --- /dev/null +++ b/tests/Morphir.Tooling.Tests/Features/WorkspaceMigration/WorkspaceMigrationRecoveryWorkerTests.cs @@ -0,0 +1,47 @@ +// Copyright 2026 FINOS +// SPDX-License-Identifier: Apache-2.0 +using System.Text; +using Morphir.Configuration; +using Morphir.Configuration.IO; + +namespace Morphir.Tooling.Tests.Features.WorkspaceMigration; + +/// A real interrupted process supplies a durable directory-ownership uncertainty fixture. +[Category("WorkspaceMigrationRecoveryWorker")] +public sealed class WorkspaceMigrationRecoveryWorkerTests +{ + [Test] + public void ChildProcess() + { + var root = Environment.GetEnvironmentVariable("MORPHIR_WORKSPACE_RECOVERY_CHILD_ROOT"); + if (root is null) return; + root = ProjectPath.normalize(root); + var target = root + "/new/target.toml"; + var expected = ConfigurationTransaction.Observe(target); + new ConfigurationTransaction(new InterruptedDirectoryHost()).Apply(new ConfigurationTransactionPlan( + [ConfigurationTransaction.Observe(root), expected], + [new ConfigurationFileChange(target, expected, Encoding.UTF8.GetBytes("candidate=true\n"))])); + throw new InvalidOperationException("The controlled child must stop after creating its parent directory"); + } + + private sealed class InterruptedDirectoryHost : IConfigurationTransactionHost + { + private readonly NativeConfigurationTransactionHost native = new(); + public bool SupportsDurableRecovery => true; + public ConfigurationObservation Observe(string path, IReadOnlyCollection? excludedPaths = null) => native.Observe(path, excludedPaths); + public byte[] ReadFile(string path) => native.ReadFile(path); + public IDisposable Acquire(IReadOnlyList paths) => native.Acquire(paths); + public IReadOnlyList CreateParentDirectories(string destination) + { + native.CreateParentDirectories(destination); + Environment.Exit(79); + return []; + } + public string Stage(string destination, ReadOnlyMemory content, string transactionId, string role) => native.Stage(destination, content, transactionId, role); + public void Install(string stagedPath, string destination, bool overwrite) => native.Install(stagedPath, destination, overwrite); + public void DeleteFile(string path) => native.DeleteFile(path); + public void DeleteDirectory(string path) => native.DeleteDirectory(path); + public IReadOnlyList FindRecoveryJournals(IReadOnlyList paths) => native.FindRecoveryJournals(paths); + public IReadOnlyList? EnumerateTransactionArtifacts(string directory, string transactionId) => native.EnumerateTransactionArtifacts(directory, transactionId); + } +} diff --git a/tests/Morphir.Tooling.Tests/Features/WorkspaceMigration/WorkspaceMigrationSteps.cs b/tests/Morphir.Tooling.Tests/Features/WorkspaceMigration/WorkspaceMigrationSteps.cs new file mode 100644 index 00000000..1433b39f --- /dev/null +++ b/tests/Morphir.Tooling.Tests/Features/WorkspaceMigration/WorkspaceMigrationSteps.cs @@ -0,0 +1,371 @@ +// Copyright 2026 FINOS +// SPDX-License-Identifier: Apache-2.0 +using System.Diagnostics; +using System.Reflection; +using System.Text.Json; +using FluentAssertions; +using Json.Schema; +using Morphir.Configuration; +using Morphir.Configuration.IO; + +namespace Morphir.Tooling.Tests.Features.WorkspaceMigration; + +[Binding] +public sealed class WorkspaceMigrationSteps : IDisposable +{ + private readonly string _root = Path.Combine(Path.GetTempPath(), "morphir-workspace-cli-" + Guid.NewGuid().ToString("N")); + private readonly Dictionary _sources = new(StringComparer.Ordinal); + private readonly Dictionary _targets = new(StringComparer.Ordinal); + private string _working = ""; + private string _stdout = ""; + private string _stderr = ""; + private int _exit; + private JsonDocument? _report; + private string? _pendingJournal; + private string? _pendingTarget; + private string? _uncertainDirectory; + private const string ExistingMember = "# existing member stays intact\n[project]\nname='Existing'\nsource_directory='kept'\n"; + private const string ExistingWorkspace = "# preserve workspace\n[workspace]\nmembers=['existing']\ndefault_member='existing'\nout_dir='kept-output'\n[custom]\nkeep=true\n"; + + [Given("an empty workspace migration repository")] + public void Empty() + { + Directory.CreateDirectory(_root); + _working = _root; + } + + [Given("a workspace migration repository")] + public void Repository() + { + Empty(); + Project("apps/a"); + Project("apps/b"); + } + + [Given("workspace migration project {string}")] + public void Project(string relative) => Classic(relative, "{name:'" + relative.Replace('/', '.') + "',sourceDirectory:'elm-src',exposedModules:[],custom:{keep:true}}\n"); + + private void Classic(string relative, string text) + { + var path = Path.Combine(_root, relative, "morphir.json"); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllText(path, text); + _sources[path] = File.ReadAllBytes(path); + } + + private void Write(string relative, string text) + { + var path = Path.Combine(_root, relative); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllText(path, text); + } + + [Given("workspace migration skipped and nested projects")] + public void Skipped() + { + foreach (var path in new[] { ".git/internal", "node_modules/package", "bin/generated", "obj/generated", "nested/child" }) Project(path); + Write("nested/morphir.toml", "[workspace]\nmembers=['child']\n"); + } + + [Given("an existing workspace migration member and default")] + public void Existing() + { + Write("morphir.toml", ExistingWorkspace); + Write("existing/morphir.toml", ExistingMember); + } + + [Given("a hidden workspace migration member primary")] + public void HiddenMember() + { + var original = Path.Combine(_root, "apps", "a", "morphir.json"); + var hidden = Path.Combine(_root, "apps", "a", ".config", "morphir", "config.json"); + Directory.CreateDirectory(Path.GetDirectoryName(hidden)!); + File.Move(original, hidden); + _sources.Remove(original); + _sources[hidden] = File.ReadAllBytes(hidden); + } + + [Given("a declared empty migration workspace")] + public void EmptyWorkspace() => Write("morphir.toml", "[workspace]\nmembers=[]\n"); + + [Given("a workspace migration root project")] + public void RootProject() => Classic("", "{name:'Root',exposedModules:[]}\n"); + + [Given("workspace migration invoked from a bounded child")] + public void BoundedChild() + { + RootProject(); + _working = Path.Combine(_root, "apps", "a"); + } + + [Given("workspace migration fixture {string}")] + public void Fixture(string fixture) + { + var text = fixture switch + { + "dependencies" => "{name:'Blocked',dependencies:['unsupported'],exposedModules:[]}", + "decorations" => "{name:'Blocked',decorations:{unsupported:true},exposedModules:[]}", + "canonical-conflict" => "{name:'Blocked',project:{name:'Blocked'},exposedModules:[]}", + "invalid-json5" => "{name:'Blocked',exposedModules:[}", + "unsupported-secrets" => "{name:'Private',exposedModules:[],dependencies:[{env:'never-print-reference'}],custom:{password:'never-print-literal'}}", + "deep" => "{name:'Deep',exposedModules:[],custom:" + string.Concat(Enumerable.Repeat("{child:", 120)) + "true" + new string('}', 120) + "}", + "secrets" => "{name:'Private',exposedModules:[],custom:{password:'never-print-literal',ref:{env:'never-print-reference'}}}", + _ => throw new ArgumentException("Unknown workspace fixture") + }; + Classic("apps/b", text); + } + + [Given("workspace migration has pending durable recovery")] + public void PendingRecovery() + { + _pendingTarget = ProjectPath.normalize(Path.Combine(_root, "apps", "a", "migration-guard.toml")); + var expected = ConfigurationTransaction.Observe(_pendingTarget); + var transaction = new ConfigurationTransaction(fault: (phase, _) => + { + if (phase is ConfigurationTransactionPhase.AfterInstall or ConfigurationTransactionPhase.BeforeRollback) + throw new IOException("Controlled test interruption requiring explicit recovery"); + }); + var applied = transaction.Apply(new ConfigurationTransactionPlan([expected], + [new ConfigurationFileChange(_pendingTarget, expected, System.Text.Encoding.UTF8.GetBytes("retained recovery guard"))])); + applied.Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); + _pendingJournal = applied.Artifacts.First(artifact => artifact.Kind == ConfigurationArtifactKind.Journal).Path; + File.Exists(_pendingJournal).Should().BeTrue(); + } + + [Given("workspace migration has interrupted directory recovery")] + public async Task InterruptedDirectoryRecovery() + { + var info = new ProcessStartInfo("dotnet") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false + }; + info.ArgumentList.Add(GetType().Assembly.Location); + foreach (var argument in new[] { "--maximum-parallel-tests", "1", "--treenode-filter", "/*/*/*/*[Category=WorkspaceMigrationRecoveryWorker]" }) + info.ArgumentList.Add(argument); + info.Environment["MORPHIR_WORKSPACE_RECOVERY_CHILD_ROOT"] = _root; + using var process = Process.Start(info)!; + var output = process.StandardOutput.ReadToEndAsync(); + var error = process.StandardError.ReadToEndAsync(); + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + try { await process.WaitForExitAsync(timeout.Token); } + catch (OperationCanceledException) { process.Kill(true); throw; } + process.ExitCode.Should().Be(79, await output + await error); + _uncertainDirectory = ProjectPath.normalize(Path.Combine(_root, "new")); + var pending = new ConfigurationTransaction().InspectPending([ProjectPath.normalize(_root)]); + pending.Should().NotBeEmpty(); + _pendingJournal = pending[0].JournalPath; + pending[0].UncertainDirectories.Should().Contain(_uncertainDirectory); + } + + [Given("malformed workspace migration appsettings")] + public void InvalidHost() => Write("appsettings.json", "{never-print-host-value"); + + [When("I migrate the workspace with {string}")] + public Task Migrate(string arguments) => Run(["workspace", "migrate", .. arguments.Split(' ', StringSplitOptions.RemoveEmptyEntries), "--json"]); + + [When("I migrate the workspace with human output")] + public Task Human() => Run(["workspace", "migrate", "--dry-run"]); + + [When("I request the configuration migration alias")] + public Task Alias() => Run(["config", "migrate"]); + + private async Task Run(string[] arguments) + { + _targets.Clear(); + foreach (var path in Directory.GetFiles(_root, "*.toml", SearchOption.AllDirectories)) + _targets[path] = (File.ReadAllBytes(path), File.GetLastWriteTimeUtc(path)); + var repo = Repo(); + var configuration = GetType().Assembly.GetCustomAttribute()!.Configuration; + var cli = Environment.GetEnvironmentVariable("MORPHIR_CONFIGURATION_CLI") ?? Path.Combine(repo, "src", "Morphir", "bin", configuration, "net10.0", "morphir.dll"); + var dll = cli.EndsWith(".dll", StringComparison.Ordinal); + var info = new ProcessStartInfo(dll ? "dotnet" : cli) + { + WorkingDirectory = _working, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false + }; + if (dll) info.ArgumentList.Add(cli); + foreach (var argument in arguments) info.ArgumentList.Add(argument); + foreach (var key in info.Environment.Keys.Where(key => key.StartsWith("MORPHIR_", StringComparison.Ordinal)).ToArray()) info.Environment.Remove(key); + using var process = Process.Start(info)!; + var stdout = process.StandardOutput.ReadToEndAsync(); + var stderr = process.StandardError.ReadToEndAsync(); + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + try { await process.WaitForExitAsync(timeout.Token); } + catch (OperationCanceledException) { process.Kill(true); throw; } + _exit = process.ExitCode; + _stdout = await stdout; + _stderr = await stderr; + } + + [Then("workspace migration exits {int} with outcome {string}")] + public void Result(int exitCode, string outcome) + { + _exit.Should().Be(exitCode, _stdout + _stderr); + _report?.Dispose(); + _report = JsonDocument.Parse(_stdout, new JsonDocumentOptions { MaxDepth = 256 }); + _report.RootElement.GetProperty("outcome").GetString().Should().Be(outcome); + _report.RootElement.GetProperty("exitCode").GetInt32().Should().Be(exitCode); + var schema = JsonSchema.FromText(File.ReadAllText(Path.Combine(Repo(), "docs", "spec", "schemas", "workspace-migration-result.schema.json")), + new BuildOptions { SchemaRegistry = new SchemaRegistry() }); + schema.Evaluate(_report.RootElement).IsValid.Should().BeTrue("every workspace response must match the published schema"); + } + + [Then("workspace migration retains every classic source")] + public void SourceBytes() + { + foreach (var source in _sources) File.ReadAllBytes(source.Key).Should().Equal(source.Value); + } + + [Then("workspace migration verifies the aggregate and every conversion")] + public void Verified() + { + _report!.RootElement.GetProperty("verified").GetBoolean().Should().BeTrue(); + foreach (var project in _report.RootElement.GetProperty("projects").EnumerateArray()) + project.GetProperty("verification").GetProperty("verified").GetBoolean().Should().BeTrue(); + } + + [Then("workspace migration reports members {string}")] + public void Members(string expected) => _report!.RootElement.GetProperty("members").EnumerateArray().Select(e => e.GetString()) + .Should().Equal(expected.Split(',', StringSplitOptions.RemoveEmptyEntries)); + + [Then("workspace migration reports excluded {string}")] + public void Excluded(string paths) + { + var inventory = _report!.RootElement.GetProperty("inventory").EnumerateArray().ToArray(); + foreach (var path in paths.Split(',')) inventory.Should().Contain(e => e.GetProperty("relativePath").GetString() == path && e.GetProperty("kind").GetString() == "Excluded"); + } + + [Then("workspace migration reports boundary {string}")] + public void Boundary(string relative) => _report!.RootElement.GetProperty("inventory").EnumerateArray().Should() + .Contain(e => e.GetProperty("relativePath").GetString() == relative && e.GetProperty("kind").GetString() == "Boundary"); + + [Then("workspace migration writes no destinations")] + public void NoDestinations() + { + var current = Directory.GetFiles(_root, "*.toml", SearchOption.AllDirectories); + current.Should().BeEquivalentTo(_targets.Keys); + foreach (var target in _targets) File.ReadAllBytes(target.Key).Should().Equal(target.Value.Bytes); + } + + [Then("workspace migration retains the existing member and default")] + public void ExistingRetained() + { + File.ReadAllText(Path.Combine(_root, "existing", "morphir.toml")).Should().Be(ExistingMember); + var text = File.ReadAllText(Path.Combine(_root, "morphir.toml")); + text.Should().Contain("default_member='existing'").And.Contain("out_dir='kept-output'").And.Contain("keep=true").And.Contain("# preserve workspace"); + _report!.RootElement.GetProperty("members").EnumerateArray().Select(e => e.GetString()).Should().Contain("existing"); + } + + [Then("workspace migration retains root project identity")] + public void RootIdentity() => _report!.RootElement.GetProperty("rootIsProject").GetBoolean().Should().BeTrue(); + + [Then("workspace migration keeps the hidden member anchor")] + public void HiddenAnchor() + { + var project = _report!.RootElement.GetProperty("projects").EnumerateArray() + .Single(p => p.GetProperty("projectPath").GetString()!.EndsWith("/apps/a", StringComparison.Ordinal)); + project.GetProperty("target").GetProperty("path").GetString().Should().EndWith("/apps/a/.config/morphir/config.toml"); + File.Exists(Path.Combine(_root, "apps", "a", ".config", "morphir", "config.toml")).Should().BeTrue(); + } + + [Then("workspace migration preserves every target byte and timestamp")] + public void Repeat() + { + _targets.Should().NotBeEmpty(); + foreach (var target in _targets) + { + File.ReadAllBytes(target.Key).Should().Equal(target.Value.Bytes); + File.GetLastWriteTimeUtc(target.Key).Should().Be(target.Value.Timestamp); + } + } + + [Then("workspace migration reports diagnostics on stderr")] + public void Diagnostics() + { + _stderr.Should().NotBeNullOrWhiteSpace(); + _report!.RootElement.GetProperty("diagnostics").EnumerateArray().Should().Contain(d => d.GetProperty("severity").GetString() == "error"); + } + + [Then("workspace migration preserves and reports pending recovery")] + public void PendingRecoveryReport() + { + File.Exists(_pendingJournal).Should().BeTrue(); + File.ReadAllText(_pendingTarget!).Should().Be("retained recovery guard"); + _report!.RootElement.GetProperty("recoveryArtifacts").EnumerateArray() + .Should().Contain(artifact => artifact.GetProperty("kind").GetString() == "Journal"); + _report.RootElement.GetProperty("unresolvedPaths").EnumerateArray().Select(path => path.GetString()) + .Should().Contain(_pendingTarget); + _report.RootElement.GetProperty("verified").GetBoolean().Should().BeFalse(); + // The scenario now explicitly requests recovery after verifying the command preserved evidence. + new ConfigurationTransaction().Recover(_pendingJournal!).Status.Should().Be(ConfigurationTransactionStatus.RolledBack); + File.Exists(_pendingTarget).Should().BeFalse(); + File.Exists(_pendingJournal).Should().BeFalse(); + } + + [Then("workspace migration reports uncertain directory ownership")] + public void UncertainDirectoryReport() + { + File.Exists(_pendingJournal).Should().BeTrue(); + Directory.Exists(_uncertainDirectory).Should().BeTrue(); + _report!.RootElement.GetProperty("uncertainDirectories").EnumerateArray().Select(path => path.GetString()) + .Should().Contain(_uncertainDirectory); + _report.RootElement.GetProperty("preservedDirectories").EnumerateArray().Should().BeEmpty(); + var transaction = new ConfigurationTransaction(); + transaction.Recover(_pendingJournal!).Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); + var recovered = transaction.Recover(_pendingJournal!, [_uncertainDirectory!]); + recovered.Status.Should().Be(ConfigurationTransactionStatus.RolledBack); + recovered.PreservedDirectories.Should().Contain(_uncertainDirectory); + Directory.Exists(_uncertainDirectory).Should().BeTrue(); + File.Exists(_pendingJournal).Should().BeFalse(); + } + + [Then("workspace migration accounts for unsupported raw fields")] + public void RawLedger() => _report!.RootElement.GetProperty("projects").EnumerateArray() + .SelectMany(project => project.GetProperty("ledger").EnumerateArray()) + .Should().Contain(field => field.GetProperty("rawPath").GetString() == "dependencies" + && field.GetProperty("disposition").GetString() == "blocking"); + + [Then("workspace migration streams redact secret values")] + public void Redaction() + { + (_stdout + _stderr).Should().NotContain("never-print-literal").And.NotContain("never-print-reference"); + _stdout.Should().Contain("[REDACTED]"); + } + + [Then("workspace migration human output reports verified preview")] + public void HumanResult() + { + _exit.Should().Be(0, _stdout + _stderr); + _stdout.Should().Contain("preview").And.Contain("verified").And.Contain("apps/a").And.Contain("apps/b"); + } + + [Then("configuration migration remains unavailable")] + public void NoAlias() + { + _exit.Should().NotBe(0); + (_stdout + _stderr).Should().Contain("config.usage"); + } + + private static string Repo() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null && !File.Exists(Path.Combine(directory.FullName, "Morphir.slnx"))) directory = directory.Parent; + return directory!.FullName; + } + + public void Dispose() + { + _report?.Dispose(); + if (!Directory.Exists(_root)) return; + if (Directory.EnumerateFiles(_root, ".morphir-edit-*", SearchOption.AllDirectories).Any()) + { + Console.Error.WriteLine("Retained workspace migration CLI recovery evidence: " + _root); + return; + } + Directory.Delete(_root, true); + } +} From de44fa5227b1bfa5ff791ec5a6ee4a7df2ff91d7 Mon Sep 17 00:00:00 2001 From: Damian Reeves <957246+DamianReeves@users.noreply.github.com> Date: Tue, 3 Nov 2026 17:46:58 -0600 Subject: [PATCH 05/14] fix: verify aggregate context and artifact ownership Retain real discovery semantics and original conflict checks while composing root adoption. Reject unverified transaction receipts and preserve changed recovery evidence. --- .../configuration/transaction-recovery.md | 21 +- .../docs/configuration/workspace-migration.md | 18 + .../design/prds/layered-configuration.md | 10 +- .../prds/workspace-migration-delivery.md | 79 ++++ .../ConfigurationRecovery.cs | 47 ++- .../ConfigurationTransaction.cs | 166 +++++++-- .../ProjectMigrationService.cs | 52 ++- .../WorkspaceMigrationService.cs | 156 +++++--- src/Morphir.Configuration/ProjectDiscovery.fs | 3 +- .../Morphir.Configuration.IO.Smoke.csproj | 1 + .../Morphir.Configuration.IO.Smoke/Program.cs | 21 +- .../Morphir.Configuration.IO.Smoke/README.md | 37 +- .../WorkspaceMigrationSmoke.cs | 128 +++++++ .../ConfigurationArtifactOwnershipTests.cs | 161 ++++++++ .../ConfigurationRecoveryApiTests.cs | 126 +++++++ .../ConfigurationTransactionTests.cs | 35 +- .../WorkspaceMigrationApiBoundaryTests.cs | 185 +++++++++ .../WorkspaceMigrationCompositionTests.cs | 109 ++++++ .../WorkspaceMigrationDiscoveryTests.cs | 153 ++++++++ .../WorkspaceMigrationSteps.cs | 16 +- .../WorkspaceMigrationTests.cs | 350 +++++++++--------- 21 files changed, 1570 insertions(+), 304 deletions(-) create mode 100644 tests/Morphir.Configuration.IO.Smoke/WorkspaceMigrationSmoke.cs create mode 100644 tests/Morphir.Configuration.IO.Tests/ConfigurationArtifactOwnershipTests.cs create mode 100644 tests/Morphir.Configuration.IO.Tests/ConfigurationRecoveryApiTests.cs create mode 100644 tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationApiBoundaryTests.cs create mode 100644 tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationCompositionTests.cs create mode 100644 tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationDiscoveryTests.cs diff --git a/docs/content/docs/configuration/transaction-recovery.md b/docs/content/docs/configuration/transaction-recovery.md index e80e11c5..0a8f9ddc 100644 --- a/docs/content/docs/configuration/transaction-recovery.md +++ b/docs/content/docs/configuration/transaction-recovery.md @@ -18,6 +18,12 @@ checks exact source and target bytes, observed absences, directory inventories and link topology. It creates durable write-ahead metadata before staging every candidate and original backup on the corresponding target filesystem. All stages, backups and preconditions must match before the first destination replacement. +Artifact registration requires the predeclared deterministic path to have been +absent before creation and its returned bytes to match the requested content. +A host-returned path or matching transaction ID does not transfer ownership of +an existing file or directory. Directory receipts must be a canonical subset of +the parents observed missing in the plan. + The transaction rechecks remaining inputs between installations. Atomic native replacement protects one destination at a time. @@ -80,8 +86,21 @@ This option accepts only the journal's uncertain paths that remain regular directories. Linked paths, file replacements and unrelated paths remain blocked. It does not authorize deletion or discard user content. +If an adapter returns a foreign file or directory, or claims one in an exception, +the live result reports that path as unresolved without claiming it as an owned +artifact. A separate recovery journal records the rejected receipt. Explicit +recovery preserves the unverified external paths, rolls back only verified owned +changes, and removes only the transaction's verified artifacts. A matching owner +ID or filename prefix alone never authorizes moving, restoring or deleting a file. + If a target, backup, stage or journal has unexpected content or link topology, -recovery preserves it and reports an unresolved path. Do not overwrite the user's +recovery preserves it and reports an unresolved path. Live cleanup validates the +complete artifact set before deleting any artifact, then rechecks each artifact +immediately before deletion. Changing a backup or even reformatting a journal +therefore produces `RecoveryRequired` and retains the guard. Journal bytes must +match their canonical versioned encoding, including known roles and ownership +paths; whitespace, duplicate/unknown fields and role changes are not silently +normalized during recovery. Do not overwrite the user's new bytes with an old backup or delete evidence to bypass the guard. Preserve both versions, review the metadata and restore a specifically approved state before retrying recovery. Invalid/truncated journals need manual review; automatic diff --git a/docs/content/docs/configuration/workspace-migration.md b/docs/content/docs/configuration/workspace-migration.md index f27d12ef..65fa301b 100644 --- a/docs/content/docs/configuration/workspace-migration.md +++ b/docs/content/docs/configuration/workspace-migration.md @@ -45,6 +45,13 @@ empty filters and `..` segments are invalid. Excluded directories are pruned. Existing `workspace.exclude` patterns retain their declared workspace-base anchor and cannot be overridden by an include filter. +Filters select conversions; they do not remove existing declared members. Shared +discovery still inspects those members to verify their unchanged ownership and +selection. If a retained membership glob would enter a skipped output/dependency +tree or nested workspace, the plan reports `workspace.migrate_membership_boundary` +and blocks. Refine the declared membership or exclusions explicitly; migration +does not silently narrow a glob to the projects it happened to inventory. + Version control directories `.git`, `.hg`, `.svn`, dependency directories `node_modules`, `elm-stuff`, `vendor`, `.nuget`, `packages`, and output directories `bin`, `obj`, `dist`, `build`, `target` are skipped. Primary layout directories @@ -72,6 +79,10 @@ undeclared empty root reports that adoption is impossible. project's `Presentation` for logs. The canonical `Plan.Candidate` and `Plan.Edits` contain application values, including secrets; do not log them. Secret references remain opaque leaves. Unsupported values cannot be dropped to make a batch pass. +The classic source contract is the same as project migration: a supported `.json` +manifest can contain full accepted JSON5 syntax. Legacy-shaped primaries in other +extensions are reported as blockers by the project service, rather than falsely +labeled already migrated. Canonical multi-format primaries remain readable. Every conversion receives explicit proposed base workspace context. Verification disables activation and compares declared profile controls separately. The @@ -81,6 +92,13 @@ paths and non-adoption settings. Intended ownership, membership, default selecti and shared output-root changes appear separately in `AdoptionEffects`. They do not authorize other value changes. Existing generated files stay where they are. +Root classic conversion uses the original target for conflict and loss checks, +then composes the reviewed member additions in the common workspace context. +Existing conflicting membership values still block. Declared workspace bases and +output roots do not reanchor the root project's source directory. The additive +`ProjectMigrationService.PlanInWorkspace` API supplies component plans for this +composition; these read-only components cannot bypass aggregate application. + ## Application and recovery Application validates source bytes, target bytes or absence, relevant directory diff --git a/docs/content/docs/contributing/design/prds/layered-configuration.md b/docs/content/docs/contributing/design/prds/layered-configuration.md index a66904e7..88c42158 100644 --- a/docs/content/docs/contributing/design/prds/layered-configuration.md +++ b/docs/content/docs/contributing/design/prds/layered-configuration.md @@ -17,7 +17,7 @@ description: "Shared configuration, project discovery, and configuration inspect Provide one reusable Morphir configuration and project system for the CLI and other hosts. Read TOML, YAML, JSON, and JSON5; discover projects and declared workspaces; and resolve settings with provenance. Shared APIs support `morphir config` inspection/editing, `morphir project migrate` for one classic project, and `morphir workspace migrate` for coordinated workspace adoption. -The original cache-only TOML implementation keeps its Git-first compatibility behavior. Campaign 1 added typed multi-format documents, normalization, resolution, named profiles, validation, and provenance. Campaign 2 adds document-aware discovery through separate library entry points. Campaign 3 adds inspection and standard .NET consumption. Campaign 4 added shared editing and TOML transactions. Campaign 5 adds loss-checked classic project migration. Workspace migration remains a separate later campaign. +The original cache-only TOML implementation keeps its Git-first compatibility behavior. Campaign 1 added typed multi-format documents, normalization, resolution, named profiles, validation, and provenance. Campaign 2 adds document-aware discovery through separate library entry points. Campaign 3 adds inspection and standard .NET consumption. Campaign 4 added shared editing and TOML transactions. Campaign 5 added loss-checked classic project migration. Campaign 6 implements bounded workspace migration and explicit transaction recovery. ## Compatibility and decisions @@ -121,7 +121,7 @@ Each campaign is one cohesive implementation change delivered through one review | 5 | `add-classic-project-migration` | Shared loss-checked migration and project migrate | 2, 4 | | 6 | `add-workspace-migration` | Bounded repository adoption and coordinated workspace migrate | 5 | -Planning artifacts are under `openspec/changes//`, or their dated archive after implementation. Campaigns 1 through 4 are landed and cleaned. Campaign 3 tracks [issue #468](https://github.com/finos/morphir-dotnet/issues/468); its [delivery journal](config-inspection-delivery.md) records shared queries, CLI reads, Microsoft integration and verified landing/cleanup evidence. Campaign 4 tracks [issue #473](https://github.com/finos/morphir-dotnet/issues/473), landed through PR #475 at `56d9de917f6c3035c8a2f994f49dfc96294619d8` with parent cleanup confirmed. Its [delivery journal](config-editing-delivery.md) and [completion receipt](https://github.com/finos/morphir-dotnet/pull/475#issuecomment-6610672018) retain qualification evidence. Campaign 5 tracks [issue #476](https://github.com/finos/morphir-dotnet/issues/476); its [delivery journal](project-migration-delivery.md) records implementation and validation. Parent review, landing and cleanup remain. Campaign 6 remains planned. +Planning artifacts are under `openspec/changes//`, or their dated archive after implementation. Campaigns 1 through 5 are landed and cleaned. Campaign 3 tracks [issue #468](https://github.com/finos/morphir-dotnet/issues/468); its [delivery journal](config-inspection-delivery.md) records shared queries, CLI reads, Microsoft integration and verified landing/cleanup evidence. Campaign 4 tracks [issue #473](https://github.com/finos/morphir-dotnet/issues/473), landed through PR #475 at `56d9de917f6c3035c8a2f994f49dfc96294619d8` with parent cleanup confirmed. Its [delivery journal](config-editing-delivery.md) and [completion receipt](https://github.com/finos/morphir-dotnet/pull/475#issuecomment-6610672018) retain qualification evidence. Campaign 5 tracks [issue #476](https://github.com/finos/morphir-dotnet/issues/476); its [delivery journal](project-migration-delivery.md) records implementation and validation. PR #477 landed campaign 5 at `e3b9a053120a49e94c70c30cd4f62ef783672c64`; its [completion receipt](https://github.com/finos/morphir-dotnet/pull/477#issuecomment-6613145141) confirms review, checks and owned cleanup. Campaign 6 tracks [#479](https://github.com/finos/morphir-dotnet/issues/479) and its [delivery journal](workspace-migration-delivery.md). ### Campaign completion @@ -180,8 +180,8 @@ Follow red-green-refactor for implementation. Start with BDD scenarios and focus | Shared query and edit services | Implemented and landed | Campaigns 3 and 4, PR #475; shared plans, TOML editor, transactions and parent cleanup confirmed | | Classic project migration | Implemented and landed | Campaign 5, #476; pure planner, independent verification, guarded IO, project migrate and managed/trimmed/native qualification | | Workspace migration | In Progress | Bounded inventory, coordinated project plans, membership, and recovery | -| Config CLI commands | Implemented locally; parent gates pending | Campaign 3 get/list/show/path/explain are landed. Campaign 4 set/unset use shared services; all 360 tooling tests pass, including 33 editing subprocess scenarios. Published-host qualification passes; parent delivery gates remain | -| Documentation foundation | Updated for inspection and editing | Shared API and Microsoft consumer examples, read CLI schema/options, walkthrough and troubleshooting; editing API, preview/recovery and CLI examples are implemented; classic project migration reference and walkthrough implemented; workspace migration remains planned | +| Config CLI commands | Implemented and landed | Campaign 3 get/list/show/path/explain are landed. Campaign 4 set/unset use shared services; all 360 tooling tests pass, including 33 editing subprocess scenarios. Published-host qualification and parent landing/cleanup are recorded in the prerequisite receipts | +| Documentation foundation | Updated for inspection and editing | Shared API and Microsoft consumer examples, read CLI schema/options, walkthrough and troubleshooting; editing API, preview/recovery and CLI examples are implemented; classic project migration reference and walkthrough implemented; workspace migration reference, API and recovery docs are implemented in campaign 6 | | Compatibility and end-to-end verification | In Progress | Campaign 1 landed baseline tooling 245, reader 66, core 95 and warning-qualified runtime evidence; current campaign results recorded separately | ## Implementation notes @@ -298,7 +298,7 @@ Follow red-green-refactor for implementation. Start with BDD scenarios and focus discovery now uses `DiscoveryPolicy`, `IProjectTree`, and `ProjectSystem`. Editing now uses shared immutable plans, `ConfigurationEditService` and preconditioned transactions; Microsoft integration landed in campaign 3. - Existing cache-only APIs retain their legacy behavior through additive compatibility entry points. Document how consumers explicitly opt into the new discovery and profile model before moving them. -- Inspection signatures and generated JSON schema are documented in the shared API and CLI references. The editing result schema is published and checked against subprocess reports. The project migration result schema is implemented in campaign 5; workspace migration remains later campaign work. +- Inspection signatures and generated JSON schema are documented in the shared API and CLI references. The editing result schema is published and checked against subprocess reports. The project migration result schema is implemented in campaign 5; workspace migration schema and shared outcomes are implemented in campaign 6. - Campaign 1 pins parser/merge/profile fixtures and records coverage/native results. Pinned dependency linker/AOT warnings remain a qualification limitation; strict warning-free trimming is not claimed. diff --git a/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md b/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md index 53ef7190..7425b567 100644 --- a/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md +++ b/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md @@ -133,3 +133,82 @@ build retains two existing package warnings; tooling retains three existing package/generated warnings. `cli/format-03.log` passes own C# formatter verification. `cli/final-receipt.json` and `cli/final-source-sha256.json` record exact commands, counts and source identity. Task 4.1 was checked at its refreshed returned location. + +### Live artifact ownership review + +During final qualification, lead review identified two unverified live-application +ownership boundaries. `Stage` return paths were registered as owned without +checking their deterministic target/role identity, and live cleanup did not first +check whether a backup/journal had changed. Eight new regressions are written +before fixes. Tasks 3.2 and 3.3 were explicitly reopened at fresh returned +checkbox locations; earlier passing receipts are source checkpoints, not final +qualification of these guards. Broad runner evidence is retained as an initial +checkpoint while the new cases are reproduced and resolved. + +The initial broad run completed all 17 configured executable runners successfully +before the live-ownership fixes. Its receipts are preserved in +`validation-initial/`; they are not substituted for final-source qualification. +Counts include core 177, IO 371, tooling 433 and Microsoft 22. The initial full +solution build has 11 existing dependency/generated warnings and no errors. +C# full formatter verification and changed F# Fantomas checks pass. The final +ownership regression set reproduces all ten failures in +`transactions/live-ownership-red-run.log`, with a clean preceding build. + +### Coverage-driven public API qualification + +Fresh managed non-CLI qualification passes 28 workspace checks, 32 project +migration checks, 44 editing checks, 228 JSON5 grammar cases and 26 pinned Morphir +documents. The compiled host exercises batch preview/application, safe generated +report serialization, source retention, output retention, byte/timestamp repeats, +drift and explicit recovery. `qualification-first-summary.json` records exact +commands and exits; all six format/build/smoke/collector gates pass. + +The first current-source coverage union is core 4188/4345, 96.39%, and IO +2367/2459, 96.26%. IO is below prerequisite 97.15%, so final qualification remains +open. Public API boundary tests were added for bounded invalid inputs, mounted +listing/read/drift failures, native staging/parent failures, explicit recovery +faults/capabilities/options, encodings and full JSON5 escaped values/opaque arrays. + +That work also reproduced a real multi-format no-op defect: an existing declared +workspace without root TOML could stage an empty primary despite `Changed=false`. +`format-noop-red-run.log` fails all six YAML/YML/JSON/JSON5/supplemental/hidden +variants before the fix. The aggregate now overlays a root file only when it +has workspace edits or an actual root conversion. `format-noop-green-run.log` +passes all six with unchanged source bytes and no new primary. Task 2.2 was +explicitly reopened for this finding before the minimal production fix. + +### Final discovery and composition corrections + +`discovery-verification/actual-discovery-probe.json` records the initial defect: +application could discover four members after a preview verified only one through +a sanitized inventory tree. The native discovery regression suite first fails +three of four cases, then passes all four. Verification now uses actual shared +discovery listings, retains filtered existing membership and rejects declared +globs that cross a hard dependency/output/nested-workspace boundary. It verifies +both retained-classic and source-excluded views. No generic discovery semantics +or existing member declarations are changed to make verification pass. + +`root-composition-red-run.log` reproduces a root classic manifest's original +membership being rejected against adoption's own new members. Shared component +planning now maps against the original target, validates explicit proposed +membership against discovery, and composes only the reviewed membership edit. +A fabricated original target and undeclared membership are rejected. A separate +red case proves component plans must not independently apply hypothetical +workspace context; components are now read-only and aggregate application owns +all writes. Existing target membership conflicts remain blockers. + +`root-base-run.log` passes all seven composition cases after actual rebuilding, +including custom output roots and `workspace.root='sub'` with a root source path. +The common model uses canonical ownership controls without rendering temporary +configuration or weakening raw unsupported-value accounting. Every member uses +the same base workspace; ordinary source paths/settings retain the shared +single-project verifier. `root-composition-green-io.log` passes the preceding +428-case full IO checkpoint; final qualification below supersedes it. + +Live artifact ownership regressions now pass. Stage and directory receipts must +match predeclared deterministic ownership; cleanup validates every owned artifact +before removing any. Altered backups/journals and unverified foreign stage paths +retain evidence and durable overlap guards. Inspection exposes unresolved paths +after process-local result loss. Deliberately unresolved fault fixtures remain +under the owned external artifact root for parent inspection; they are not +silently cleaned or represented as successful recovery. diff --git a/src/Morphir.Configuration.IO/ConfigurationRecovery.cs b/src/Morphir.Configuration.IO/ConfigurationRecovery.cs index b79d64b6..2723190d 100644 --- a/src/Morphir.Configuration.IO/ConfigurationRecovery.cs +++ b/src/Morphir.Configuration.IO/ConfigurationRecovery.cs @@ -23,6 +23,8 @@ internal sealed record RecoveryDocument(string TransactionId, bool Committed, ImmutableArray Artifacts) { internal ImmutableArray PendingDirectories { get; init; } = []; + internal ImmutableArray PlannedDirectories { get; init; } = []; + internal ImmutableArray UnverifiedPaths { get; init; } = []; // A committed witness must remain above every destination while cleanup runs. internal string MetadataDestination => ProjectPath.join(Guards.First(g => Entries.All(e => ProjectPath.contains(g, e.Target))), ".morphir-transaction"); @@ -41,6 +43,8 @@ internal byte[] Encode() { writer.WriteStartObject(); writer.WriteNumber("version", 2); writer.WriteString("transactionId", TransactionId); writer.WriteBoolean("committed", Committed); + writer.WriteStartArray("plannedDirectories"); foreach (var path in PlannedDirectories) writer.WriteStringValue(path); writer.WriteEndArray(); + writer.WriteStartArray("unverifiedPaths"); foreach (var path in UnverifiedPaths) writer.WriteStringValue(path); writer.WriteEndArray(); writer.WriteStartArray("pendingDirectories"); foreach (var path in PendingDirectories) writer.WriteStringValue(path); writer.WriteEndArray(); writer.WriteStartArray("guards"); foreach (var path in Guards) writer.WriteStringValue(path); writer.WriteEndArray(); writer.WriteStartArray("entries"); @@ -71,6 +75,8 @@ internal static RecoveryDocument Decode(byte[] bytes, string journalPath) if (id.Length != 32 || id.Any(c => !Uri.IsHexDigit(c))) throw new InvalidDataException("Invalid journal owner"); var pendingDirectories = root.TryGetProperty("pendingDirectories", out var pending) ? pending.EnumerateArray().Select(Text).ToImmutableArray() : []; + var plannedDirectories = root.GetProperty("plannedDirectories").EnumerateArray().Select(Text).ToImmutableArray(); + var unverifiedPaths = root.GetProperty("unverifiedPaths").EnumerateArray().Select(Text).ToImmutableArray(); var guards = root.GetProperty("guards").EnumerateArray().Select(Text).ToImmutableArray(); var entries = root.GetProperty("entries").EnumerateArray().Select(e => new RecoveryEntry( Text(e.GetProperty("target")), Enum.Parse(Text(e.GetProperty("originalKind"))), @@ -82,6 +88,19 @@ internal static RecoveryDocument Decode(byte[] bytes, string journalPath) static bool Hash(string value) => value.Length == 64 && value.All(Uri.IsHexDigit); bool Owned(string path) => Canonical(path) && Path.GetFileName(path).StartsWith(".morphir-edit-" + id + "-", StringComparison.Ordinal) && guards.Contains(ProjectPath.parent(path), StringComparer.Ordinal); + var result = new RecoveryDocument(id, root.GetProperty("committed").GetBoolean(), guards, entries, artifacts) + { PendingDirectories = pendingDirectories, PlannedDirectories = plannedDirectories, UnverifiedPaths = unverifiedPaths }; + var knownJournals = guards.Select(g => ArtifactPath(ProjectPath.join(g, ".morphir-transaction"), id, "journal")) + .Concat(new[] { "journal-directories", "journal-ready", "journal-committed", "journal-recovery" } + .Select(role => ArtifactPath(result.MetadataDestination, id, role))).ToHashSet(StringComparer.Ordinal); + bool ExpectedArtifact(ConfigurationArtifact artifact) => artifact.Kind switch + { + ConfigurationArtifactKind.Stage => entries.Any(e => e.Stage == artifact.Path), + ConfigurationArtifactKind.Backup => entries.Any(e => e.Backup == artifact.Path), + ConfigurationArtifactKind.Journal => knownJournals.Contains(artifact.Path), + ConfigurationArtifactKind.Directory => plannedDirectories.Contains(artifact.Path, StringComparer.Ordinal), + _ => false + }; if (guards.IsDefaultOrEmpty || guards.Any(p => !Canonical(p)) || entries.IsDefaultOrEmpty || entries.Select(e => e.Target).Distinct(StringComparer.Ordinal).Count() != entries.Length || entries.Any(e => !Canonical(e.Target) || !Hash(e.CandidateIdentity) @@ -91,14 +110,17 @@ bool Owned(string path) => Canonical(path) && Path.GetFileName(path).StartsWith( || !Owned(e.Stage) || e.Stage != ArtifactPath(e.Target, id, "stage") || e.Backup is not null && (!Owned(e.Backup) || e.Backup != ArtifactPath(e.Target, id, "backup")) || !guards.Contains(ProjectPath.parent(e.Target), StringComparer.Ordinal)) - || pendingDirectories.Any(p => !Canonical(p) || !entries.Any(e => e.Target != p && ProjectPath.contains(p, e.Target))) + || plannedDirectories.Any(p => !Canonical(p) || !entries.Any(e => e.Target != p && ProjectPath.contains(p, e.Target))) + || pendingDirectories.Any(p => !plannedDirectories.Contains(p, StringComparer.Ordinal)) + || artifacts.Any(a => !ExpectedArtifact(a)) || artifacts.Any(a => !Enum.IsDefined(a.Kind)) || artifacts.Any(a => a.Kind == ConfigurationArtifactKind.Directory ? !Canonical(a.Path) || !guards.Any(g => ProjectPath.contains(a.Path, g)) : !Owned(a.Path)) - || !Owned(journalPath) || !IsJournal(journalPath)) + || !Owned(journalPath) || !knownJournals.Contains(journalPath) + || result.Committed != journalPath.EndsWith("-journal-committed", StringComparison.Ordinal)) throw new InvalidDataException("Invalid journal ownership"); - return new(id, root.GetProperty("committed").GetBoolean(), guards, entries, artifacts) { PendingDirectories = pendingDirectories }; + return result; } } @@ -155,7 +177,7 @@ ConfigurationTransactionResult Required(string code, IEnumerable? restor var document = LatestRecoveryDocument(ReadRecovery(journalPath)); if (preserved.Any(p => !document.PendingDirectories.Contains(p, StringComparer.Ordinal) || host.Observe(p).Kind != ConfigurationObservationKind.Directory)) return Required("config.recovery_invalid_options"); - if (RecoveryUnresolved(document, RecoveryArtifacts(document, journalPath), preserved).Length > 0) return Required(inspection.Code); + if (RecoveryUnresolved(document, RecoveryArtifacts(document, journalPath), preserved, ignoreUnverified: true).Length > 0) return Required(inspection.Code); if (!SameOperation(initial, document)) return Required("config.recovery_changed"); if (document.Guards.Any(g => host.EnumerateTransactionArtifacts(g, document.TransactionId) is null)) return Required("config.recovery_host_capability"); @@ -193,7 +215,7 @@ ConfigurationTransactionResult Required(string code, IEnumerable? restor } } var artifacts = RecoveryArtifacts(document, journalPath).ToList(); - if (unresolved.Count > 0 || !TryCleanup(artifacts, document.Committed, document)) + if (unresolved.Count > 0 || !TryCleanup(artifacts, document.Committed, document, preservedDirectories: preserved)) return new(ConfigurationTransactionStatus.RecoveryRequired, "config.recovery_required", inspection.Targets, Retained(artifacts)) { RestoredPaths = restored.ToImmutableArray(), UnresolvedPaths = unresolved.Count > 0 ? unresolved.ToImmutableArray() : inspection.Targets }; return new(document.Committed ? ConfigurationTransactionStatus.Written : ConfigurationTransactionStatus.RolledBack, @@ -205,7 +227,7 @@ ConfigurationTransactionResult Required(string code, IEnumerable? restor private RecoveryDocument LatestRecoveryDocument(RecoveryDocument initial) { - foreach (var role in new[] { "journal-committed", "journal-ready", "journal-directories" }) + foreach (var role in new[] { "journal-committed", "journal-recovery", "journal-ready", "journal-directories" }) { var path = RecoveryDocument.ArtifactPath(initial.MetadataDestination, initial.TransactionId, role); if (host.Observe(path).Kind == ConfigurationObservationKind.Missing) continue; @@ -224,7 +246,9 @@ private RecoveryDocument ReadRecovery(string path) var bytes = host.ReadFile(path); if (NativeConfigurationTransactionHost.Hash(bytes) != observed.Identity || host.Observe(path) != observed) throw new InvalidDataException("Journal changed"); - return RecoveryDocument.Decode(bytes, path); + var document = RecoveryDocument.Decode(bytes, path); + if (!document.Encode().AsSpan().SequenceEqual(bytes)) throw new InvalidDataException("Journal bytes are not canonical"); + return document; } private ImmutableArray RecoveryArtifacts(RecoveryDocument document, string journalPath) @@ -238,13 +262,13 @@ private ImmutableArray RecoveryArtifacts(RecoveryDocument known.Add(new(path, RecoveryDocument.IsJournal(path) ? ConfigurationArtifactKind.Journal : path.EndsWith("-backup", StringComparison.Ordinal) ? ConfigurationArtifactKind.Backup : ConfigurationArtifactKind.Stage, document.TransactionId)); } - return known.Distinct().ToImmutableArray(); + return known.Where(a => !document.UnverifiedPaths.Contains(a.Path, StringComparer.Ordinal)).Distinct().ToImmutableArray(); } private ImmutableArray RecoveryUnresolved(RecoveryDocument document, IEnumerable artifacts, - IReadOnlySet? preservedDirectories = null) + IReadOnlySet? preservedDirectories = null, bool ignoreUnverified = false) { - var unresolved = new HashSet(StringComparer.Ordinal); + var unresolved = new HashSet(ignoreUnverified ? [] : document.UnverifiedPaths, StringComparer.Ordinal); foreach (var path in document.PendingDirectories) { var observed = host.Observe(path); @@ -284,6 +308,7 @@ private ImmutableArray RecoveryUnresolved(RecoveryDocument document, IEn private static bool SameOperation(RecoveryDocument first, RecoveryDocument second) => first.TransactionId == second.TransactionId && first.Guards.SequenceEqual(second.Guards) - && first.Entries.SequenceEqual(second.Entries); + && first.Entries.SequenceEqual(second.Entries) + && first.PlannedDirectories.SequenceEqual(second.PlannedDirectories); private static bool IsRecoveryError(Exception error) => IsIo(error) || error is JsonException or InvalidDataException or InvalidOperationException or ArgumentException or KeyNotFoundException or FormatException or OverflowException; } diff --git a/src/Morphir.Configuration.IO/ConfigurationTransaction.cs b/src/Morphir.Configuration.IO/ConfigurationTransaction.cs index 2e610326..6dd24cf9 100644 --- a/src/Morphir.Configuration.IO/ConfigurationTransaction.cs +++ b/src/Morphir.Configuration.IO/ConfigurationTransaction.cs @@ -121,6 +121,11 @@ public ConfigurationTransactionResult Apply(ConfigurationTransactionPlan plan) var paths = observations.Select(o => o.Path).Concat(targets).Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal).ToArray(); var transactionId = Guid.NewGuid().ToString("N"); var artifacts = new List(); + var artifactEvidence = new Dictionary(StringComparer.Ordinal); + var stageAttempts = new Dictionary(StringComparer.Ordinal); + var partialArtifacts = new HashSet(StringComparer.Ordinal); + var plannedDirectories = new HashSet(StringComparer.Ordinal); + var unverified = new HashSet(StringComparer.Ordinal); var attempted = new List<(ConfigurationFileChange Change, string? Backup)>(); var staged = new List<(ConfigurationFileChange Change, string Stage, string? Backup)>(); var ownedDirectories = new Dictionary(StringComparer.Ordinal); @@ -139,6 +144,78 @@ bool StagesMatch() => staged.All(s => (attempted.Any(a => a.Change.Path == s.Change.Path) || host.Observe(s.Stage) == new ConfigurationObservation(s.Stage, ConfigurationObservationKind.File, CandidateObservation(s.Change).Identity)) && (s.Backup is null || host.Observe(s.Backup) == new ConfigurationObservation(s.Backup, ConfigurationObservationKind.File, s.Change.Expected.Identity))); + static ConfigurationArtifactKind RoleKind(string role) => role.StartsWith("journal", StringComparison.Ordinal) + ? ConfigurationArtifactKind.Journal : role == "backup" ? ConfigurationArtifactKind.Backup : ConfigurationArtifactKind.Stage; + void Unverified(string path) + { + path ??= "[null artifact path]"; + unverified.Add(path); + unresolved.Add(path); + } + string StageOwned(string destination, ReadOnlyMemory content, string role) + { + var expected = GetArtifactPath(destination, transactionId, role); + if (host.Observe(expected).Kind != ConfigurationObservationKind.Missing) + { Unverified(expected); throw new OwnershipException(); } + var kind = RoleKind(role); + stageAttempts[expected] = kind; + var returned = host.Stage(destination, content, transactionId, role); + if (returned != expected) + { + Unverified(returned); + if (host.Observe(expected).Kind != ConfigurationObservationKind.Missing) Unverified(expected); + throw new OwnershipException(); + } + var observed = host.Observe(expected); + var identity = NativeConfigurationTransactionHost.Hash(content.Span); + if (observed != new ConfigurationObservation(expected, ConfigurationObservationKind.File, identity)) + { Unverified(expected); throw new OwnershipException(); } + artifacts.Add(new(expected, kind, transactionId)); + artifactEvidence[expected] = observed; + return expected; + } + void RegisterDirectory(string path) + { + if (!plannedDirectories.Contains(path) || host.Observe(path).Kind != ConfigurationObservationKind.Directory) + { Unverified(path); throw new OwnershipException(); } + if (!artifacts.Any(a => a.Path == path)) artifacts.Add(new(path, ConfigurationArtifactKind.Directory, transactionId)); + } + void ReportArtifacts(ConfigurationArtifactFailureException failure) + { + foreach (var artifact in failure.Artifacts) + { + if (artifact.TransactionId != transactionId) + { Unverified(artifact.Path); continue; } + if (artifact.Kind == ConfigurationArtifactKind.Directory && plannedDirectories.Contains(artifact.Path) + && host.Observe(artifact.Path).Kind == ConfigurationObservationKind.Directory) + { RegisterDirectory(artifact.Path); continue; } + if (stageAttempts.TryGetValue(artifact.Path, out var kind) && kind == artifact.Kind + && host.Observe(artifact.Path) is { Kind: ConfigurationObservationKind.File } observed) + { + if (!artifacts.Any(a => a.Path == artifact.Path)) + { + artifacts.Add(new(artifact.Path, kind, transactionId)); + artifactEvidence[artifact.Path] = observed; + partialArtifacts.Add(artifact.Path); + } + continue; + } + Unverified(artifact.Path); + } + } + void RecordRecoveryEvidence() + { + if (document is null) return; + document = document with + { + Committed = false, + Artifacts = artifacts.Where(a => !unverified.Contains(a.Path)).ToImmutableArray(), + UnverifiedPaths = unverified.Order(StringComparer.Ordinal).ToImmutableArray() + }; + try { StageOwned(document.MetadataDestination, document.Encode(), "journal-recovery"); } + catch (ConfigurationArtifactFailureException failure) { ReportArtifacts(failure); } + catch (Exception journalError) when (IsIo(journalError)) { } + } IDisposable? coordination = null; try { @@ -157,7 +234,6 @@ bool StagesMatch() => staged.All(s => var changes = plan.Changes.Where(c => c.Expected.Kind != ConfigurationObservationKind.File || !host.ReadFile(c.Path).AsSpan().SequenceEqual(c.Content.AsSpan())).ToArray(); if (changes.Length == 0) return new(ConfigurationTransactionStatus.Unchanged, "config.edit_unchanged", targets, []); - var plannedDirectories = new HashSet(StringComparer.Ordinal); foreach (var change in changes) { for (var parent = ProjectPath.parent(change.Path); ; parent = ProjectPath.parent(parent)) @@ -187,20 +263,21 @@ bool StagesMatch() => staged.All(s => .Concat(guards.Select(g => new ConfigurationArtifact(RecoveryDocument.ArtifactPath(ProjectPath.join(g, ".morphir-transaction"), transactionId, "journal"), ConfigurationArtifactKind.Journal, transactionId))) .Concat(artifacts).ToImmutableArray(); document = new(transactionId, false, guards, entries, plannedArtifacts) - { PendingDirectories = plannedDirectories.Order(StringComparer.Ordinal).ToImmutableArray() }; + { + PendingDirectories = plannedDirectories.Order(StringComparer.Ordinal).ToImmutableArray(), + PlannedDirectories = plannedDirectories.Order(StringComparer.Ordinal).ToImmutableArray() + }; // Write-ahead metadata is durable before any candidate or original backup is staged. // The common target ancestor is first, so even an interrupted setup blocks its descendants. foreach (var directory in guards.OrderBy(g => g == common ? 0 : 1).ThenBy(g => g, StringComparer.Ordinal)) { if (host.Observe(directory).Kind == ConfigurationObservationKind.Missing) continue; - var journal = host.Stage(ProjectPath.join(directory, ".morphir-transaction"), document.Encode(), transactionId, "journal"); - artifacts.Add(new(journal, ConfigurationArtifactKind.Journal, transactionId)); + StageOwned(ProjectPath.join(directory, ".morphir-transaction"), document.Encode(), "journal"); } foreach (var change in changes) { var created = host.CreateParentDirectories(change.Path, transactionId); - foreach (var directory in created) - artifacts.Add(new(directory, ConfigurationArtifactKind.Directory, transactionId)); + foreach (var directory in created) RegisterDirectory(directory); foreach (var directory in created) ownedDirectories[directory] = host.Observe(directory, artifacts.Select(a => a.Path).ToArray()); } @@ -211,31 +288,30 @@ bool StagesMatch() => staged.All(s => && host.Observe(p).Kind != ConfigurationObservationKind.Missing).Order(StringComparer.Ordinal).ToImmutableArray() }; if (document.PendingDirectories.Length > 0) throw new IOException("Directory creation ownership is uncertain"); - var directoryReceipt = host.Stage(document.MetadataDestination, document.Encode(), transactionId, "journal-directories"); - artifacts.Add(new(directoryReceipt, ConfigurationArtifactKind.Journal, transactionId)); + StageOwned(document.MetadataDestination, document.Encode(), "journal-directories"); foreach (var directory in guards.Where(g => !artifacts.Any(a => a.Kind == ConfigurationArtifactKind.Journal && ProjectPath.parent(a.Path) == g))) { - var journal = host.Stage(ProjectPath.join(directory, ".morphir-transaction"), document.Encode(), transactionId, "journal"); - artifacts.Add(new(journal, ConfigurationArtifactKind.Journal, transactionId)); + StageOwned(ProjectPath.join(directory, ".morphir-transaction"), document.Encode(), "journal"); } foreach (var change in changes) { fault?.Invoke(ConfigurationTransactionPhase.BeforeStage, change.Path); - var stage = host.Stage(change.Path, change.Content.AsMemory(), transactionId, "stage"); - artifacts.Add(new(stage, ConfigurationArtifactKind.Stage, transactionId)); + var stage = StageOwned(change.Path, change.Content.AsMemory(), "stage"); string? backup = null; if (change.Expected.Kind == ConfigurationObservationKind.File) { - backup = host.Stage(change.Path, host.ReadFile(change.Path), transactionId, "backup"); - artifacts.Add(new(backup, ConfigurationArtifactKind.Backup, transactionId)); + backup = StageOwned(change.Path, host.ReadFile(change.Path), "backup"); } staged.Add((change, stage, backup)); } // A ready snapshot records actual artifact paths for non-native storage adapters. - document = document with { Entries = staged.Select(s => new RecoveryEntry(s.Change.Path, s.Change.Expected.Kind, - s.Change.Expected.Identity, CandidateObservation(s.Change).Identity, s.Stage, s.Backup)).ToImmutableArray(), Artifacts = artifacts.ToImmutableArray() }; - var ready = host.Stage(document.MetadataDestination, document.Encode(), transactionId, "journal-ready"); - artifacts.Add(new(ready, ConfigurationArtifactKind.Journal, transactionId)); + document = document with + { + Entries = staged.Select(s => new RecoveryEntry(s.Change.Path, s.Change.Expected.Kind, + s.Change.Expected.Identity, CandidateObservation(s.Change).Identity, s.Stage, s.Backup)).ToImmutableArray(), + Artifacts = artifacts.ToImmutableArray() + }; + StageOwned(document.MetadataDestination, document.Encode(), "journal-ready"); if (!StagesMatch() || !ObservationsMatch()) throw new ConflictException(); foreach (var (change, stage, backup) in staged) { @@ -247,21 +323,17 @@ bool StagesMatch() => staged.All(s => } if (!StagesMatch() || !ObservationsMatch()) throw new ConflictException(); document = document with { Committed = true, Artifacts = artifacts.ToImmutableArray() }; - var committed = host.Stage(document.MetadataDestination, document.Encode(), transactionId, "journal-committed"); - artifacts.Add(new(committed, ConfigurationArtifactKind.Journal, transactionId)); - if (!TryCleanup(artifacts, true, document)) + StageOwned(document.MetadataDestination, document.Encode(), "journal-committed"); + if (!TryCleanup(artifacts, true, document, artifactEvidence, partialArtifacts)) return new(ConfigurationTransactionStatus.RecoveryRequired, "config.edit_cleanup_required", targets, Retained(artifacts)) { UnresolvedPaths = targets }; return new(ConfigurationTransactionStatus.Written, "config.edit_written", targets, []); } catch (Exception error) when (IsIo(error)) { - var foreignArtifacts = ImmutableArray.Empty; - if (error is ConfigurationArtifactFailureException artifactFailure) - { - artifacts.AddRange(artifactFailure.Artifacts.Where(a => a.TransactionId == transactionId)); - foreignArtifacts = artifactFailure.Artifacts.Where(a => a.TransactionId != transactionId).ToImmutableArray(); - } + if (error is ConfigurationArtifactFailureException artifactFailure) ReportArtifacts(artifactFailure); + foreach (var path in stageAttempts.Keys.Where(p => !artifacts.Any(a => a.Path == p))) + if (host.Observe(path).Kind != ConfigurationObservationKind.Missing) Unverified(path); if (document is not null) foreach (var directory in document.PendingDirectories) if (!artifacts.Any(a => a.Kind == ConfigurationArtifactKind.Directory && a.Path == directory) @@ -290,26 +362,34 @@ bool StagesMatch() => staged.All(s => catch (Exception observationError) when (IsIo(observationError)) { unresolved.Add(change.Path); } } } - if (unresolved.Count > 0 || foreignArtifacts.Length > 0) + if (unverified.Count > 0) RecordRecoveryEvidence(); + if (unresolved.Count > 0) return new(ConfigurationTransactionStatus.RecoveryRequired, "config.edit_recovery_required", targets, - Retained(artifacts).AddRange(foreignArtifacts)) { RestoredPaths = restored.ToImmutableArray(), UnresolvedPaths = unresolved.ToImmutableArray() }; - if (!TryCleanup(artifacts, false, document)) + Retained(artifacts)) + { RestoredPaths = restored.ToImmutableArray(), UnresolvedPaths = unresolved.Distinct().ToImmutableArray() }; + if (!TryCleanup(artifacts, false, document, artifactEvidence, partialArtifacts)) return new(ConfigurationTransactionStatus.RecoveryRequired, "config.edit_cleanup_required", targets, Retained(artifacts)) { RestoredPaths = restored.ToImmutableArray(), UnresolvedPaths = targets }; return new(error is ConflictException ? ConfigurationTransactionStatus.Conflict : attempted.Count > 0 ? ConfigurationTransactionStatus.RolledBack : ConfigurationTransactionStatus.IoFailure, error is ConflictException ? "config.edit_conflict" - : attempted.Count > 0 ? "config.edit_rolled_back" : "config.edit_io", targets, []) { RestoredPaths = restored.ToImmutableArray() }; + : attempted.Count > 0 ? "config.edit_rolled_back" : "config.edit_io", targets, []) + { RestoredPaths = restored.ToImmutableArray() }; } finally { coordination?.Dispose(); } } - private bool TryCleanup(List artifacts, bool committed, RecoveryDocument? document) + private bool TryCleanup(List artifacts, bool committed, RecoveryDocument? document, + IReadOnlyDictionary? evidence = null, IReadOnlySet? partialArtifacts = null, + IReadOnlySet? preservedDirectories = null) { if (artifacts.Count == 0) return true; try { fault?.Invoke(ConfigurationTransactionPhase.BeforeCleanup, artifacts[0].Path); + var cleanupDirectories = artifacts.Where(a => a.Kind == ConfigurationArtifactKind.Directory).Select(a => a.Path) + .Concat(preservedDirectories ?? new HashSet()).ToHashSet(StringComparer.Ordinal); + if (CleanupUnresolved(artifacts, document, evidence, partialArtifacts, cleanupDirectories).Length > 0) return false; // Keep metadata outside newly created directories until directory cleanup is proven. // A committed witness remains last, above all destinations, throughout cleanup. var directories = artifacts.Where(a => a.Kind == ConfigurationArtifactKind.Directory).Select(a => a.Path).ToHashSet(StringComparer.Ordinal); @@ -319,6 +399,7 @@ int Order(ConfigurationArtifact artifact) => artifact.Kind != ConfigurationArtif : artifact.Path.EndsWith("-journal-committed", StringComparison.Ordinal) ? 4 : 3; foreach (var artifact in artifacts.OrderBy(Order).ThenByDescending(a => a.Path.Length)) { + if (CleanupUnresolved([artifact], document, evidence, partialArtifacts, cleanupDirectories).Length > 0) return false; if (artifact.Kind == ConfigurationArtifactKind.Directory) { host.DeleteDirectory(artifact.Path); @@ -332,6 +413,24 @@ int Order(ConfigurationArtifact artifact) => artifact.Kind != ConfigurationArtif catch (Exception error) when (IsIo(error)) { return false; } } + private ImmutableArray CleanupUnresolved(IEnumerable artifacts, RecoveryDocument? document, + IReadOnlyDictionary? evidence, IReadOnlySet? partialArtifacts, + IReadOnlySet? preservedDirectories) + { + var list = artifacts.ToArray(); + var unresolved = new HashSet(StringComparer.Ordinal); + foreach (var artifact in list.Where(a => a.Kind != ConfigurationArtifactKind.Directory)) + { + var observed = host.Observe(artifact.Path); + if (observed.Kind == ConfigurationObservationKind.Missing) continue; + if (evidence is not null && (!evidence.TryGetValue(artifact.Path, out var expected) || expected != observed)) + unresolved.Add(artifact.Path); + } + if (document is not null) + unresolved.UnionWith(RecoveryUnresolved(document, list.Where(a => partialArtifacts?.Contains(a.Path) != true), preservedDirectories, ignoreUnverified: true)); + return unresolved.Order(StringComparer.Ordinal).ToImmutableArray(); + } + private ImmutableArray Retained(IEnumerable artifacts) => artifacts.Distinct().Where(a => { @@ -342,6 +441,7 @@ private ImmutableArray Retained(IEnumerable new(change.Path, ConfigurationObservationKind.File, NativeConfigurationTransactionHost.Hash(change.Content.AsSpan())); private sealed class ConflictException : IOException; + private sealed class OwnershipException : IOException; private static bool IsIo(Exception error) => error is IOException or UnauthorizedAccessException or NotSupportedException; } @@ -476,7 +576,7 @@ public string Stage(string destination, ReadOnlyMemory content, string tra try { File.Delete(path); } catch (Exception cleanupError) when (cleanupError is IOException or UnauthorizedAccessException or NotSupportedException) { - var kind = role == "backup" ? ConfigurationArtifactKind.Backup : role == "journal" ? ConfigurationArtifactKind.Journal : ConfigurationArtifactKind.Stage; + var kind = role == "backup" ? ConfigurationArtifactKind.Backup : role.StartsWith("journal", StringComparison.Ordinal) ? ConfigurationArtifactKind.Journal : ConfigurationArtifactKind.Stage; throw new ConfigurationArtifactFailureException([new(Canonical(path), kind, transactionId)], error); } throw; diff --git a/src/Morphir.Configuration.IO/ProjectMigrationService.cs b/src/Morphir.Configuration.IO/ProjectMigrationService.cs index 919ce19b..c05f57f3 100644 --- a/src/Morphir.Configuration.IO/ProjectMigrationService.cs +++ b/src/Morphir.Configuration.IO/ProjectMigrationService.cs @@ -37,11 +37,14 @@ internal ProjectMigrationPreview(object owner, string? source, string? target, s /// Explicit proposed base workspace shared by aggregate conversion hosts. /// Contains only declared base primaries; no ambient overlays are loaded. -public sealed class ProjectMigrationContext(string rootPath, FSharpOption? workspace, IEnumerable rootSources) +public sealed class ProjectMigrationContext(string rootPath, FSharpOption? workspace, IEnumerable rootSources, + IEnumerable? proposedMembers = null, SourceInput? originalTarget = null) { public string RootPath { get; } = rootPath; public FSharpOption? Workspace { get; } = workspace; public ImmutableArray RootSources { get; } = rootSources.ToImmutableArray(); + public ImmutableArray ProposedMembers { get; } = proposedMembers?.ToImmutableArray() ?? default; + public SourceInput? OriginalTarget { get; } = originalTarget; } /// Plans one classic project over an explicitly bounded tree. A storage adapter is optional @@ -51,6 +54,8 @@ public sealed class ProjectMigrationService(IProjectTree tree, IConfigurationTra public ProjectMigrationPreview Plan(string startPath, string? inputPath = null, string? outputPath = null) => PlanCore(startPath, inputPath, outputPath, null); + /// Read-only component composition for an aggregate. Apply the aggregate through + /// WorkspaceMigrationService; hypothetical base contexts never grant individual write capability. public ProjectMigrationPreview PlanInWorkspace(string startPath, ProjectMigrationContext context, string? inputPath = null, string? outputPath = null) => PlanCore(startPath, inputPath, outputPath, context); @@ -115,6 +120,13 @@ private ProjectMigrationPreview PlanCore(string startPath, string? inputPath, st || context.RootSources.Any(s => s.Source.Role != SourceRole.RootPrimary || !DiscoveryPolicy.candidates(context.RootPath).Contains(s.Source.Path)))) return Fail("migration.context", "Proposed context must belong to the discovered workspace base primaries."); + if (context is { ProposedMembers.IsDefault: false }) + { + var declared = ConfigTree.tryFind(ReaderValues.List(new[] { "workspace", "members" }), found.Discovery.Root.Values)?.Value; + var memberPaths = declared is ConfigValue.Array declaredArray ? declaredArray.Item.OfType().Select(s => s.Item) : []; + if (!context.ProposedMembers.SequenceEqual(memberPaths)) + return Fail("migration.context", "Proposed members must match the declared candidate workspace."); + } var sources = context is null ? found.Discovery.Root.Sources.ToList() : context.RootSources.ToList(); if (role == SourceRole.MemberPrimary) sources.AddRange(project.Location.Sources.Select(s => Rebind(s, role))); @@ -142,6 +154,28 @@ private ProjectMigrationPreview PlanCore(string startPath, string? inputPath, st return Fail("migration.output", "Output is not a writable primary file location."); var targetSource = targetEntry is ProjectEntry.File file ? new SourceInput(target, file.Item, false) : new SourceInput(target, SourceContent.Missing, false); + if (context?.OriginalTarget is { } original && role == SourceRole.RootPrimary) + { + if (original.Source.Path != targetPath || original.Source.Format != DocumentFormat.Toml || original.Source.Role != SourceRole.RootPrimary) + return Fail("migration.context", "Original target must identify this root TOML primary."); + if (storage is not null) + { + var expected = evidence[targetPath]; + if (expected.Kind == ConfigurationObservationKind.Missing && !original.Content.IsMissing + || expected.Kind == ConfigurationObservationKind.File && original.Content is not SourceContent.Available) + return Fail("migration.context", "Original target does not match observed target bytes."); + if (expected.Kind == ConfigurationObservationKind.File) + { + var bytes = storage.ReadFile(targetPath); + if (storage.Observe(targetPath) != expected) return Fail("migration.stale", "Target changed while planning."); + var actual = ConfigurationReader.ReadText(target, TextBuffer.Decode(bytes).Text); + if (actual.Content is not SourceContent.Available actualDocument || original.Content is not SourceContent.Available originalDocument + || !ProjectMigration.same(actualDocument.Item.Raw, originalDocument.Item.Raw)) + return Fail("migration.context", "Original target does not match observed target bytes."); + } + } + targetSource = new SourceInput(target, original.Content, false); + } if (targetSource.Content is SourceContent.Invalid invalid) { diagnostics.AddRange(invalid.Item); @@ -154,6 +188,20 @@ private ProjectMigrationPreview PlanCore(string startPath, string? inputPath, st plan = ProjectMigration.planWithContext(classic, target, raw, baseInput); diagnostics.AddRange(plan.Diagnostics); if (plan.Blocked) return new(this, sourcePath, targetPath, projectPath, plan, null, diagnostics); + if (role == SourceRole.RootPrimary && context is { ProposedMembers.IsDefault: false }) + { + // Membership adoption is an explicit composition step after the unchanged loss/conflict mapper. + // Only this declared field can differ; independent verification still checks every value. + var key = ReaderValues.List(new[] { "workspace", "members" }); + var value = ConfigValue.NewArray(ReaderValues.List(context.ProposedMembers.Select(ConfigValue.NewString))); + var currentMembers = ConfigTree.tryFind(key, plan.Candidate)?.Value; + if (currentMembers is null || !ProjectMigration.same(currentMembers, value)) + { + var composedEdits = plan.Edits.Where(e => !e.KeyPath.SequenceEqual(key)).Append(new MigrationEdit(key, value)); + plan = new MigrationPlan(plan.Source, plan.Target, ConfigTree.set(key, value, plan.Candidate), ReaderValues.List(composedEdits), + plan.Ledger, plan.Conflicts, plan.Diagnostics, plan.Blocked); + } + } TextBuffer buffer; if (storage is not null) { @@ -191,7 +239,7 @@ private ProjectMigrationPreview PlanCore(string startPath, string? inputPath, st if (!verification.Verified) return Fail("migration.verification", "Candidate changes project values or resolved path meaning."); var changed = plan.Edits.Any(); ConfigurationTransactionPlan? transaction = null; - if (storage is not null) + if (storage is not null && context is null) { // Directory and missing-primary evidence captures topology drift in addition to exact source bytes. transaction = new ConfigurationTransactionPlan(evidence.Values, diff --git a/src/Morphir.Configuration.IO/WorkspaceMigrationService.cs b/src/Morphir.Configuration.IO/WorkspaceMigrationService.cs index 6d33c319..a4b96fac 100644 --- a/src/Morphir.Configuration.IO/WorkspaceMigrationService.cs +++ b/src/Morphir.Configuration.IO/WorkspaceMigrationService.cs @@ -151,7 +151,7 @@ WorkspaceMigrationPreview Result(bool verified = false, bool changed = false, Co // Existing declarations are preserved verbatim. Only append explicit paths not already represented. var originalMembers = Strings(rootLocation.Values, "workspace", "members").ToArray(); members.AddRange(originalMembers); - var bounded = new InventoryTree(observed, root, availableDirectories); + var bounded = new MembershipTree(observed, root, availableDirectories, outputDirectory); var existingDiscovery = declared ? ProjectSystem.Discover(bounded, root).Discovery : null; if (existingDiscovery is not null) { @@ -162,7 +162,8 @@ WorkspaceMigrationPreview Result(bool verified = false, bool changed = false, Co if (!ProjectPath.contains(root, old.Path) || inventory.Any(i => i.Kind == WorkspaceInventoryKind.Boundary && ProjectPath.contains(i.Path, old.Path))) Error(old.Path, "workspace.migrate_membership", "Existing membership crosses the operation or nested-workspace boundary."); if (!inventory.Any(i => i.Path == old.Path)) - inventory.Add(new(old.Path, ProjectPath.relative(root, old.Path), WorkspaceInventoryKind.AlreadyMigrated, "Existing declared member retained.", old.Name?.Value)); + inventory.Add(new(old.Path, ProjectPath.relative(root, old.Path), Classics(old.Location).Any() ? WorkspaceInventoryKind.Excluded : WorkspaceInventoryKind.AlreadyMigrated, + "Existing declared member retained without conversion.", old.Name?.Value)); } } var existingPaths = existingDiscovery?.Projects.Select(p => p.Path).ToHashSet(StringComparer.Ordinal) ?? []; @@ -194,12 +195,30 @@ WorkspaceMigrationPreview Result(bool verified = false, bool changed = false, Co workspaceText = rendered.Text; } var overlay = new CandidateTree(bounded, root); - overlay.Set(target, workspaceText); + if (edits.Count > 0) overlay.Set(target, workspaceText); // Convert root first so all members resolve against its final proposed canonical base. if (locations.TryGetValue(root, out var selectedRoot) && Classics(selectedRoot).Any()) { var service = new ProjectMigrationService(overlay); - var rootPreview = service.PlanInWorkspace(root, Context(overlay, root, Classics(selectedRoot).Select(s => s.Source.Path)), Classics(selectedRoot).Single().Source.Path, target); + var originalTarget = rootTomls.FirstOrDefault() ?? new SourceInput(new SourceIdentity(target, DocumentFormat.Toml, SourceRole.RootPrimary, 4, null), SourceContent.Missing, false); + var rootContext = Context(overlay, root, Classics(selectedRoot).Select(s => s.Source.Path), + edits.Count > 0 ? members : null, originalTarget); + var projected = rootContext.RootSources.FirstOrDefault(s => s.Source.Path == target)?.Content is SourceContent.Available projectedDocument + ? projectedDocument.Item.Raw : DocumentDefaults.emptyValue; + // Carry declared ownership controls without retaining the converted classic as a base layer. + // All remaining root conversion values still come through the unchanged loss/equivalence mapper. + foreach (var control in new[] { "root", "out_dir" }) + { + var key = ReaderValues.List(new[] { "workspace", control }); + var value = ConfigTree.tryFind(key, rootLocation.Values); + if (value is not null) projected = ConfigTree.set(key, value.Value, projected); + } + var projectedIdentity = new SourceIdentity(target, DocumentFormat.Toml, SourceRole.RootPrimary, 4, null); + var projectedInput = new SourceInput(projectedIdentity, SourceContent.NewAvailable(Normalization.normalize(projectedIdentity, projected, + MapModule.Empty, SourcePosition>())), false); + rootContext = new ProjectMigrationContext(root, rootContext.Workspace, + rootContext.RootSources.Where(s => s.Source.Path != target).Append(projectedInput), edits.Count > 0 ? members : null, originalTarget); + var rootPreview = service.PlanInWorkspace(root, rootContext, Classics(selectedRoot).Single().Source.Path, target); projects.Add(rootPreview); diagnostics.AddRange(rootPreview.Diagnostics); if (rootPreview.CanApply) { @@ -230,6 +249,8 @@ WorkspaceMigrationPreview Result(bool verified = false, bool changed = false, Co if (index >= 0) inventory[index] = inventory[index] with { Kind = WorkspaceInventoryKind.AlreadyMigrated, Reason = "Equivalent migrated project retained without rewriting." }; } if (projects.Any(p => !p.CanApply) || diagnostics.Any(d => d.Severity == DiagnosticSeverity.Error)) return Result(); + if (projects.Count > 1 && projects.Any(p => !Equals(p.Verification!.BaseWorkspace, projects[0].Verification!.BaseWorkspace))) + { Error(root, "workspace.migrate_context", "Conversions do not share the same proposed base workspace context."); return Result(); } if (!declared) { // Adoption may change topology/output roots, but never unrelated values. @@ -245,10 +266,15 @@ WorkspaceMigrationPreview Result(bool verified = false, bool changed = false, Co Error(path, "workspace.migrate_adoption_value", "Adoption changes a non-adoption setting; review the project and proposed shared base settings."); } } + // Check the actual retained-source view as well as independent conversion discovery. + var retained = ProjectSystem.Discover(overlay, root).Discovery; // Remove every converted classic input together. Rerun actual discovery/selection over the whole candidate. foreach (var project in projects) overlay.Hide(project.SourcePath!); var candidate = ProjectSystem.Discover(overlay, root).Discovery; diagnostics.AddRange(candidate.Diagnostics); + if (retained.Location != candidate.Location || !Equals(retained.Workspace, candidate.Workspace) + || !retained.Projects.Select(p => (p.Path, p.Name?.Value, p.IsValid)).SequenceEqual(candidate.Projects.Select(p => (p.Path, p.Name?.Value, p.IsValid)))) + Error(root, "workspace.migrate_verification", "Retained-source discovery differs from the independently converted workspace."); if (candidate.Location != root || candidate.Workspace is null || candidate.Projects.Any(p => !p.IsValid)) Error(root, "workspace.migrate_verification", "Proposed workspace discovery is invalid."); var expectedPaths = locations.Keys.Concat(existingPaths).Concat(rootProject ? new[] { root } : []).Where(p => p == root || ProjectPath.contains(workspaceBase, p)).Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal).ToArray(); @@ -262,6 +288,10 @@ WorkspaceMigrationPreview Result(bool verified = false, bool changed = false, Co } var beforeSelection = existingDiscovery is null ? null : DiscoveryPolicy.select(existingDiscovery, null); var afterSelection = DiscoveryPolicy.select(candidate, null); + var retainedSelection = DiscoveryPolicy.select(retained, null); + if (!retainedSelection.Projects.Select(p => p.Path).SequenceEqual(afterSelection.Projects.Select(p => p.Path)) + || !retainedSelection.Diagnostics.Select(d => d.Code).SequenceEqual(afterSelection.Diagnostics.Select(d => d.Code))) + Error(root, "workspace.migrate_selection", "Retained-source default selection differs from the independently converted workspace."); if (Has(rootLocation.Values, "workspace", "default_member") && afterSelection.Diagnostics.Any(d => d.Severity == DiagnosticSeverity.Error)) Error(root, "workspace.migrate_default", "The declared default member is not a valid unambiguous project in the proposed workspace."); var beforeSelected = beforeSelection is null ? "" : string.Join(",", beforeSelection.Projects.Select(p => p.Path)); @@ -298,6 +328,10 @@ WorkspaceMigrationPreview Result(bool verified = false, bool changed = false, Co var transaction = storage is null ? null : new ConfigurationTransactionPlan(observed.Evidence.Values, changes); return Result(verified, changed, transaction); } + catch (MembershipBoundaryException boundary) + { Error(boundary.BoundaryPath, boundary.Code, boundary.Message); return Result(); } + catch (RegexMatchTimeoutException) + { Error(root, "workspace.migrate_filter_limit", "Filter evaluation exceeded its bounded matching time. Simplify the include/exclude patterns."); return Result(); } catch (ObservationConflictException) { Error(root, "workspace.migrate_stale", "Inventory or source changed while planning."); return Result(); } catch (Exception error) when (error is IOException or UnauthorizedAccessException or ArgumentException or NotSupportedException or InvalidOperationException) { Error(root, "workspace.migrate_io", "Unable to inventory, parse or compose workspace migration. Check paths and permissions."); return Result(); } @@ -326,11 +360,12 @@ private static ProjectLocation Inspect(IProjectTree tree, string path) var entries = DiscoveryPolicy.candidates(path).Select(p => Tuple.Create(p, tree.Observe(p))); return DiscoveryPolicy.inspectLocation(path, new ProjectSnapshot(MapModule.OfSeq(entries))); } - private static ProjectMigrationContext Context(IProjectTree tree, string root, IEnumerable convertedSources) + private static ProjectMigrationContext Context(IProjectTree tree, string root, IEnumerable convertedSources, + IEnumerable? proposedMembers = null, SourceInput? originalTarget = null) { var found = ProjectSystem.Discover(tree, root).Discovery; var converted = convertedSources.ToHashSet(StringComparer.Ordinal); - return new(root, found.Workspace, found.Root.Sources.Where(s => !converted.Contains(s.Source.Path))); + return new(root, found.Workspace, found.Root.Sources.Where(s => !converted.Contains(s.Source.Path)), proposedMembers, originalTarget); } private static IEnumerable Classics(ProjectLocation location) => location.Sources.Where(s => s.Content is SourceContent.Available a @@ -343,21 +378,21 @@ private static IEnumerable Strings(ConfigValue value, params string[] pa private static ConfigValue ArrayValue(IEnumerable values) => ConfigValue.NewArray(ReaderValues.List(values.Select(ConfigValue.NewString))); private static bool Skipped(string name) => name is ".git" or ".hg" or ".svn" or "node_modules" or "elm-stuff" or "bin" or "obj" or "dist" or "build" or "target" or "vendor" or ".nuget" or "packages"; private static bool ValidPattern(string pattern) => !string.IsNullOrWhiteSpace(pattern) && !ProjectPath.isAbsolute(pattern) - && !pattern.Replace('\\','/').Split('/').Contains("..", StringComparer.Ordinal) && !pattern.Contains(':'); + && !pattern.Replace('\\', '/').Split('/').Contains("..", StringComparer.Ordinal) && !pattern.Contains(':'); internal static bool Glob(string pattern, string relative) { - var parts = pattern.Replace('\\','/').Split('/', StringSplitOptions.RemoveEmptyEntries).Where(s => s != ".").ToArray(); + var parts = pattern.Replace('\\', '/').Split('/', StringSplitOptions.RemoveEmptyEntries).Where(s => s != ".").ToArray(); var path = relative.Split('/', StringSplitOptions.RemoveEmptyEntries).Where(s => s != ".").ToArray(); - var memo = new Dictionary<(int,int),bool>(); - bool Match(int p,int v) + var memo = new Dictionary<(int, int), bool>(); + bool Match(int p, int v) { - if (memo.TryGetValue((p,v),out var found)) return found; + if (memo.TryGetValue((p, v), out var found)) return found; var result = p == parts.Length ? v == path.Length : parts[p] == "**" - ? Match(p+1,v) || v < path.Length && Match(p,v+1) - : v < path.Length && Regex.IsMatch(path[v], "\\A" + Regex.Escape(parts[p]).Replace("\\*", ".*").Replace("\\?", ".") + "\\z", RegexOptions.CultureInvariant, TimeSpan.FromSeconds(1)) && Match(p+1,v+1); - memo[(p,v)] = result; return result; + ? Match(p + 1, v) || v < path.Length && Match(p, v + 1) + : v < path.Length && Regex.IsMatch(path[v], "\\A" + Regex.Escape(parts[p]).Replace("\\*", ".*").Replace("\\?", ".") + "\\z", RegexOptions.CultureInvariant, TimeSpan.FromSeconds(1)) && Match(p + 1, v + 1); + memo[(p, v)] = result; return result; } - return Match(0,0); + return Match(0, 0); } private static bool Excluded(string relative, IEnumerable patterns) => patterns.Any(p => Glob(p, relative)); private static ConfigDiagnostic Diagnostic(string path, string code, string message) => new(code, message, DiagnosticSeverity.Error, @@ -384,7 +419,7 @@ private static TextBuffer BufferFor(ObservedTree tree, string path, ConfigValue } var text = ""; if (raw is ConfigValue.Object fields) - foreach (var field in fields.Item) text = Render(text, [new(ReaderValues.List(new[] { field.Key }),field.Value)]); + foreach (var field in fields.Item) text = Render(text, [new(ReaderValues.List(new[] { field.Key }), field.Value)]); return TextBuffer.Decode(Encoding.UTF8.GetBytes(text)); } private sealed class ObservationConflictException : IOException; @@ -392,66 +427,97 @@ private sealed class ObservedTree(IProjectTree inner, string root, IConfiguratio { public string Boundary => root; public IConfigurationTransactionHost? Storage => storage; - public Dictionary Evidence { get; } = new(StringComparer.Ordinal); + public Dictionary Evidence { get; } = new(StringComparer.Ordinal); private readonly Dictionary entries = new(StringComparer.Ordinal); public ProjectEntry Observe(string path) { - if (!ProjectPath.contains(root,path)) return ProjectEntry.NewFailure("Outside operation boundary"); + if (!ProjectPath.contains(root, path)) return ProjectEntry.NewFailure("Outside operation boundary"); var before = storage?.Observe(path); - if (before is not null && Evidence.TryGetValue(path,out var prior) && prior != before) throw new ObservationConflictException(); - var entry=inner.Observe(path); + if (before is not null && Evidence.TryGetValue(path, out var prior) && prior != before) throw new ObservationConflictException(); + var entry = inner.Observe(path); if (entry is ProjectEntry.File && before?.Kind == ConfigurationObservationKind.File) { - var bytes=storage!.ReadFile(path); - if (NativeConfigurationTransactionHost.Hash(bytes)!=before.Identity) throw new ObservationConflictException(); - entry=ProjectEntry.NewFile(ConfigurationReader.ReadText(path,TextBuffer.Decode(bytes).Text).Content); + var bytes = storage!.ReadFile(path); + if (NativeConfigurationTransactionHost.Hash(bytes) != before.Identity) throw new ObservationConflictException(); + entry = ProjectEntry.NewFile(ConfigurationReader.ReadText(path, TextBuffer.Decode(bytes).Text).Content); } if (storage is not null) { - if (storage.Observe(path)!=before) throw new ObservationConflictException(); - Evidence[path]=before!; + if (storage.Observe(path) != before) throw new ObservationConflictException(); + Evidence[path] = before!; } - if (entries.TryGetValue(path,out var old) && !old.Equals(entry)) throw new ObservationConflictException(); - entries[path]=entry; + if (entries.TryGetValue(path, out var old) && !old.Equals(entry)) throw new ObservationConflictException(); + entries[path] = entry; return entry; } } - // Reuse discovery over exactly the inventoried directories. Globs must not reopen - // excluded output trees or nested workspace boundaries during aggregate verification. - private sealed class InventoryTree(IProjectTree inner, string root, HashSet directories) : IProjectTree + // Keep the real shared discovery semantics. Existing ownership may require reading + // filtered members, but a declaration cannot force reads across hard boundaries. + private sealed class MembershipBoundaryException(string path, string code, string message) : IOException(message) + { + public string BoundaryPath { get; } = path; + public string Code { get; } = code; + } + private sealed class MembershipTree(IProjectTree inner, string root, HashSet inventoried, string? outputDirectory) : IProjectTree { public string Boundary => root; - private readonly HashSet primaries = directories.SelectMany(DiscoveryPolicy.candidates).ToHashSet(StringComparer.Ordinal); + private readonly HashSet checkedDirectories = new(inventoried, StringComparer.Ordinal); public ProjectEntry Observe(string path) { - if (!directories.Contains(path) && !primaries.Contains(path)) return ProjectEntry.Missing; - var entry = inner.Observe(path); - return entry is ProjectEntry.Directory directory - ? ProjectEntry.NewDirectory(ReaderValues.List(directory.children.Where(directories.Contains))) : entry; + if (!ProjectPath.contains(root, path)) return inner.Observe(path); + var anchor = path; + // Resolve hidden primary ownership exactly as the single-project input policy does. + for (var directory = ProjectPath.parent(path); ProjectPath.contains(root, directory); directory = ProjectPath.parent(directory)) + { + if (DiscoveryPolicy.candidates(directory).Contains(path)) anchor = directory; + if (directory == root) break; + } + var ancestors = new Stack(); + for (var directory = anchor; directory != root && ProjectPath.contains(root, directory); directory = ProjectPath.parent(directory)) ancestors.Push(directory); + while (ancestors.TryPop(out var directory)) + { + if (checkedDirectories.Contains(directory)) continue; + if (Skipped(Path.GetFileName(directory)) || Path.GetFileName(directory) is ".morphir" or ".config" + || outputDirectory is not null && ProjectPath.contains(outputDirectory, directory)) throw Conflict(directory); + var entry = inner.Observe(directory); + if (entry.IsLink) + { + if (directory != anchor || path != anchor) throw Conflict(directory); + break; + } + if (entry is not ProjectEntry.Directory) break; + if (Has(Inspect(inner, directory).Values, "workspace")) throw Conflict(directory); + if (checkedDirectories.Count >= 100000) + throw new MembershipBoundaryException(directory, "workspace.migrate_inventory_limit", "Inventory directory limit exceeded while verifying existing membership."); + checkedDirectories.Add(directory); + } + return inner.Observe(path); } + private static MembershipBoundaryException Conflict(string path) => new(path, "workspace.migrate_membership_boundary", + "Existing membership reaches a skipped directory or nested declared workspace. Refine its declarations or exclusions before migration."); } - private sealed class CandidateTree(IProjectTree inner,string root) : IProjectTree + private sealed class CandidateTree(IProjectTree inner, string root) : IProjectTree { public string Boundary => root; - public Dictionary Texts { get; }=new(StringComparer.Ordinal); + public Dictionary Texts { get; } = new(StringComparer.Ordinal); private readonly HashSet hidden = new(StringComparer.Ordinal); - public void Set(string path,string text) => Texts[path]=text; + public void Set(string path, string text) => Texts[path] = text; public void Hide(string path) => hidden.Add(path); public ProjectEntry Observe(string path) => hidden.Contains(path) ? ProjectEntry.Missing - : Texts.TryGetValue(path,out var text) ? ProjectEntry.NewFile(ConfigurationReader.ReadText(path,text).Content) : inner.Observe(path); + : Texts.TryGetValue(path, out var text) ? ProjectEntry.NewFile(ConfigurationReader.ReadText(path, text).Content) : inner.Observe(path); } private sealed record TextBuffer(string Text, Encoding Encoding, byte[] Preamble) { - public byte[] Encode(string text) => [.. Preamble,.. Encoding.GetBytes(text)]; + public byte[] Encode(string text) => [.. Preamble, .. Encoding.GetBytes(text)]; public static TextBuffer Decode(byte[] bytes) { - Encoding[] encodings=[new UTF32Encoding(false,true,true),new UTF32Encoding(true,true,true),new UTF8Encoding(true,true),new UnicodeEncoding(false,true,true),new UnicodeEncoding(true,true,true)]; - foreach(var encoding in encodings) + Encoding[] encodings = [new UTF32Encoding(false, true, true), new UTF32Encoding(true, true, true), new UTF8Encoding(true, true), new UnicodeEncoding(false, true, true), new UnicodeEncoding(true, true, true)]; + foreach (var encoding in encodings) { - var preamble=encoding.GetPreamble(); - if(bytes.AsSpan().StartsWith(preamble)) return new(encoding.GetString(bytes.AsSpan(preamble.Length)),encoding,preamble); + var preamble = encoding.GetPreamble(); + if (bytes.AsSpan().StartsWith(preamble)) return new(encoding.GetString(bytes.AsSpan(preamble.Length)), encoding, preamble); } - var utf8=new UTF8Encoding(false,true);return new(utf8.GetString(bytes),utf8,[]); + var utf8 = new UTF8Encoding(false, true); return new(utf8.GetString(bytes), utf8, []); } } } diff --git a/src/Morphir.Configuration/ProjectDiscovery.fs b/src/Morphir.Configuration/ProjectDiscovery.fs index dfbce42c..025a5e18 100644 --- a/src/Morphir.Configuration/ProjectDiscovery.fs +++ b/src/Morphir.Configuration/ProjectDiscovery.fs @@ -192,7 +192,8 @@ module DiscoveryPolicy = /// Inspect only one supplied location using the shared primary/format policy. /// No upward search or membership expansion is performed. - let inspectLocation path snapshot = location SourceRole.RootPrimary path snapshot + let inspectLocation path snapshot = + location SourceRole.RootPrimary path snapshot let private reports role path (snapshot: ProjectSnapshot) = candidates path diff --git a/tests/Morphir.Configuration.IO.Smoke/Morphir.Configuration.IO.Smoke.csproj b/tests/Morphir.Configuration.IO.Smoke/Morphir.Configuration.IO.Smoke.csproj index d965b244..5308253a 100644 --- a/tests/Morphir.Configuration.IO.Smoke/Morphir.Configuration.IO.Smoke.csproj +++ b/tests/Morphir.Configuration.IO.Smoke/Morphir.Configuration.IO.Smoke.csproj @@ -14,5 +14,6 @@ + diff --git a/tests/Morphir.Configuration.IO.Smoke/Program.cs b/tests/Morphir.Configuration.IO.Smoke/Program.cs index f864cd6d..c25b3d82 100644 --- a/tests/Morphir.Configuration.IO.Smoke/Program.cs +++ b/tests/Morphir.Configuration.IO.Smoke/Program.cs @@ -151,16 +151,16 @@ ConfigurationTransactionResult Remember(ConfigurationTransactionResult result) && failedRecovery.Artifacts.Any(a => a.Kind == ConfigurationArtifactKind.Journal), "Incomplete recovery lost typed artifact evidence"); Transaction(File.ReadAllBytes(transactionTarget).AsSpan().SequenceEqual(candidateBytes), "Uncertain outcome falsely claimed original bytes"); var backup = failedRecovery.Artifacts.Single(a => a.Kind == ConfigurationArtifactKind.Backup); - var journal = failedRecovery.Artifacts.Single(a => a.Kind == ConfigurationArtifactKind.Journal); - using (var metadata = JsonDocument.Parse(File.ReadAllBytes(journal.Path))) - Transaction(metadata.RootElement.GetProperty("transactionId").GetString() == backup.TransactionId - && metadata.RootElement.GetProperty("target").GetString() == transactionTarget, "Recovery ownership metadata differs"); + var journal = failedRecovery.Artifacts.First(a => a.Kind == ConfigurationArtifactKind.Journal); + var recoveryService = new ConfigurationTransaction(host); + var metadata = recoveryService.InspectRecovery(journal.Path); + Transaction(metadata.Valid && metadata.TransactionId == backup.TransactionId + && metadata.Targets.Contains(transactionTarget), "Recovery ownership metadata differs"); Transaction(File.ReadAllBytes(backup.Path).AsSpan().SequenceEqual(originalBytes), "Recovery backup lost original bytes"); - // Simulated manual recovery is guarded by the candidate/backup checks above. - host.Install(backup.Path, transactionTarget, true); - foreach (var artifact in failedRecovery.Artifacts) host.DeleteFile(artifact.Path); - Transaction(File.ReadAllBytes(transactionTarget).AsSpan().SequenceEqual(originalBytes) - && failedRecovery.Artifacts.All(a => host.Observe(a.Path).Kind == ConfigurationObservationKind.Missing), "Explicit recovery did not restore bytes and clean owned artifacts"); + var recovered = recoveryService.Recover(journal.Path); + Transaction(recovered.Status == ConfigurationTransactionStatus.RolledBack + && File.ReadAllBytes(transactionTarget).AsSpan().SequenceEqual(originalBytes) + && recovered.Artifacts.IsEmpty, "Explicit recovery did not restore bytes and clean owned artifacts"); recoveryPending = false; var staleTransaction = TransactionPlan(); File.WriteAllText(transactionTarget, "value = 'external'\n"); @@ -374,7 +374,8 @@ ConfigurationTransactionResult Remember(ConfigurationTransactionResult result) finally { if (Directory.Exists(discoveryRoot)) Directory.Delete(discoveryRoot, true); } var editingChecks = EditingRuntimeSmoke(); var migrationChecks = ProjectMigrationSmoke.Run(); - Console.WriteLine($"{{\"success\":true,\"formats\":3,\"json5\":true,\"normalization\":true,\"profile_loading\":true,\"host_parity\":true,\"discovery\":true,\"selection\":true,\"bounded_loading\":true,\"machine_roots\":true,\"migration_checks\":{migrationChecks},\"editing\":true,\"editor_checks\":{editingChecks.Editor},\"shared_editing_checks\":{editingChecks.Shared},\"transaction_checks\":{editingChecks.Transaction},\"syntax_cases\":{syntaxCases},\"morphir_cases\":{morphirCases}}}"); + var workspaceChecks = WorkspaceMigrationSmoke.Run(); + Console.WriteLine($"{{\"success\":true,\"formats\":3,\"json5\":true,\"normalization\":true,\"profile_loading\":true,\"host_parity\":true,\"discovery\":true,\"selection\":true,\"bounded_loading\":true,\"machine_roots\":true,\"migration_checks\":{migrationChecks},\"workspace_migration_checks\":{workspaceChecks},\"editing\":true,\"editor_checks\":{editingChecks.Editor},\"shared_editing_checks\":{editingChecks.Shared},\"transaction_checks\":{editingChecks.Transaction},\"syntax_cases\":{syntaxCases},\"morphir_cases\":{morphirCases}}}"); return 0; } catch (Exception error) diff --git a/tests/Morphir.Configuration.IO.Smoke/README.md b/tests/Morphir.Configuration.IO.Smoke/README.md index 82c4d6c5..88264b3a 100644 --- a/tests/Morphir.Configuration.IO.Smoke/README.md +++ b/tests/Morphir.Configuration.IO.Smoke/README.md @@ -22,8 +22,8 @@ alternate-primary preconditions, staging/install/rollback faults and external writer conflicts. Incomplete recovery must return owned backup/journal metadata and preserve the -uncertain installed bytes. The host verifies the candidate and backup before -simulating manual recovery, then removes the returned owned artifacts. If a check +uncertain installed bytes. The host inspects the durable journal and verifies the candidate and backup before +calling the explicit shared recovery API. It checks restored bytes and owned-artifact cleanup. If a check fails while recovery remains pending, the host retains that temporary directory and prints its location to stderr. These checks qualify per-file installation; they make no multi-file group-atomicity claim. @@ -33,24 +33,24 @@ and executions sequentially. The reflection serialization switch stays disabled in all three modes: ```sh -mkdir -p /private/tmp/morphir-config-editing/smoke/tmp -export TMPDIR=/private/tmp/morphir-config-editing/smoke/tmp +mkdir -p /private/tmp/morphir-workspace-migration/publish/tmp +export TMPDIR=/private/tmp/morphir-workspace-migration/publish/tmp export TEMP="$TMPDIR" TMP="$TMPDIR" DOTNET_PROCESSOR_COUNT=2 dotnet publish tests/Morphir.Configuration.IO.Smoke \ -c Release -r osx-arm64 -m:1 -p:BuildInParallel=false \ -p:PublishTrimmed=false -p:PublishAot=false \ -p:JsonSerializerIsReflectionEnabledByDefault=false \ - -o /private/tmp/morphir-config-editing/smoke/managed -/private/tmp/morphir-config-editing/smoke/managed/Morphir.Configuration.IO.Smoke \ + -o /private/tmp/morphir-workspace-migration/publish/managed +/private/tmp/morphir-workspace-migration/publish/managed/Morphir.Configuration.IO.Smoke \ --fixtures tests/Fixtures/Configuration DOTNET_PROCESSOR_COUNT=2 dotnet publish tests/Morphir.Configuration.IO.Smoke \ -c Release -r osx-arm64 -m:1 -p:BuildInParallel=false \ -p:PublishTrimmed=true -p:ILLinkTreatWarningsAsErrors=false \ -p:JsonSerializerIsReflectionEnabledByDefault=false \ - -o /private/tmp/morphir-config-editing/smoke/trimmed -/private/tmp/morphir-config-editing/smoke/trimmed/Morphir.Configuration.IO.Smoke \ + -o /private/tmp/morphir-workspace-migration/publish/trimmed +/private/tmp/morphir-workspace-migration/publish/trimmed/Morphir.Configuration.IO.Smoke \ --fixtures tests/Fixtures/Configuration DOTNET_PROCESSOR_COUNT=2 dotnet publish tests/Morphir.Configuration.IO.Smoke \ @@ -58,8 +58,8 @@ DOTNET_PROCESSOR_COUNT=2 dotnet publish tests/Morphir.Configuration.IO.Smoke \ -p:PublishAot=true -p:PublishTrimmed=true \ -p:ILLinkTreatWarningsAsErrors=false -p:TrimmerSingleWarn=false \ -p:JsonSerializerIsReflectionEnabledByDefault=false \ - -o /private/tmp/morphir-config-editing/smoke/native -/private/tmp/morphir-config-editing/smoke/native/Morphir.Configuration.IO.Smoke \ + -o /private/tmp/morphir-workspace-migration/publish/native +/private/tmp/morphir-workspace-migration/publish/native/Morphir.Configuration.IO.Smoke \ --fixtures tests/Fixtures/Configuration ``` @@ -93,7 +93,7 @@ The local development build passes with zero warnings/errors. Its executable passes all 44 new editing checks: 9 editor, 22 shared planning/application and 13 transaction checks. The same invocation retains all 228 grammar checks and 26 pinned Morphir documents. Evidence is under -`/private/tmp/morphir-config-editing/smoke/managed-build.log` and +`/private/tmp/morphir-workspace-migration/publish/managed-build.log` and `managed-run.log`. This checkpoint is a managed build/run; published managed, trimmed and native results belong in the campaign delivery journal after running the commands above. @@ -116,3 +116,18 @@ publish root in the commands above. Keep publishes serial, use two logical processors and retain the existing explicit dependency-warning qualification. The campaign delivery journal records actual commands and results. Preserve any reported recovery directory until its artifacts are resolved. + +## Workspace migration qualification + +The host adds batch planning, application, repeated-run byte/timestamp checks and +explicit durable inspection/recovery outside the CLI. It uses the same bounded +inventory and project verification services as the CLI, including a hidden primary +anchor and common proposed base workspace. It verifies raw secret redaction using +the actual linked-source generated workspace report with reflection serialization +disabled. The JSON receipt adds `workspace_migration_checks`. + +A deliberate interrupted transaction must block a subsequent workspace preview. +The host explicitly recovers its journal, proves originals restored and verifies +that no pending state or owned artifacts remain. If recovery cannot be proved, +it retains the temporary tree and reports its physical path on stderr. The +workspace delivery journal records the managed, trimmed and native receipts. diff --git a/tests/Morphir.Configuration.IO.Smoke/WorkspaceMigrationSmoke.cs b/tests/Morphir.Configuration.IO.Smoke/WorkspaceMigrationSmoke.cs new file mode 100644 index 00000000..0379468c --- /dev/null +++ b/tests/Morphir.Configuration.IO.Smoke/WorkspaceMigrationSmoke.cs @@ -0,0 +1,128 @@ +// Copyright 2026 FINOS +// SPDX-License-Identifier: Apache-2.0 +using System.Text; +using System.Text.Json; +using Morphir.Configuration; +using Morphir.Configuration.IO; +using Morphir.Tooling.Features.WorkspaceMigration; + +internal static class WorkspaceMigrationSmoke +{ + internal static int Run() + { + var checks = 0; + void Check(bool value, string message) { if (!value) throw new InvalidOperationException("Workspace smoke: " + message); checks++; } + const string classic = "{name:'A',sourceDirectory:'../source',exposedModules:[],custom:{password:'private-literal',ref:{env:'PRIVATE_REFERENCE'}}}"; + var files = new Dictionary { ["/repo/a/.morphir/morphir.json"] = classic, ["/repo/b/morphir.json"] = "{name:'B',exposedModules:[]}" }; + var memory = new InMemoryProjectTree("/", files); + var readOnly = new WorkspaceMigrationService(memory); + var preview = readOnly.Plan("/repo"); + Check(preview.CanApply && preview.Verified && !preview.RootIsProject, "read-only aggregate verification"); + Check(preview.Members.SequenceEqual(new[] { "a", "b" }), "explicit sorted membership"); + Check(preview.Projects.All(p => p.Verification!.Verified && p.Verification.BaseWorkspace!.Value.Root == "/repo"), "common proposed workspace"); + Check(preview.Projects[0].ProjectPath == "/repo/a", "hidden primary owner anchor"); + Check(preview.AdoptionEffects.Any(e => e.Kind == "shared-output-root" && e.ProjectPath == "/repo/a"), "adoption effects missing"); + Check(memory.ObservedPaths.All(p => ProjectPath.contains("/repo", p)), "ancestor adoption scan"); + var json = WorkspaceMigrationReport.From(preview, "preview", true).ToJson(); + Check(!json.Contains("private-literal", StringComparison.Ordinal) && !json.Contains("PRIVATE_REFERENCE", StringComparison.Ordinal), "raw secret escaped redaction"); + using (var parsed = JsonDocument.Parse(json)) Check(parsed.RootElement.GetProperty("verified").GetBoolean(), "generated report verification lost"); + Check(readOnly.Apply(preview).Status == ConfigurationTransactionStatus.IoFailure, "read-only apply allowed"); + var root = ProjectPath.join(Path.GetTempPath(), "workspace-smoke-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + try + { + foreach (var file in files) + { + var path = ProjectPath.join(root, ProjectPath.relative("/repo", file.Key)); + Directory.CreateDirectory(ProjectPath.parent(path)); File.WriteAllText(path, file.Value); + } + var sourceBytes = Directory.GetFiles(root, "*.json", SearchOption.AllDirectories).ToDictionary(p => p, File.ReadAllBytes); + Directory.CreateDirectory(root + "/a/.morphir/out"); File.WriteAllText(root + "/a/.morphir/out/generated.json", "keep-generated"); + var service = new WorkspaceMigrationService(new FileSystemProjectTree(root), new NativeConfigurationTransactionHost()); + var plan = service.Plan(root); + Check(plan.CanApply && plan.Projects.Length == 2, "native plan differs from mounted plan"); + Check(service.Apply(plan).Status == ConfigurationTransactionStatus.Written, "batch application failed"); + Check(sourceBytes.All(p => File.ReadAllBytes(p.Key).SequenceEqual(p.Value)), "classic source bytes changed"); + Check(File.ReadAllText(root + "/a/.morphir/out/generated.json") == "keep-generated", "generated artifact moved"); + var targets = Directory.GetFiles(root, "*.toml", SearchOption.AllDirectories); + var bytes = targets.ToDictionary(p => p, File.ReadAllBytes); + var timestamp = new DateTime(2020, 1, 1, 0, 0, 0, DateTimeKind.Utc); + foreach (var path in targets) File.SetLastWriteTimeUtc(path, timestamp); + var repeat = service.Plan(root); + Check(repeat.CanApply && !repeat.Changed, "repeat not unchanged"); + Check(repeat.Inventory.Count(i => i.Kind == WorkspaceInventoryKind.AlreadyMigrated) == 2, "migrated inventory not retained"); + Check(service.Apply(repeat).Status == ConfigurationTransactionStatus.Unchanged, "repeat rewrote files"); + Check(targets.All(p => File.ReadAllBytes(p).SequenceEqual(bytes[p]) && File.GetLastWriteTimeUtc(p) == timestamp), "no-op bytes/timestamps changed"); + var stale = service.Plan(root); File.AppendAllText(root + "/b/morphir.json", "\n"); + Check(service.Apply(stale).Status == ConfigurationTransactionStatus.Conflict, "source drift ignored"); + var transaction = new ConfigurationTransaction(); + var target = root + "/b/morphir.toml"; var original = File.ReadAllBytes(target); + var candidate = Encoding.UTF8.GetBytes("[project]\nname='B'\n"); + var pending = new ConfigurationTransaction(fault: (phase, _) => { if (phase is ConfigurationTransactionPhase.AfterInstall or ConfigurationTransactionPhase.BeforeRollback) throw new IOException("smoke interruption"); }) + .Apply(new ConfigurationTransactionPlan([ConfigurationTransaction.Observe(root)], [new ConfigurationFileChange(target, ConfigurationTransaction.Observe(target), candidate)])); + Check(pending.Status == ConfigurationTransactionStatus.RecoveryRequired, "interruption falsely completed"); + var journal = pending.Artifacts.First(a => a.Kind == ConfigurationArtifactKind.Journal).Path; + Check(transaction.InspectRecovery(journal).Valid, "durable inspection rejected journal"); + Check(transaction.InspectPending([root, target]).Length > 0, "durable pending index absent"); + var blocked = service.Plan(root); + Check(!blocked.CanApply && blocked.PendingRecovery.Length > 0, "pending recovery preview applicable"); + var failureJson = WorkspaceMigrationReport.From(blocked, "recovery-required", true, 2).ToJson(); + using (var parsed = JsonDocument.Parse(failureJson)) Check(parsed.RootElement.GetProperty("exitCode").GetInt32() == 2, "recovery report lost exit status"); + Check(transaction.Recover(journal).Status == ConfigurationTransactionStatus.RolledBack, "explicit recovery did not restore batch"); + Check(File.ReadAllBytes(target).SequenceEqual(original), "recovery lost original bytes"); + Check(transaction.InspectPending([root, target]).IsEmpty, "recovery left pending state"); + Check(!Directory.EnumerateFileSystemEntries(root, ".morphir-edit-*", SearchOption.AllDirectories).Any(), "owned artifacts remain after recovery"); + var yamlRoot = root + "/yaml-workspace"; + Directory.CreateDirectory(yamlRoot + "/app"); + const string yaml = "workspace:\n members: [app]\n"; + File.WriteAllText(yamlRoot + "/morphir.yaml", yaml); + File.WriteAllText(yamlRoot + "/app/morphir.toml", "[project]\nname='YamlApp'\n"); + var yamlService = new WorkspaceMigrationService(new FileSystemProjectTree(yamlRoot), new NativeConfigurationTransactionHost()); + var yamlPreview = yamlService.Plan(yamlRoot); + Check(yamlPreview.CanApply && !yamlPreview.Changed, "existing YAML workspace not a no-op"); + Check(yamlService.Apply(yamlPreview).Status == ConfigurationTransactionStatus.Unchanged, "YAML no-op wrote a target"); + Check(!File.Exists(yamlRoot + "/morphir.toml"), "YAML no-op created an empty TOML primary"); + Check(File.ReadAllText(yamlRoot + "/morphir.yaml") == yaml, "YAML source was rewritten"); + var uncertainTarget = root + "/uncertain/new/morphir.toml"; + var uncertain = new ConfigurationTransaction(new InterruptedParentHost()).Apply(new ConfigurationTransactionPlan( + [ConfigurationTransaction.Observe(root)], [new ConfigurationFileChange(uncertainTarget, ConfigurationTransaction.Observe(uncertainTarget), Encoding.UTF8.GetBytes("value=true\n"))])); + Check(uncertain.Status == ConfigurationTransactionStatus.RecoveryRequired, "uncertain directory receipt falsely completed"); + var uncertainJournal = uncertain.Artifacts.First(a => a.Kind == ConfigurationArtifactKind.Journal).Path; + var uncertainInspection = transaction.InspectRecovery(uncertainJournal); + Check(uncertainInspection.Valid && uncertainInspection.UncertainDirectories.Length > 0, "directory uncertainty lost after inspection"); + Check(transaction.Recover(uncertainJournal).Status == ConfigurationTransactionStatus.RecoveryRequired, "default recovery assumed directory ownership"); + var preserved = transaction.Recover(uncertainJournal, uncertainInspection.UncertainDirectories); + Check(preserved.Status == ConfigurationTransactionStatus.RolledBack && preserved.PreservedDirectories.SequenceEqual(uncertainInspection.UncertainDirectories), "explicit reviewed retention failed"); + Check(Directory.Exists(root + "/uncertain/new"), "recovery deleted an unproven directory"); + Check(transaction.InspectPending([root]).IsEmpty, "reviewed retention left a recovery guard"); + var lossTree = new InMemoryProjectTree("/repo", new Dictionary { ["/repo/a/morphir.json"] = "{name:'A',exposedModules:[],decorations:{lost:true}}" }); + Check(!new WorkspaceMigrationService(lossTree).Plan("/repo").CanApply, "lossy batch accepted"); + } + finally + { + if (Directory.EnumerateFileSystemEntries(root, ".morphir-edit-*", SearchOption.AllDirectories).Any()) Console.Error.WriteLine("Retained workspace smoke recovery: " + root); + else Directory.Delete(root, true); + } + return checks; + } + + private sealed class InterruptedParentHost : IConfigurationTransactionHost + { + private readonly NativeConfigurationTransactionHost inner = new(); + public bool SupportsDurableRecovery => true; + public ConfigurationObservation Observe(string path, IReadOnlyCollection? excludedPaths = null) => inner.Observe(path, excludedPaths); + public byte[] ReadFile(string path) => inner.ReadFile(path); + public IDisposable Acquire(IReadOnlyList paths) => inner.Acquire(paths); + public IReadOnlyList FindRecoveryJournals(IReadOnlyList paths) => inner.FindRecoveryJournals(paths); + public IReadOnlyList? EnumerateTransactionArtifacts(string directory, string transactionId) => inner.EnumerateTransactionArtifacts(directory, transactionId); + public IReadOnlyList CreateParentDirectories(string destination) + { + inner.CreateParentDirectories(destination); + throw new IOException("injected post-mkdir receipt interruption"); + } + public string Stage(string destination, ReadOnlyMemory content, string transactionId, string role) => inner.Stage(destination, content, transactionId, role); + public void Install(string stagedPath, string destination, bool overwrite) => inner.Install(stagedPath, destination, overwrite); + public void DeleteFile(string path) => inner.DeleteFile(path); + public void DeleteDirectory(string path) => inner.DeleteDirectory(path); + } +} diff --git a/tests/Morphir.Configuration.IO.Tests/ConfigurationArtifactOwnershipTests.cs b/tests/Morphir.Configuration.IO.Tests/ConfigurationArtifactOwnershipTests.cs new file mode 100644 index 00000000..88d3403c --- /dev/null +++ b/tests/Morphir.Configuration.IO.Tests/ConfigurationArtifactOwnershipTests.cs @@ -0,0 +1,161 @@ +using System.Text; +using FluentAssertions; +using Morphir.Configuration.IO; + +namespace Morphir.Configuration.IO.Tests; + +public sealed class ConfigurationArtifactOwnershipTests +{ + [Test] + [Arguments("journal", false)] + [Arguments("stage", false)] + [Arguments("backup", false)] + [Arguments("journal", true)] + [Arguments("stage", true)] + [Arguments("backup", true)] + public void AHostCannotTransferAUserFileIntoTransactionOwnership(string role, bool forgedException) + { + var root = ProjectPath.join(Path.GetTempPath(), "transaction-foreign-artifact-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + var resolved = false; + try + { + var target = root + "/morphir.toml"; + var foreign = root + "/user-owned.txt"; + File.WriteAllText(target, "original=true\n"); + var foreignText = role == "stage" ? "candidate=true\n" : role == "backup" ? "original=true\n" : "user-owned=true\n"; + File.WriteAllText(foreign, foreignText); + var host = new ForeignArtifactHost(role, foreign, forgedException); + var result = new ConfigurationTransaction(host).Apply(new ConfigurationTransactionPlan([], + [new ConfigurationFileChange(target, host.Observe(target), Encoding.UTF8.GetBytes("candidate=true\n"))])); + File.Exists(foreign).Should().BeTrue("an adapter response cannot transfer ownership of a pre-existing user file"); + File.ReadAllText(foreign).Should().Be(foreignText); + File.ReadAllText(target).Should().Be("original=true\n"); + result.Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); + result.UnresolvedPaths.Should().Contain(foreign); + result.Artifacts.Should().NotContain(a => a.Path == foreign, "unverified paths are not owned cleanup artifacts"); + var journal = result.Artifacts.First(a => a.Kind == ConfigurationArtifactKind.Journal).Path; + var transaction = new ConfigurationTransaction(); + transaction.InspectPending([root]).Should().NotBeEmpty(); + transaction.InspectRecovery(journal).UnresolvedPaths.Should().Contain(foreign, + "inspection must report rejected receipt paths even when the Apply result was lost"); + transaction.Recover(journal).Status.Should().Be(ConfigurationTransactionStatus.RolledBack); + File.ReadAllText(foreign).Should().Be(foreignText); + transaction.InspectPending([root]).Should().BeEmpty(); + resolved = true; + } + finally + { + if (resolved) Directory.Delete(root, true); + else Console.Error.WriteLine("Retained foreign-artifact evidence: " + root); + } + } + + [Test] + [Arguments(false)] + [Arguments(true)] + public void ParentCreationCannotTransferAnExistingForeignDirectoryIntoOwnership(bool forgedException) + { + var root = ProjectPath.join(Path.GetTempPath(), "transaction-foreign-directory-" + Guid.NewGuid().ToString("N")); + var foreign = root + "/user-owned-empty"; + Directory.CreateDirectory(foreign); + var resolved = false; + try + { + var target = root + "/new/morphir.toml"; + var host = new ForeignArtifactHost("directory", foreign, forgedException); + var result = new ConfigurationTransaction(host).Apply(new ConfigurationTransactionPlan([], + [new ConfigurationFileChange(target, host.Observe(target), Encoding.UTF8.GetBytes("candidate=true\n"))])); + Directory.Exists(foreign).Should().BeTrue("an existing empty directory is not transaction-owned"); + File.Exists(target).Should().BeFalse(); + result.Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); + result.UnresolvedPaths.Should().Contain(foreign); + result.Artifacts.Should().NotContain(a => a.Path == foreign); + var journal = result.Artifacts.First(a => a.Kind == ConfigurationArtifactKind.Journal).Path; + new ConfigurationTransaction().InspectRecovery(journal).UnresolvedPaths.Should().Contain(foreign); + new ConfigurationTransaction().Recover(journal).Status.Should().Be(ConfigurationTransactionStatus.RolledBack); + Directory.Exists(foreign).Should().BeTrue(); + resolved = true; + } + finally + { + if (resolved) Directory.Delete(root, true); + else Console.Error.WriteLine("Retained foreign-directory evidence: " + root); + } + } + + [Test] + [Arguments("backup")] + [Arguments("journal-ready")] + public void CleanupPreservesAnArtifactWhoseBytesChangedAfterStaging(string role) + { + var root = ProjectPath.join(Path.GetTempPath(), "transaction-cleanup-drift-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + var resolved = false; + string? changed = null; + byte[]? original = null; + try + { + var target = root + "/morphir.toml"; + File.WriteAllText(target, "original=true\n"); + var changedText = role == "backup" ? "user-backup=true\n" : " "; + var result = new ConfigurationTransaction(fault: (phase, _) => + { + if (phase != ConfigurationTransactionPhase.BeforeCleanup) return; + changed = Directory.GetFiles(root, ".morphir-edit-*-" + role).Single(); + original = File.ReadAllBytes(changed); + // A valid JSON journal with added whitespace still has different user-owned bytes. + if (role == "backup") File.WriteAllText(changed, changedText); + else File.AppendAllText(changed, changedText); + }).Apply(new ConfigurationTransactionPlan([], + [new ConfigurationFileChange(target, ConfigurationTransaction.Observe(target), Encoding.UTF8.GetBytes("candidate=true\n"))])); + result.Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); + File.Exists(changed).Should().BeTrue("cleanup must not discard changed evidence"); + if (role == "backup") File.ReadAllText(changed!).Should().Be(changedText); + else File.ReadAllBytes(changed!).Should().Equal(original!.Concat(Encoding.UTF8.GetBytes(changedText))); + result.Artifacts.Should().Contain(a => a.Path == changed); + var transaction = new ConfigurationTransaction(); + transaction.InspectPending([root]).Should().NotBeEmpty(); + File.ReadAllText(target).Should().Be("candidate=true\n"); + // The fixture explicitly restores its recorded bytes after checking preservation. + File.WriteAllBytes(changed!, original!); + var journal = result.Artifacts.First(a => a.Kind == ConfigurationArtifactKind.Journal).Path; + transaction.Recover(journal).Status.Should().Be(ConfigurationTransactionStatus.Written); + transaction.InspectPending([root]).Should().BeEmpty(); + resolved = true; + } + finally + { + if (resolved) Directory.Delete(root, true); + else Console.Error.WriteLine("Retained cleanup-drift evidence: " + root); + } + } + + private sealed class ForeignArtifactHost(string role, string foreign, bool forgedException) : IConfigurationTransactionHost + { + private readonly NativeConfigurationTransactionHost native = new(); + public bool SupportsDurableRecovery => true; + public ConfigurationObservation Observe(string path, IReadOnlyCollection? excludedPaths = null) => native.Observe(path, excludedPaths); + public byte[] ReadFile(string path) => native.ReadFile(path); + public IDisposable Acquire(IReadOnlyList paths) => native.Acquire(paths); + public IReadOnlyList CreateParentDirectories(string destination) => native.CreateParentDirectories(destination); + public IReadOnlyList CreateParentDirectories(string destination, string transactionId) + { + if (role != "directory") return native.CreateParentDirectories(destination, transactionId); + if (!forgedException) return [foreign]; + throw new ConfigurationArtifactFailureException([new(foreign, ConfigurationArtifactKind.Directory, transactionId)]); + } + public string Stage(string destination, ReadOnlyMemory content, string transactionId, string artifactRole) + { + if (artifactRole != role) return native.Stage(destination, content, transactionId, artifactRole); + if (!forgedException) return foreign; + var kind = role == "backup" ? ConfigurationArtifactKind.Backup : role == "stage" ? ConfigurationArtifactKind.Stage : ConfigurationArtifactKind.Journal; + throw new ConfigurationArtifactFailureException([new(foreign, kind, transactionId)]); + } + public void Install(string stagedPath, string destination, bool overwrite) => native.Install(stagedPath, destination, overwrite); + public void DeleteFile(string path) => native.DeleteFile(path); + public void DeleteDirectory(string path) => native.DeleteDirectory(path); + public IReadOnlyList FindRecoveryJournals(IReadOnlyList paths) => native.FindRecoveryJournals(paths); + public IReadOnlyList? EnumerateTransactionArtifacts(string directory, string transactionId) => native.EnumerateTransactionArtifacts(directory, transactionId); + } +} diff --git a/tests/Morphir.Configuration.IO.Tests/ConfigurationRecoveryApiTests.cs b/tests/Morphir.Configuration.IO.Tests/ConfigurationRecoveryApiTests.cs new file mode 100644 index 00000000..e64c6dd3 --- /dev/null +++ b/tests/Morphir.Configuration.IO.Tests/ConfigurationRecoveryApiTests.cs @@ -0,0 +1,126 @@ +// Copyright 2026 FINOS +// SPDX-License-Identifier: Apache-2.0 +using System.Text; +using FluentAssertions; +using Morphir.Configuration.IO; + +namespace Morphir.Configuration.IO.Tests; + +public sealed class ConfigurationRecoveryApiTests +{ + [Test] + [Arguments(null)] + [Arguments("")] + [Arguments("relative")] + public void InvalidJournalPathsReturnMetadataWithoutAttemptingAWrite(string? path) + { + var transaction = new ConfigurationTransaction(); + transaction.InspectRecovery(path!).Valid.Should().BeFalse(); + transaction.Recover(path!).Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); + } + + [Test] + public void UnrelatedDirectoryCannotBePresentedAsAnUncertainCreationReceipt() + { + using var fixture = new RecoveryFixture(); + new ConfigurationTransaction().Recover(fixture.Journal, [fixture.Root]).Code.Should().Be("config.recovery_invalid_options"); + File.ReadAllText(fixture.Target).Should().Be("candidate=true\n"); + fixture.Resolve(); + } + + [Test] + public void AHostWithoutArtifactEnumerationCanInspectButCannotRecover() + { + using var fixture = new RecoveryFixture(); + var transaction = new ConfigurationTransaction(new RecoveryHost("enumeration")); + transaction.InspectRecovery(fixture.Journal).Valid.Should().BeTrue(); + transaction.Recover(fixture.Journal).Code.Should().Be("config.recovery_host_capability"); + File.ReadAllText(fixture.Target).Should().Be("candidate=true\n"); + fixture.Resolve(); + } + + [Test] + [Arguments(ConfigurationTransactionPhase.BeforeRollback, ConfigurationTransactionStatus.RecoveryRequired)] + [Arguments(ConfigurationTransactionPhase.AfterRollback, ConfigurationTransactionStatus.RolledBack)] + public void ExplicitRecoveryFaultsReportOnlyProvenRestoration(ConfigurationTransactionPhase phase, ConfigurationTransactionStatus expected) + { + using var fixture = new RecoveryFixture(); + var result = new ConfigurationTransaction(fault: (current, _) => { if (current == phase) throw new IOException("injected recovery boundary"); }).Recover(fixture.Journal); + result.Status.Should().Be(expected); + if (expected == ConfigurationTransactionStatus.RecoveryRequired) + { + result.UnresolvedPaths.Should().Contain(fixture.Target); + File.ReadAllText(fixture.Target).Should().Be("candidate=true\n"); + fixture.Resolve(); + } + else File.ReadAllText(fixture.Target).Should().Be("original=true\n"); + } + + [Test] + public void RecoveryCoordinationFailureRetainsTheJournalAndCandidate() + { + using var fixture = new RecoveryFixture(); + new ConfigurationTransaction(new RecoveryHost("acquire")).Recover(fixture.Journal).Code.Should().Be("config.recovery_io"); + File.Exists(fixture.Journal).Should().BeTrue(); + File.ReadAllText(fixture.Target).Should().Be("candidate=true\n"); + fixture.Resolve(); + } + + [Test] + public void NativeHostRefusesBlockedParentsAndMissingStageDirectoriesWithoutChangingThem() + { + var root = ProjectPath.join(Path.GetTempPath(), "native-boundary-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + try + { + File.WriteAllText(root + "/block", "user-owned"); + var host = new NativeConfigurationTransactionHost(); + Action create = () => host.CreateParentDirectories(root + "/block/target.toml"); + create.Should().Throw(); + File.ReadAllText(root + "/block").Should().Be("user-owned"); + Action stage = () => host.Stage(root + "/absent/target.toml", Encoding.UTF8.GetBytes("value=true\n"), Guid.NewGuid().ToString("N"), "stage"); + stage.Should().Throw(); + Directory.Exists(root + "/absent").Should().BeFalse(); + } + finally { Directory.Delete(root, true); } + } + + private sealed class RecoveryFixture : IDisposable + { + public string Root { get; } = ProjectPath.join(Path.GetTempPath(), "recovery-api-" + Guid.NewGuid().ToString("N")); + public string Target => Root + "/morphir.toml"; + public string Journal { get; } + public RecoveryFixture() + { + Directory.CreateDirectory(Root); + File.WriteAllText(Target, "original=true\n"); + var failed = new ConfigurationTransaction(fault: (phase, _) => + { + if (phase is ConfigurationTransactionPhase.AfterInstall or ConfigurationTransactionPhase.BeforeRollback) throw new IOException("injected fixture interruption"); + }).Apply(new ConfigurationTransactionPlan([], [new ConfigurationFileChange(Target, ConfigurationTransaction.Observe(Target), Encoding.UTF8.GetBytes("candidate=true\n"))])); + failed.Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); + Journal = failed.Artifacts.First(a => a.Kind == ConfigurationArtifactKind.Journal).Path; + } + public void Resolve() => new ConfigurationTransaction().Recover(Journal).Status.Should().Be(ConfigurationTransactionStatus.RolledBack); + public void Dispose() + { + if (Directory.EnumerateFileSystemEntries(Root, ".morphir-edit-*", SearchOption.AllDirectories).Any()) Console.Error.WriteLine("Retained API recovery evidence: " + Root); + else Directory.Delete(Root, true); + } + } + + private sealed class RecoveryHost(string mode) : IConfigurationTransactionHost + { + private readonly NativeConfigurationTransactionHost inner = new(); + public bool SupportsDurableRecovery => true; + public ConfigurationObservation Observe(string path, IReadOnlyCollection? excludedPaths = null) => inner.Observe(path, excludedPaths); + public byte[] ReadFile(string path) => inner.ReadFile(path); + public IDisposable Acquire(IReadOnlyList paths) => mode == "acquire" ? throw new IOException("injected coordinator failure") : inner.Acquire(paths); + public IReadOnlyList? EnumerateTransactionArtifacts(string directory, string transactionId) => mode == "enumeration" ? null : inner.EnumerateTransactionArtifacts(directory, transactionId); + public IReadOnlyList CreateParentDirectories(string destination) => throw new InvalidOperationException("Recovery created directories"); + public string Stage(string destination, ReadOnlyMemory content, string transactionId, string role) => throw new InvalidOperationException("Recovery staged bytes"); + public void Install(string stagedPath, string destination, bool overwrite) => inner.Install(stagedPath, destination, overwrite); + public void DeleteFile(string path) => inner.DeleteFile(path); + public void DeleteDirectory(string path) => inner.DeleteDirectory(path); + } +} diff --git a/tests/Morphir.Configuration.IO.Tests/ConfigurationTransactionTests.cs b/tests/Morphir.Configuration.IO.Tests/ConfigurationTransactionTests.cs index 3358a665..07f8b650 100644 --- a/tests/Morphir.Configuration.IO.Tests/ConfigurationTransactionTests.cs +++ b/tests/Morphir.Configuration.IO.Tests/ConfigurationTransactionTests.cs @@ -294,7 +294,7 @@ public void PartialStagingFailureHasExplicitArtifactOwnership(bool blockCleanup) if (blockCleanup) { var artifact = result.Artifacts.Single(); - artifact.Kind.Should().Be(ConfigurationArtifactKind.Stage); + artifact.Kind.Should().Be(ConfigurationArtifactKind.Journal); artifact.TransactionId.Should().Be(host.Owner); File.Exists(artifact.Path).Should().BeTrue(); } @@ -432,14 +432,22 @@ public void StagingIntegrityIsCheckedBeforeInstallation(string tampered) File.WriteAllText(target, "value = 1\n"); var observed = ConfigurationTransaction.Observe(target); var plan = new ConfigurationTransactionPlan([observed], [new ConfigurationFileChange(target, observed, Encoding.UTF8.GetBytes("value = 2\n"))]); + string? artifactPath = null; + byte[]? original = null; var result = new ConfigurationTransaction(fault: (phase, _) => { - if (phase == ConfigurationTransactionPhase.BeforeInstall) - File.WriteAllText(Directory.GetFiles(directory.Path, ".morphir-edit-*-" + tampered).Single(), "tampered = true\n"); + if (phase != ConfigurationTransactionPhase.BeforeInstall) return; + artifactPath = Directory.GetFiles(directory.Path, ".morphir-edit-*-" + tampered).Single(); + original = File.ReadAllBytes(artifactPath); + File.WriteAllText(artifactPath, "tampered = true\n"); }).Apply(plan); - result.Status.Should().Be(ConfigurationTransactionStatus.Conflict); + result.Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); File.ReadAllText(target).Should().Be("value = 1\n"); - result.Artifacts.Should().BeEmpty(); + File.ReadAllText(artifactPath!).Should().Be("tampered = true\n"); + result.Artifacts.Should().Contain(a => a.Path == artifactPath); + File.WriteAllBytes(artifactPath!, original!); + var journal = result.Artifacts.First(a => a.Kind == ConfigurationArtifactKind.Journal).Path; + new ConfigurationTransaction().Recover(journal).Status.Should().Be(ConfigurationTransactionStatus.RolledBack); Directory.GetFileSystemEntries(directory.Path).Select(ProjectPath.normalize).Should().Equal(target); } @@ -548,9 +556,9 @@ public void AHostCannotTransferOwnershipOfAnotherTransactionsArtifact() var result = new ConfigurationTransaction(new PartialStageHost(false, true)).Apply(new ConfigurationTransactionPlan([observed], [new ConfigurationFileChange(target, observed, Encoding.UTF8.GetBytes("value = 2\n"))])); result.Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); - var foreign = result.Artifacts.Single(); - foreign.TransactionId.Should().Be("another-transaction"); - File.Exists(foreign.Path).Should().BeTrue(); + result.Artifacts.Should().BeEmpty("unverified foreign paths never become owned artifacts"); + result.UnresolvedPaths.Should().NotBeEmpty(); + result.UnresolvedPaths.Should().OnlyContain(path => File.Exists(path)); File.ReadAllText(target).Should().Be("value = 1\n"); } @@ -660,7 +668,9 @@ public string Stage(string destination, ReadOnlyMemory content, string tra { var path = native.Stage(destination, content[..Math.Min(content.Length, 3)], transactionId, role); Owner = foreignOwner ? "another-transaction" : transactionId; - throw new ConfigurationArtifactFailureException([new ConfigurationArtifact(path, ConfigurationArtifactKind.Stage, Owner)]); + var kind = role.StartsWith("journal", StringComparison.Ordinal) ? ConfigurationArtifactKind.Journal + : role == "backup" ? ConfigurationArtifactKind.Backup : ConfigurationArtifactKind.Stage; + throw new ConfigurationArtifactFailureException([new ConfigurationArtifact(path, kind, Owner)]); } public void Install(string stagedPath, string destination, bool overwrite) => native.Install(stagedPath, destination, overwrite); public void DeleteFile(string path) @@ -677,6 +687,11 @@ private sealed class TestDirectory : IDisposable public string Path { get; } = ProjectPath.join(System.IO.Path.GetTempPath(), "morphir-transaction-" + Guid.NewGuid().ToString("N")); public TestDirectory() => Directory.CreateDirectory(Path); public string PathFor(string name) => ProjectPath.join(Path, name); - public void Dispose() => Directory.Delete(Path, true); + public void Dispose() + { + if (Directory.GetFiles(Path, ".morphir-edit-*").Length > 0) + Console.Error.WriteLine("Retained transaction fixture recovery evidence: " + Path); + else Directory.Delete(Path, true); + } } } diff --git a/tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationApiBoundaryTests.cs b/tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationApiBoundaryTests.cs new file mode 100644 index 00000000..9b55a4fc --- /dev/null +++ b/tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationApiBoundaryTests.cs @@ -0,0 +1,185 @@ +// Copyright 2026 FINOS +// SPDX-License-Identifier: Apache-2.0 +using FluentAssertions; +using Morphir.Configuration.IO; + +namespace Morphir.Configuration.IO.Tests; + +public sealed class WorkspaceMigrationNoopFormatTests +{ + [Test] + [Arguments("morphir.yaml", "workspace:\n members: [app]\ncustom:\n enabled: false\n")] + [Arguments("morphir.yml", "workspace:\n members: [app]\n")] + [Arguments("morphir.json", "{workspace:{members:['app']}}")] + [Arguments("morphir.json5", "{workspace:{members:['app']}}")] + [Arguments("morphir-workspace.json", "{workspace:{members:['app']}}")] + [Arguments(".config/morphir/config.yaml", "workspace:\n members: [app]\n")] + public void ExistingNonTomlWorkspaceWithNoConversionsDoesNotCreateAnEmptyPrimary(string primary, string text) + { + using var fixture = new WorkspaceFixture(); + fixture.Write(primary, text); + fixture.Write("app/morphir.toml", "# preserve member\n[project]\nname='App'\n"); + var source = File.ReadAllBytes(ProjectPath.join(fixture.Root, primary)); + var preview = fixture.Service.Plan(fixture.Root); + preview.CanApply.Should().BeTrue(string.Join(",", preview.Diagnostics.Select(d => d.Code))); + preview.Changed.Should().BeFalse(); + fixture.Service.Apply(preview).Status.Should().Be(ConfigurationTransactionStatus.Unchanged); + File.Exists(fixture.Root + "/morphir.toml").Should().BeFalse(); + File.ReadAllBytes(ProjectPath.join(fixture.Root, primary)).Should().Equal(source); + } +} + +public sealed class WorkspaceMigrationApiBoundaryTests +{ + [Test] + [Arguments(null, "workspace.migrate_boundary")] + [Arguments("", "workspace.migrate_boundary")] + [Arguments("relative", "workspace.migrate_boundary")] + [Arguments("/outside", "workspace.migrate_boundary")] + [Arguments("/repo/missing", "workspace.migrate_root")] + [Arguments("/repo/app/morphir.json", "workspace.migrate_root")] + public void InvalidInvocationHasAStructuredBoundaryFailure(string? root, string code) + { + var tree = new InMemoryProjectTree("/repo", new Dictionary { ["/repo/app/morphir.json"] = "{name:'App',exposedModules:[]}" }); + var preview = new WorkspaceMigrationService(tree).Plan(root!); + preview.CanApply.Should().BeFalse(); + preview.Diagnostics.Should().Contain(d => d.Code == code); + } + + [Test] + [Arguments("root", "workspace.migrate_boundary")] + [Arguments("candidate", "workspace.migrate_membership")] + public void DeclaredWorkspaceBaseCannotBroadenTheOperationOrSilentlyAbsorbOtherRoots(string mode, string code) + { + var files = new Dictionary + { + ["/repo/morphir.toml"] = "[workspace]\nmembers=[]\nroot='" + (mode == "root" ? "../outside" : "sub") + "'\n", + ["/repo/app/morphir.json"] = "{name:'App',exposedModules:[]}" + }; + var preview = new WorkspaceMigrationService(new InMemoryProjectTree("/repo", files, ["/repo/sub"])).Plan("/repo"); + preview.CanApply.Should().BeFalse(); + preview.Diagnostics.Should().Contain(d => d.Code == code); + } + + [Test] + [Arguments("listing", "workspace.migrate_listing")] + [Arguments("directory", "workspace.migrate_inventory")] + [Arguments("link-root", "workspace.migrate_root")] + [Arguments("io", "workspace.migrate_io")] + [Arguments("changing", "workspace.migrate_stale")] + public void BrokenOrChangingMountedEvidenceNeverBecomesApplicable(string mode, string code) + { + var preview = new WorkspaceMigrationService(new FaultTree(mode)).Plan("/repo"); + preview.CanApply.Should().BeFalse(); + preview.Diagnostics.Should().Contain(d => d.Code == code); + preview.Diagnostics.Should().OnlyContain(d => !d.Message.Contains("private-host-detail", StringComparison.Ordinal)); + } + + [Test] + [Arguments(false)] + [Arguments(true)] + public void StorageDriftOrReadFailureIsStructuredBeforeAnyWrite(bool drift) + { + using var fixture = new WorkspaceFixture(); + fixture.Write("app/morphir.json", "{name:'App',exposedModules:[]}"); + var service = new WorkspaceMigrationService(new FileSystemProjectTree(fixture.Root), new ReadBoundaryHost(drift)); + var preview = service.Plan(fixture.Root); + preview.CanApply.Should().BeFalse(); + preview.Diagnostics.Should().Contain(d => d.Code == (drift ? "workspace.migrate_stale" : "workspace.migrate_io")); + Directory.GetFiles(fixture.Root, "*.toml", SearchOption.AllDirectories).Should().BeEmpty(); + } + + [Test] + public void BatchRetainsFullJson5EscapesAndOpaqueArrayReferences() + { + using var fixture = new WorkspaceFixture(); + const string source = "{name:'App',exposedModules:[],custom:{escaped:'\\b\\f\\n\\r\\t',refs:[{env:'PRIVATE_REFERENCE'}]}}"; + fixture.Write("app/morphir.json", source); + var preview = fixture.Service.Plan(fixture.Root); + preview.CanApply.Should().BeTrue(string.Join(",", preview.Diagnostics.Select(d => d.Code))); + fixture.Service.Apply(preview).Status.Should().Be(ConfigurationTransactionStatus.Written); + File.ReadAllText(fixture.Root + "/app/morphir.json").Should().Be(source); + var result = ConfigurationReader.ReadFile(new SourceIdentity(fixture.Root + "/app/morphir.toml", DocumentFormat.Toml, SourceRole.MemberPrimary, 0, null)); + var document = ((SourceContent.Available)result.Content).Item; + ConfigTree.tryFind(Microsoft.FSharp.Collections.ListModule.OfSeq(new[] { "custom", "escaped" }), document.Values)!.Value.Should().Be(ConfigValue.NewString("\b\f\n\r\t")); + } + + [Test] + [Arguments("utf8")] + [Arguments("utf16le")] + [Arguments("utf16be")] + [Arguments("utf32le")] + [Arguments("utf32be")] + public void WorkspaceMembershipMergeRetainsTheExistingEncodingAndBom(string kind) + { + using var fixture = new WorkspaceFixture(); + System.Text.Encoding encoding = kind switch + { + "utf16le" => new System.Text.UnicodeEncoding(false, true, true), + "utf16be" => new System.Text.UnicodeEncoding(true, true, true), + "utf32le" => new System.Text.UTF32Encoding(false, true, true), + "utf32be" => new System.Text.UTF32Encoding(true, true, true), + _ => new System.Text.UTF8Encoding(true, true) + }; + var path = fixture.Root + "/morphir.toml"; + const string original = "# retain é\r\n[workspace]\r\nmembers=[]\r\n"; + File.WriteAllBytes(path, [.. encoding.GetPreamble(), .. encoding.GetBytes(original)]); + fixture.Write("app/morphir.json", "{name:'App',exposedModules:[]}"); + var preview = fixture.Service.Plan(fixture.Root); + preview.CanApply.Should().BeTrue(); + fixture.Service.Apply(preview).Status.Should().Be(ConfigurationTransactionStatus.Written); + var bytes = File.ReadAllBytes(path); + bytes.Take(encoding.GetPreamble().Length).Should().Equal(encoding.GetPreamble()); + encoding.GetString(bytes.AsSpan(encoding.GetPreamble().Length)).Should().Contain("# retain é\r\n"); + } + + [Test] + public void ExpensiveFilterEvaluationReturnsAStructuredFailureWithoutThrowing() + { + var name = new string('a', 100) + "b" + new string('a', 49); + var pattern = string.Concat(Enumerable.Repeat("*a", 50)) + "b" + string.Concat(Enumerable.Repeat("*a", 50)); + var service = new WorkspaceMigrationService(new InMemoryProjectTree("/repo", new Dictionary + { + ["/repo/" + name + "/morphir.json"] = "{name:'App',exposedModules:[]}" + })); + WorkspaceMigrationPreview? preview = null; + Action plan = () => preview = service.Plan("/repo", [pattern]); + plan.Should().NotThrow("bounded matching failures must remain scriptable structured outcomes"); + preview!.CanApply.Should().BeFalse(); + } + + private sealed class FaultTree(string mode) : IProjectTree + { + private readonly IProjectTree inner = new InMemoryProjectTree("/repo", new Dictionary { ["/repo/app/morphir.json"] = "{name:'App',exposedModules:[]}" }); + private int reads; + public string Boundary => "/repo"; + public ProjectEntry Observe(string path) + { + if (mode == "link-root" && path == Boundary) return ProjectEntry.Link; + if (mode == "listing" && path == Boundary) return ProjectEntry.NewDirectory(Microsoft.FSharp.Collections.ListModule.OfSeq(new[] { "/repo/nested/not-immediate" })); + if (mode == "directory" && path == "/repo/app") return ProjectEntry.NewFailure("private-host-detail"); + if (mode == "io" && path == "/repo/app/morphir.json") throw new IOException("private-host-detail"); + if (mode == "changing" && path == "/repo/app/morphir.json" && ++reads > 1) + return ProjectEntry.NewFile(ConfigurationReader.ReadText(path, "{name:'Changed',exposedModules:[]}").Content); + return inner.Observe(path); + } + } + + private sealed class ReadBoundaryHost(bool drift) : IConfigurationTransactionHost + { + private readonly NativeConfigurationTransactionHost inner = new(); + private int observations; + public ConfigurationObservation Observe(string path, IReadOnlyCollection? excludedPaths = null) + { + var observed = inner.Observe(path, excludedPaths); + return drift && path.EndsWith("morphir.json", StringComparison.Ordinal) && ++observations > 1 ? observed with { Identity = "changed" } : observed; + } + public byte[] ReadFile(string path) => drift ? inner.ReadFile(path) : throw new IOException("private-host-detail"); + public IDisposable Acquire(IReadOnlyList paths) => throw new InvalidOperationException("Preview acquired a writer"); + public IReadOnlyList CreateParentDirectories(string destination) => throw new InvalidOperationException("Preview created directories"); + public string Stage(string destination, ReadOnlyMemory content, string transactionId, string role) => throw new InvalidOperationException("Preview staged files"); + public void Install(string stagedPath, string destination, bool overwrite) => throw new InvalidOperationException("Preview installed files"); + public void DeleteFile(string path) => throw new InvalidOperationException("Preview deleted files"); + public void DeleteDirectory(string path) => throw new InvalidOperationException("Preview deleted directories"); + } +} diff --git a/tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationCompositionTests.cs b/tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationCompositionTests.cs new file mode 100644 index 00000000..2b0ed82a --- /dev/null +++ b/tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationCompositionTests.cs @@ -0,0 +1,109 @@ +// Copyright 2026 FINOS +// SPDX-License-Identifier: Apache-2.0 +using FluentAssertions; +using Morphir.Configuration.IO; + +namespace Morphir.Configuration.IO.Tests; + +public sealed class WorkspaceMigrationCompositionTests +{ + [Test] + [Arguments(null)] + [Arguments("shared-output")] + public void RootClassicConversionAndMembershipAdditionShareTheProposedContext(string? output) + { + using var fixture = new WorkspaceFixture(); + var root = "{name:'Root',exposedModules:[],workspace:{members:['old']" + (output is null ? "" : ",out_dir:'" + output + "'") + "}}"; + fixture.Write("morphir.json", root); + fixture.Write("old/morphir.toml", "# retain old\n[project]\nname='Old'\n"); + fixture.Write("new/morphir.json", "{name:'New',exposedModules:[]}"); + var oldBytes = File.ReadAllBytes(fixture.Root + "/old/morphir.toml"); + var plan = fixture.Service.Plan(fixture.Root); + plan.CanApply.Should().BeTrue(string.Join(",", plan.Diagnostics.Select(d => d.Code))); + plan.Members.Should().Equal("old", "new"); + plan.Projects.Should().HaveCount(2).And.OnlyContain(p => p.Verification!.Verified); + plan.Projects.Should().OnlyContain(p => p.Verification!.BaseWorkspace!.Value.OutputDirectory!.Value == ProjectPath.join(fixture.Root, output ?? ".morphir/out")); + fixture.Service.Apply(plan).Status.Should().Be(ConfigurationTransactionStatus.Written); + var actual = ProjectSystem.Discover(new FileSystemProjectTree(fixture.Root), fixture.Root); + actual.Discovery.Projects.Select(p => p.Path).Should().Equal(fixture.Root, fixture.Root + "/new", fixture.Root + "/old"); + File.ReadAllBytes(fixture.Root + "/old/morphir.toml").Should().Equal(oldBytes); + File.ReadAllText(fixture.Root + "/morphir.json").Should().Be(root); + } + + [Test] + public void AnExistingRootMembershipConflictIsNotHiddenByAdoptionComposition() + { + using var fixture = new WorkspaceFixture(); + fixture.Write("morphir.json", "{name:'Root',exposedModules:[],workspace:{members:['other']}}"); + const string target = "[workspace]\nmembers=['old']\n"; + fixture.Write("morphir.toml", target); + fixture.Write("old/morphir.toml", "[project]\nname='Old'\n"); + fixture.Write("new/morphir.json", "{name:'New',exposedModules:[]}"); + var plan = fixture.Service.Plan(fixture.Root); + plan.CanApply.Should().BeFalse(); + plan.Projects.Should().Contain(p => p.Plan != null && p.Plan.Conflicts.Any(c => c.KeyPath.SequenceEqual(new[] { "workspace", "members" }))); + File.ReadAllText(fixture.Root + "/morphir.toml").Should().Be(target); + File.Exists(fixture.Root + "/new/morphir.toml").Should().BeFalse(); + } + [Test] + [Arguments("original")] + [Arguments("members")] + public void ProposedContextCannotInventOriginalTargetDataOrUndeclaredMembership(string mode) + { + using var fixture = new WorkspaceFixture(); + fixture.Write("morphir.json", "{name:'Root',exposedModules:[],custom:{value:'source'}}"); + const string targetText = "[workspace]\nmembers=[]\n[custom]\nvalue='existing'\n"; + fixture.Write("morphir.toml", targetText); + var tree = new FileSystemProjectTree(fixture.Root); + var discovered = ProjectSystem.Discover(tree, fixture.Root).Discovery; + var target = new SourceIdentity(fixture.Root + "/morphir.toml", DocumentFormat.Toml, SourceRole.RootPrimary, 4, null); + var context = new ProjectMigrationContext(fixture.Root, discovered.Workspace, discovered.Root.Sources, + proposedMembers: mode == "members" ? ["unlisted"] : [], + originalTarget: mode == "original" ? new SourceInput(target, SourceContent.Missing, false) : null); + var service = new ProjectMigrationService(tree, new NativeConfigurationTransactionHost()); + var preview = service.PlanInWorkspace(fixture.Root, context, fixture.Root + "/morphir.json"); + preview.CanApply.Should().BeFalse(); + preview.Diagnostics.Should().Contain(d => d.Code == "migration.context"); + File.ReadAllText(fixture.Root + "/morphir.toml").Should().Be(targetText); + } + + [Test] + public void ProposedContextComponentsCannotBeAppliedOutsideTheAggregateCoordinator() + { + using var fixture = new WorkspaceFixture(); + fixture.Write("morphir.json", "{name:'Root',exposedModules:[]}"); + const string original = "[workspace]\nmembers=[]\n"; + fixture.Write("morphir.toml", original); + var tree = new FileSystemProjectTree(fixture.Root); + var discovered = ProjectSystem.Discover(tree, fixture.Root).Discovery; + var context = new ProjectMigrationContext(fixture.Root, discovered.Workspace, discovered.Root.Sources, + proposedMembers: [], originalTarget: discovered.Root.Sources.Single(s => s.Source.Path == fixture.Root + "/morphir.toml")); + var service = new ProjectMigrationService(tree, new NativeConfigurationTransactionHost()); + var preview = service.PlanInWorkspace(fixture.Root, context, fixture.Root + "/morphir.json"); + preview.CanApply.Should().BeTrue(); + service.Apply(preview).Code.Should().Be("migration.unverified_plan"); + File.ReadAllText(fixture.Root + "/morphir.toml").Should().Be(original); + } + + [Test] + public void DeclaredWorkspaceBaseDoesNotReanchorTheRootProjectSourceDirectory() + { + using var fixture = new WorkspaceFixture(); + fixture.Write("morphir.json", "{name:'Root',sourceDirectory:'root-src',exposedModules:[],workspace:{root:'sub',members:['old']}}"); + fixture.Write("sub/old/morphir.toml", "[project]\nname='Old'\n"); + fixture.Write("sub/new/morphir.json", "{name:'New',exposedModules:[]}"); + var plan = fixture.Service.Plan(fixture.Root); + plan.CanApply.Should().BeTrue(string.Join(",", plan.Diagnostics.Select(d => d.Code))); + plan.RootIsProject.Should().BeTrue(); + plan.Members.Should().Equal("old", "new"); + plan.Projects.Should().OnlyContain(p => p.Verification!.BaseWorkspace!.Value.Root == fixture.Root + "/sub"); + var root = plan.Projects.Single(p => p.ProjectPath == fixture.Root); + root.Verification!.Differences.ToArray().Should().BeEmpty(); + ConfigTree.tryFind(Microsoft.FSharp.Collections.ListModule.OfSeq(new[] { "project", "source_directory" }), root.Verification.After)!.Value.Should().Be(ConfigValue.NewString("root-src")); + fixture.Service.Apply(plan).Status.Should().Be(ConfigurationTransactionStatus.Written); + var actual = ProjectSystem.Discover(new FileSystemProjectTree(fixture.Root), fixture.Root).Discovery; + actual.Workspace!.Value.Root.Should().Be(fixture.Root + "/sub"); + actual.Projects.Select(p => p.Path).Should().Equal(fixture.Root, fixture.Root + "/sub/new", fixture.Root + "/sub/old"); + } + +} diff --git a/tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationDiscoveryTests.cs b/tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationDiscoveryTests.cs new file mode 100644 index 00000000..e34d3701 --- /dev/null +++ b/tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationDiscoveryTests.cs @@ -0,0 +1,153 @@ +// Copyright 2026 FINOS +// SPDX-License-Identifier: Apache-2.0 +using FluentAssertions; + +namespace Morphir.Configuration.IO.Tests; + +[Category("WorkspaceMigrationDiscovery")] +public sealed class WorkspaceMigrationDiscoveryTests +{ + [Test] + public void ExistingGlobCrossingSkippedOrNestedBoundariesBlocksBeforeAnyWrite() + { + using var fixture = new NativeFixture(); + fixture.Write("morphir.toml", "# retain declaration\n[workspace]\nmembers=['apps/**']\n"); + fixture.WriteBoundedTree(); + var original = fixture.Snapshot(); + + var preview = fixture.Service.Plan(fixture.Root); + + preview.CanApply.Should().BeFalse("the retained glob would discover independent or skipped projects after application"); + preview.Diagnostics.Should().Contain(diagnostic => diagnostic.Code == "workspace.migrate_membership_boundary"); + fixture.Service.Apply(preview).Code.Should().Be("workspace.migrate_unverified"); + fixture.AssertUnchanged(original); + AssertHardBoundaries(fixture.Tree.ObservedPaths, fixture.Root); + } + + [Test] + public void DeclaredExclusionsKeepTheRetainedGlobEquivalentToActualNativeDiscovery() + { + using var fixture = new NativeFixture(); + const string workspace = "# retain declaration and exclusions\n[workspace]\nmembers=['apps/**']\nexclude=['apps/node_modules/**','apps/nested/**']\n"; + fixture.Write("morphir.toml", workspace); + fixture.WriteBoundedTree(); + var original = fixture.Snapshot(); + + var preview = fixture.Service.Plan(fixture.Root); + preview.CanApply.Should().BeTrue(string.Join(",", preview.Diagnostics.Select(diagnostic => diagnostic.Code))); + preview.Members.Should().Equal("apps/**"); + preview.Projects.Select(project => project.ProjectPath).Should().Equal(fixture.Root + "/apps/a"); + fixture.Service.Apply(preview).Status.Should().Be(ConfigurationTransactionStatus.Written); + + var actualTree = new TrackingTree(new FileSystemProjectTree(fixture.Root)); + var actual = ProjectSystem.Discover(actualTree, fixture.Root); + actual.Discovery.Projects.ToArray().Should().OnlyContain(project => project.IsValid); + actual.Discovery.Projects.Select(project => project.Path).Should().Equal(preview.Projects.Select(project => project.ProjectPath)); + actual.Selection.Projects.Select(project => project.Path).Should().Equal(fixture.Root + "/apps/a"); + actual.Discovery.Diagnostics.ToArray().Should().NotContain(diagnostic => diagnostic.Severity == DiagnosticSeverity.Error); + fixture.AssertOriginalBytes(original); + File.ReadAllText(fixture.Root + "/morphir.toml").Should().Be(workspace); + File.Exists(fixture.Root + "/apps/a/morphir.toml").Should().BeTrue(); + File.Exists(fixture.Root + "/apps/node_modules/pkg/morphir.toml").Should().BeFalse(); + File.Exists(fixture.Root + "/apps/nested/inside/morphir.toml").Should().BeFalse(); + AssertHardBoundaries(fixture.Tree.ObservedPaths, fixture.Root); + AssertHardBoundaries(actualTree.ObservedPaths, fixture.Root); + } + + [Test] + [Arguments("old", "old")] + [Arguments("retained/old", "retained")] + public void SuppliedExclusionsRetainDeclaredMemberDefaultAndExactPrimary(string member, string exclusion) + { + using var fixture = new NativeFixture(); + fixture.Write("morphir.toml", $"# retain default\n[workspace]\nmembers=['{member}']\ndefault_member='{member}'\nout_dir='shared'\n"); + const string existing = "# preserve this primary byte for byte\n[project]\nname='Old'\nsource_directory='old-src'\n"; + fixture.Write(member + "/morphir.toml", existing); + fixture.Write("new/morphir.json", "{name:'New',sourceDirectory:'elm-src',exposedModules:[]}\n"); + var source = File.ReadAllBytes(fixture.Root + "/new/morphir.json"); + var oldPath = fixture.Root + "/" + member + "/morphir.toml"; + var time = new DateTime(2020, 1, 1, 0, 0, 0, DateTimeKind.Utc); + File.SetLastWriteTimeUtc(oldPath, time); + + var preview = fixture.Service.Plan(fixture.Root, excludes: [exclusion]); + preview.CanApply.Should().BeTrue("supplied adoption filters cannot remove an already declared member or default: " + + string.Join(",", preview.Diagnostics.Select(diagnostic => diagnostic.Code))); + preview.Members.Should().Equal(member, "new"); + preview.Projects.Should().ContainSingle().Which.ProjectPath.Should().Be(fixture.Root + "/new"); + fixture.Service.Apply(preview).Status.Should().Be(ConfigurationTransactionStatus.Written); + + var actual = ProjectSystem.Discover(new FileSystemProjectTree(fixture.Root), fixture.Root); + actual.Discovery.Projects.ToArray().Should().OnlyContain(project => project.IsValid); + actual.Discovery.Projects.Select(project => project.Path).Should().BeEquivalentTo(new[] { fixture.Root + "/" + member, fixture.Root + "/new" }); + actual.Selection.Projects.Select(project => project.Path).Should().Equal(fixture.Root + "/" + member); + actual.Discovery.Diagnostics.ToArray().Should().NotContain(diagnostic => diagnostic.Severity == DiagnosticSeverity.Error); + File.ReadAllText(fixture.Root + "/morphir.toml").Should().Contain($"default_member='{member}'").And.Contain("out_dir='shared'"); + File.ReadAllText(oldPath).Should().Be(existing); + File.GetLastWriteTimeUtc(oldPath).Should().Be(time); + File.ReadAllBytes(fixture.Root + "/new/morphir.json").Should().Equal(source); + File.Exists(fixture.Root + "/new/morphir.toml").Should().BeTrue(); + } + + private static void AssertHardBoundaries(IEnumerable observed, string root) + { + var skipped = root + "/apps/node_modules/"; + var nested = root + "/apps/nested"; + var boundaryPrimaries = DiscoveryPolicy.candidates(nested).ToHashSet(StringComparer.Ordinal); + observed.Should().NotContain(path => path.StartsWith(skipped, StringComparison.Ordinal)); + observed.Should().NotContain(path => path.StartsWith(nested + "/", StringComparison.Ordinal) && !boundaryPrimaries.Contains(path)); + } + + private sealed class TrackingTree(IProjectTree inner) : IProjectTree + { + public string Boundary => inner.Boundary; + public List ObservedPaths { get; } = []; + public ProjectEntry Observe(string path) + { + ObservedPaths.Add(path); + return inner.Observe(path); + } + } + + private sealed class NativeFixture : IDisposable + { + public string Root { get; } = ProjectPath.join(Path.GetTempPath(), "workspace-migration-discovery-" + Guid.NewGuid().ToString("N")); + public TrackingTree Tree { get; } + public WorkspaceMigrationService Service { get; } + public NativeFixture() + { + Directory.CreateDirectory(Root); + Tree = new(new FileSystemProjectTree(Root)); + Service = new(Tree, new NativeConfigurationTransactionHost()); + } + public void Write(string relative, string text) + { + var path = ProjectPath.join(Root, relative); + Directory.CreateDirectory(ProjectPath.parent(path)); + File.WriteAllText(path, text); + } + public void WriteBoundedTree() + { + Write("apps/a/morphir.json", "{name:'A',sourceDirectory:'elm-src',exposedModules:[]}\n"); + Write("apps/node_modules/pkg/morphir.json", "{name:'Dependency',exposedModules:[]}\n"); + Write("apps/nested/morphir.toml", "[workspace]\nmembers=['inside']\n"); + Write("apps/nested/inside/morphir.json", "{name:'Independent',exposedModules:[]}\n"); + } + public Dictionary Snapshot() => Directory.GetFiles(Root, "*", SearchOption.AllDirectories) + .ToDictionary(path => path, File.ReadAllBytes, StringComparer.Ordinal); + public void AssertOriginalBytes(IReadOnlyDictionary original) + { + foreach (var file in original) File.ReadAllBytes(file.Key).Should().Equal(file.Value); + } + public void AssertUnchanged(IReadOnlyDictionary original) + { + Directory.GetFiles(Root, "*", SearchOption.AllDirectories).Should().BeEquivalentTo(original.Keys); + AssertOriginalBytes(original); + } + public void Dispose() + { + if (Directory.EnumerateFileSystemEntries(Root, ".morphir-edit-*", SearchOption.AllDirectories).Any()) + Console.Error.WriteLine("Retained workspace discovery recovery evidence: " + Root); + else Directory.Delete(Root, true); + } + } +} diff --git a/tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationSteps.cs b/tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationSteps.cs index 7c5b3d8e..3b018fa3 100644 --- a/tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationSteps.cs +++ b/tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationSteps.cs @@ -7,22 +7,22 @@ namespace Morphir.Configuration.IO.Tests; [Binding] public sealed class WorkspaceMigrationSteps { - private Dictionary files = []; + private Dictionary files = []; private WorkspaceMigrationPreview? preview; [Given("a repository with two classic workspace candidates")] - public void Two() => files = new() { ["/repo/a/morphir.json"]="{name:'A',exposedModules:[]}", ["/repo/b/morphir.json"]="{name:'B',exposedModules:[]}" }; + public void Two() => files = new() { ["/repo/a/morphir.json"] = "{name:'A',exposedModules:[]}", ["/repo/b/morphir.json"] = "{name:'B',exposedModules:[]}" }; [Given("a repository with a root classic project and one child")] - public void Root() { Two(); files.Remove("/repo/b/morphir.json"); files["/repo/morphir.json"]="{name:'Root',exposedModules:[]}"; } + public void Root() { Two(); files.Remove("/repo/b/morphir.json"); files["/repo/morphir.json"] = "{name:'Root',exposedModules:[]}"; } [Given("a repository with excluded and nested workspace candidates")] - public void Boundaries() { Two(); files["/repo/morphir.toml"]="[workspace]\nmembers=[]\nexclude=['b']\n"; files["/repo/nested/morphir.toml"]="[workspace]\nmembers=['inner']\n"; files["/repo/nested/inner/morphir.json"]="{name:'Inner',exposedModules:[]}"; } + public void Boundaries() { Two(); files["/repo/morphir.toml"] = "[workspace]\nmembers=[]\nexclude=['b']\n"; files["/repo/nested/morphir.toml"] = "[workspace]\nmembers=['inner']\n"; files["/repo/nested/inner/morphir.json"] = "{name:'Inner',exposedModules:[]}"; } [When("a host previews bounded workspace migration")] - public void Preview() => preview=new WorkspaceMigrationService(new InMemoryProjectTree("/repo", files)).Plan("/repo"); + public void Preview() => preview = new WorkspaceMigrationService(new InMemoryProjectTree("/repo", files)).Plan("/repo"); [Then("both conversions share one verified virtual workspace")] - public void Virtual() { preview!.CanApply.Should().BeTrue(string.Join(",",preview.Diagnostics.Select(d=>d.Code))); preview.Verified.Should().BeTrue(); preview.RootIsProject.Should().BeFalse(); preview.Members.Should().Equal("a","b"); preview.Projects.Should().HaveCount(2).And.OnlyContain(p=>p.Verification!.BaseWorkspace!.Value.Root=="/repo"); } + public void Virtual() { preview!.CanApply.Should().BeTrue(string.Join(",", preview.Diagnostics.Select(d => d.Code))); preview.Verified.Should().BeTrue(); preview.RootIsProject.Should().BeFalse(); preview.Members.Should().Equal("a", "b"); preview.Projects.Should().HaveCount(2).And.OnlyContain(p => p.Verification!.BaseWorkspace!.Value.Root == "/repo"); } [Then("workspace preview changes no files")] - public void Unchanged() => files.Keys.Should().OnlyContain(p=>p.EndsWith(".json",StringComparison.Ordinal)); + public void Unchanged() => files.Keys.Should().OnlyContain(p => p.EndsWith(".json", StringComparison.Ordinal)); [Then("the root remains a project outside the member list")] public void RootIdentity() { preview!.CanApply.Should().BeTrue(); preview.RootIsProject.Should().BeTrue(); preview.Members.Should().Equal("a"); } [Then("excluded and boundary paths have no conversions")] - public void Omitted() { preview!.CanApply.Should().BeTrue(); preview.Inventory.Should().Contain(i=>i.RelativePath=="b"&&i.Kind==WorkspaceInventoryKind.Excluded).And.Contain(i=>i.RelativePath=="nested"&&i.Kind==WorkspaceInventoryKind.Boundary); preview.Projects.Should().ContainSingle(); } + public void Omitted() { preview!.CanApply.Should().BeTrue(); preview.Inventory.Should().Contain(i => i.RelativePath == "b" && i.Kind == WorkspaceInventoryKind.Excluded).And.Contain(i => i.RelativePath == "nested" && i.Kind == WorkspaceInventoryKind.Boundary); preview.Projects.Should().ContainSingle(); } } diff --git a/tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationTests.cs b/tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationTests.cs index 2ffae581..d252f63f 100644 --- a/tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationTests.cs +++ b/tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationTests.cs @@ -9,107 +9,114 @@ public sealed class WorkspaceMigrationInventoryTests [Test] public void InventoryIsStableBoundedAndExplainsDefaultSkips() { - var files=new Dictionary { - ["/repo/z/morphir.json"]="{name:'Same',exposedModules:[]}", - ["/repo/a/.morphir/morphir.json"]="{name:'Same',exposedModules:[]}", - ["/repo/node_modules/pkg/morphir.json"]="{name:'Ignored',exposedModules:[]}", - ["/outside/morphir.json"]="{name:'Outside',exposedModules:[]}" + var files = new Dictionary + { + ["/repo/z/morphir.json"] = "{name:'Same',exposedModules:[]}", + ["/repo/a/.morphir/morphir.json"] = "{name:'Same',exposedModules:[]}", + ["/repo/node_modules/pkg/morphir.json"] = "{name:'Ignored',exposedModules:[]}", + ["/outside/morphir.json"] = "{name:'Outside',exposedModules:[]}" }; - var tree=new InMemoryProjectTree("/",files); - var plan=new WorkspaceMigrationService(tree).Plan("/repo"); - plan.CanApply.Should().BeTrue(string.Join(",",plan.Diagnostics.Select(d=>d.Code))); - plan.Members.Should().Equal("a","z"); - plan.Projects.Select(p=>p.ProjectPath).Should().Equal("/repo/a","/repo/z"); - plan.Inventory.Should().Contain(i=>i.RelativePath=="node_modules"&&i.Kind==WorkspaceInventoryKind.Excluded); - plan.Diagnostics.Should().Contain(d=>d.Code=="discovery.duplicate_name"); - tree.ObservedPaths.Should().OnlyContain(p=>ProjectPath.contains("/repo",p)); + var tree = new InMemoryProjectTree("/", files); + var plan = new WorkspaceMigrationService(tree).Plan("/repo"); + plan.CanApply.Should().BeTrue(string.Join(",", plan.Diagnostics.Select(d => d.Code))); + plan.Members.Should().Equal("a", "z"); + plan.Projects.Select(p => p.ProjectPath).Should().Equal("/repo/a", "/repo/z"); + plan.Inventory.Should().Contain(i => i.RelativePath == "node_modules" && i.Kind == WorkspaceInventoryKind.Excluded); + plan.Diagnostics.Should().Contain(d => d.Code == "discovery.duplicate_name"); + tree.ObservedPaths.Should().OnlyContain(p => ProjectPath.contains("/repo", p)); } [Test] public void FiltersAreRootRelativeAndExistingExclusionsRemainEffective() { - var files=new Dictionary { - ["/repo/morphir.toml"]="[workspace]\nmembers=[]\nexclude=['apps/b']\n", - ["/repo/apps/a/morphir.json"]="{name:'A',exposedModules:[]}", - ["/repo/apps/b/morphir.json"]="{name:'B',exposedModules:[]}", - ["/repo/other/morphir.json"]="{name:'Other',exposedModules:[]}" + var files = new Dictionary + { + ["/repo/morphir.toml"] = "[workspace]\nmembers=[]\nexclude=['apps/b']\n", + ["/repo/apps/a/morphir.json"] = "{name:'A',exposedModules:[]}", + ["/repo/apps/b/morphir.json"] = "{name:'B',exposedModules:[]}", + ["/repo/other/morphir.json"] = "{name:'Other',exposedModules:[]}" }; - var plan=new WorkspaceMigrationService(new InMemoryProjectTree("/repo",files)).Plan("/repo",["apps/**"],["**/ignored"]); + var plan = new WorkspaceMigrationService(new InMemoryProjectTree("/repo", files)).Plan("/repo", ["apps/**"], ["**/ignored"]); plan.CanApply.Should().BeTrue(); plan.Members.Should().Equal("apps/a"); - plan.Inventory.Should().Contain(i=>i.RelativePath=="apps/b"&&i.Kind==WorkspaceInventoryKind.Excluded).And.Contain(i=>i.RelativePath=="other"&&i.Kind==WorkspaceInventoryKind.Excluded); + plan.Inventory.Should().Contain(i => i.RelativePath == "apps/b" && i.Kind == WorkspaceInventoryKind.Excluded).And.Contain(i => i.RelativePath == "other" && i.Kind == WorkspaceInventoryKind.Excluded); } } public sealed class WorkspaceMigrationPlanningTests { - private const string Classic="{name:'A',sourceDirectory:'../source',exposedModules:[],profiles:{default:['dev']}}"; + private const string Classic = "{name:'A',sourceDirectory:'../source',exposedModules:[],profiles:{default:['dev']}}"; [Test] public void ExistingWorkspaceSettingsDefaultsAndMigratedMembersRemainEffective() { - var tree=new InMemoryProjectTree("/repo",new Dictionary { - ["/repo/morphir.toml"]="# root\n[workspace]\nmembers=['old']\ndefault_member='old'\nout_dir='shared-output'\n[pipeline]\nbackend='shared'\n", - ["/repo/old/morphir.toml"]="# exact old bytes\n[project]\nname='Old'\n", - ["/repo/new/.morphir/morphir.json"]=Classic, - ["/repo/new/morphir.user.toml"]="[project]\nsource_directory='WRONG'\n", - ["/repo/new/morphir.dev.toml"]="[project]\nsource_directory='WRONG'\n" + var tree = new InMemoryProjectTree("/repo", new Dictionary + { + ["/repo/morphir.toml"] = "# root\n[workspace]\nmembers=['old']\ndefault_member='old'\nout_dir='shared-output'\n[pipeline]\nbackend='shared'\n", + ["/repo/old/morphir.toml"] = "# exact old bytes\n[project]\nname='Old'\n", + ["/repo/new/.morphir/morphir.json"] = Classic, + ["/repo/new/morphir.user.toml"] = "[project]\nsource_directory='WRONG'\n", + ["/repo/new/morphir.dev.toml"] = "[project]\nsource_directory='WRONG'\n" }); - var plan=new WorkspaceMigrationService(tree).Plan("/repo"); - plan.CanApply.Should().BeTrue(string.Join(",",plan.Diagnostics.Select(d=>d.Code))); - plan.Members.Should().Equal("old","new"); + var plan = new WorkspaceMigrationService(tree).Plan("/repo"); + plan.CanApply.Should().BeTrue(string.Join(",", plan.Diagnostics.Select(d => d.Code))); + plan.Members.Should().Equal("old", "new"); plan.Projects.Should().ContainSingle().Which.ProjectPath.Should().Be("/repo/new"); - var verification=plan.Projects[0].Verification!; + var verification = plan.Projects[0].Verification!; verification.Verified.Should().BeTrue(); verification.BaseWorkspace!.Value.OutputDirectory!.Value.Should().Be("/repo/shared-output"); - ConfigTree.tryFind(Microsoft.FSharp.Collections.ListModule.OfSeq(new[]{"pipeline","backend"}),verification.After)!.Value.Should().Be(ConfigValue.NewString("shared")); - tree.ObservedPaths.Should().NotContain(p=>p.Contains(".user.",StringComparison.Ordinal)||p.Contains(".dev.",StringComparison.Ordinal)); - plan.Inventory.Should().Contain(i=>i.RelativePath=="old"&&i.Kind==WorkspaceInventoryKind.AlreadyMigrated); + ConfigTree.tryFind(Microsoft.FSharp.Collections.ListModule.OfSeq(new[] { "pipeline", "backend" }), verification.After)!.Value.Should().Be(ConfigValue.NewString("shared")); + tree.ObservedPaths.Should().NotContain(p => p.Contains(".user.", StringComparison.Ordinal) || p.Contains(".dev.", StringComparison.Ordinal)); + plan.Inventory.Should().Contain(i => i.RelativePath == "old" && i.Kind == WorkspaceInventoryKind.AlreadyMigrated); } [Test] public void RootAndChildConversionsShareProposedBaseWithEveryClassicExcluded() { - var tree=new InMemoryProjectTree("/repo",new Dictionary{ - ["/repo/.morphir/morphir.json"]="{name:'Root',exposedModules:[],custom:{same:1}}", - ["/repo/app/.config/morphir/config.json"]="{name:'App',exposedModules:[],custom:{same:1}}" + var tree = new InMemoryProjectTree("/repo", new Dictionary + { + ["/repo/.morphir/morphir.json"] = "{name:'Root',exposedModules:[],custom:{same:1}}", + ["/repo/app/.config/morphir/config.json"] = "{name:'App',exposedModules:[],custom:{same:1}}" }); - var plan=new WorkspaceMigrationService(tree).Plan("/repo"); - plan.CanApply.Should().BeTrue(string.Join(",",plan.Diagnostics.Select(d=>d.Code))); + var plan = new WorkspaceMigrationService(tree).Plan("/repo"); + plan.CanApply.Should().BeTrue(string.Join(",", plan.Diagnostics.Select(d => d.Code))); plan.RootIsProject.Should().BeTrue(); plan.Members.Should().Equal("app"); - plan.Projects.Select(p=>p.ProjectPath).Should().Equal("/repo","/repo/app"); - plan.Projects.Should().OnlyContain(p=>p.Verification!.BaseWorkspace!.Value.Root=="/repo"); - plan.AdoptionEffects.Should().Contain(e=>e.Kind=="shared-output-root"&&e.ProjectPath=="/repo/app"&&e.Before=="/repo/app/.morphir/out"&&e.After=="/repo/.morphir/out"); + plan.Projects.Select(p => p.ProjectPath).Should().Equal("/repo", "/repo/app"); + plan.Projects.Should().OnlyContain(p => p.Verification!.BaseWorkspace!.Value.Root == "/repo"); + plan.AdoptionEffects.Should().Contain(e => e.Kind == "shared-output-root" && e.ProjectPath == "/repo/app" && e.Before == "/repo/app/.morphir/out" && e.After == "/repo/.morphir/out"); } [Test] public void AdoptionCannotExcuseUnrelatedInheritedSettingChanges() { - var tree=new InMemoryProjectTree("/repo",new Dictionary{ - ["/repo/morphir.json"]="{name:'Root',exposedModules:[],pipeline:{backend:'root-specific'}}", - ["/repo/app/morphir.json"]="{name:'App',exposedModules:[]}" + var tree = new InMemoryProjectTree("/repo", new Dictionary + { + ["/repo/morphir.json"] = "{name:'Root',exposedModules:[],pipeline:{backend:'root-specific'}}", + ["/repo/app/morphir.json"] = "{name:'App',exposedModules:[]}" }); - var plan=new WorkspaceMigrationService(tree).Plan("/repo"); + var plan = new WorkspaceMigrationService(tree).Plan("/repo"); plan.CanApply.Should().BeFalse("a new shared output root cannot excuse an unrelated backend change"); - plan.Diagnostics.Should().Contain(d=>d.Code=="workspace.migrate_adoption_value"); + plan.Diagnostics.Should().Contain(d => d.Code == "workspace.migrate_adoption_value"); } [Test] - public void DeclaredGlobsDoNotReadSkippedOutputsOrNestedIndependentMembers() + public void ConflictingDeclaredGlobsBlockWithoutReadingSkippedOrNestedMembers() { - var tree=new InMemoryProjectTree("/repo",new Dictionary{ - ["/repo/morphir.toml"]="[workspace]\nmembers=['apps/**']\n", - ["/repo/apps/a/morphir.json"]="{name:'A',exposedModules:[]}", - ["/repo/apps/node_modules/pkg/morphir.json"]="{name:'Ignored',exposedModules:[]}", - ["/repo/apps/nested/morphir.toml"]="[workspace]\nmembers=['inside']\n", - ["/repo/apps/nested/inside/morphir.json"]="{name:'Inside',exposedModules:[]}" + var tree = new InMemoryProjectTree("/repo", new Dictionary + { + ["/repo/morphir.toml"] = "[workspace]\nmembers=['apps/**']\n", + ["/repo/apps/a/morphir.json"] = "{name:'A',exposedModules:[]}", + ["/repo/apps/node_modules/pkg/morphir.json"] = "{name:'Ignored',exposedModules:[]}", + ["/repo/apps/nested/morphir.toml"] = "[workspace]\nmembers=['inside']\n", + ["/repo/apps/nested/inside/morphir.json"] = "{name:'Inside',exposedModules:[]}" }); - var plan=new WorkspaceMigrationService(tree).Plan("/repo"); - plan.CanApply.Should().BeTrue(string.Join(",",plan.Diagnostics.Select(d=>d.Code))); - plan.Projects.Select(p=>p.ProjectPath).Should().Equal("/repo/apps/a"); - tree.ObservedPaths.Should().NotContain(p=>p.StartsWith("/repo/apps/node_modules/",StringComparison.Ordinal)||p.StartsWith("/repo/apps/nested/inside",StringComparison.Ordinal)); + var plan = new WorkspaceMigrationService(tree).Plan("/repo"); + plan.CanApply.Should().BeFalse(); + plan.Diagnostics.Should().Contain(d => d.Code == "workspace.migrate_membership_boundary"); + tree.ObservedPaths.Should().NotContain(p => p.StartsWith("/repo/apps/node_modules/", StringComparison.Ordinal) || p.StartsWith("/repo/apps/nested/inside", StringComparison.Ordinal)); } [Test] public void SuppliedFiltersDoNotRemoveExistingRootProjectIdentity() { - var plan=new WorkspaceMigrationService(new InMemoryProjectTree("/repo",new Dictionary{ - ["/repo/morphir.json"]="{name:'Root',exposedModules:[]}", - ["/repo/apps/a/morphir.json"]="{name:'A',exposedModules:[]}" - })).Plan("/repo",["apps/**"]); + var plan = new WorkspaceMigrationService(new InMemoryProjectTree("/repo", new Dictionary + { + ["/repo/morphir.json"] = "{name:'Root',exposedModules:[]}", + ["/repo/apps/a/morphir.json"] = "{name:'A',exposedModules:[]}" + })).Plan("/repo", ["apps/**"]); plan.CanApply.Should().BeTrue(); plan.RootIsProject.Should().BeTrue(); plan.Members.Should().Equal("apps/a"); } } @@ -119,21 +126,21 @@ public sealed class WorkspaceMigrationApplicationTests [Test] public void BatchPreservesSourcesAndEquivalentRepeatBytesTimestampsAndInventory() { - using var fixture=new WorkspaceFixture(); - fixture.Write("a/.morphir/morphir.json","{name:'A',sourceDirectory:'../source',exposedModules:[]}"); - fixture.Write("b/morphir.json","{name:'B',exposedModules:[]}"); - var sources=Directory.GetFiles(fixture.Root,"*.json",SearchOption.AllDirectories).ToDictionary(p=>p,File.ReadAllBytes); - var plan=fixture.Service.Plan(fixture.Root); plan.CanApply.Should().BeTrue(string.Join(",",plan.Diagnostics.Select(d=>d.Code))); + using var fixture = new WorkspaceFixture(); + fixture.Write("a/.morphir/morphir.json", "{name:'A',sourceDirectory:'../source',exposedModules:[]}"); + fixture.Write("b/morphir.json", "{name:'B',exposedModules:[]}"); + var sources = Directory.GetFiles(fixture.Root, "*.json", SearchOption.AllDirectories).ToDictionary(p => p, File.ReadAllBytes); + var plan = fixture.Service.Plan(fixture.Root); plan.CanApply.Should().BeTrue(string.Join(",", plan.Diagnostics.Select(d => d.Code))); fixture.Service.Apply(plan).Status.Should().Be(ConfigurationTransactionStatus.Written); - var targets=Directory.GetFiles(fixture.Root,"*.toml",SearchOption.AllDirectories); - var original=targets.ToDictionary(p=>p,File.ReadAllBytes); - var time=new DateTime(2020,1,1,0,0,0,DateTimeKind.Utc); - foreach(var path in targets)File.SetLastWriteTimeUtc(path,time); - var repeated=fixture.Service.Plan(fixture.Root); repeated.CanApply.Should().BeTrue(); repeated.Changed.Should().BeFalse(); - repeated.Inventory.Where(i=>i.Path.EndsWith("/a",StringComparison.Ordinal)||i.Path.EndsWith("/b",StringComparison.Ordinal)).Should().OnlyContain(i=>i.Kind==WorkspaceInventoryKind.AlreadyMigrated); + var targets = Directory.GetFiles(fixture.Root, "*.toml", SearchOption.AllDirectories); + var original = targets.ToDictionary(p => p, File.ReadAllBytes); + var time = new DateTime(2020, 1, 1, 0, 0, 0, DateTimeKind.Utc); + foreach (var path in targets) File.SetLastWriteTimeUtc(path, time); + var repeated = fixture.Service.Plan(fixture.Root); repeated.CanApply.Should().BeTrue(); repeated.Changed.Should().BeFalse(); + repeated.Inventory.Where(i => i.Path.EndsWith("/a", StringComparison.Ordinal) || i.Path.EndsWith("/b", StringComparison.Ordinal)).Should().OnlyContain(i => i.Kind == WorkspaceInventoryKind.AlreadyMigrated); fixture.Service.Apply(repeated).Status.Should().Be(ConfigurationTransactionStatus.Unchanged); - foreach(var path in targets){File.ReadAllBytes(path).Should().Equal(original[path]);File.GetLastWriteTimeUtc(path).Should().Be(time);} - foreach(var source in sources)File.ReadAllBytes(source.Key).Should().Equal(source.Value); + foreach (var path in targets) { File.ReadAllBytes(path).Should().Equal(original[path]); File.GetLastWriteTimeUtc(path).Should().Be(time); } + foreach (var source in sources) File.ReadAllBytes(source.Key).Should().Equal(source.Value); } [Test] [Arguments("source")] @@ -142,53 +149,56 @@ public void BatchPreservesSourcesAndEquivalentRepeatBytesTimestampsAndInventory( [Arguments("primary")] public void AggregateRechecksEveryRelevantPreconditionBeforeFirstReplacement(string drift) { - using var fixture=new WorkspaceFixture(); - fixture.Write("a/morphir.json","{name:'A',exposedModules:[]}"); - fixture.Write("b/morphir.json","{name:'B',exposedModules:[]}"); - var preview=fixture.Service.Plan(fixture.Root); preview.CanApply.Should().BeTrue(); - switch(drift){case "source": fixture.Write("b/morphir.json","{name:'Changed',exposedModules:[]}");break; - case "target":fixture.Write("a/morphir.toml","[custom]\nuser=true\n");break; - case "listing":fixture.Write("new/morphir.json","{name:'New',exposedModules:[]}");break; - default:fixture.Write("b/.config/morphir/config.toml","[custom]\nuser=true\n");break;} + using var fixture = new WorkspaceFixture(); + fixture.Write("a/morphir.json", "{name:'A',exposedModules:[]}"); + fixture.Write("b/morphir.json", "{name:'B',exposedModules:[]}"); + var preview = fixture.Service.Plan(fixture.Root); preview.CanApply.Should().BeTrue(); + switch (drift) + { + case "source": fixture.Write("b/morphir.json", "{name:'Changed',exposedModules:[]}"); break; + case "target": fixture.Write("a/morphir.toml", "[custom]\nuser=true\n"); break; + case "listing": fixture.Write("new/morphir.json", "{name:'New',exposedModules:[]}"); break; + default: fixture.Write("b/.config/morphir/config.toml", "[custom]\nuser=true\n"); break; + } fixture.Service.Apply(preview).Status.Should().Be(ConfigurationTransactionStatus.Conflict); - File.Exists(fixture.Root+"/morphir.toml").Should().BeFalse(); - File.Exists(fixture.Root+"/b/morphir.toml").Should().BeFalse(); + File.Exists(fixture.Root + "/morphir.toml").Should().BeFalse(); + File.Exists(fixture.Root + "/b/morphir.toml").Should().BeFalse(); } [Test] public void BlockedMemberPreventsEveryWriteAndReadonlyPlansCannotBeTransferred() { - using var fixture=new WorkspaceFixture(); - fixture.Write("a/morphir.json","{name:'A',exposedModules:[]}"); - fixture.Write("b/morphir.json","{name:'B',exposedModules:[],dependencies:[{env:'secret-reference'}]}"); - var blocked=fixture.Service.Plan(fixture.Root);blocked.CanApply.Should().BeFalse();fixture.Service.Apply(blocked).Status.Should().Be(ConfigurationTransactionStatus.IoFailure); - Directory.GetFiles(fixture.Root,"*.toml",SearchOption.AllDirectories).Should().BeEmpty(); - fixture.Write("b/morphir.json","{name:'B',exposedModules:[]}"); - var readOnly=new WorkspaceMigrationService(new FileSystemProjectTree(fixture.Root)); - var preview=readOnly.Plan(fixture.Root);preview.CanApply.Should().BeTrue(); - readOnly.Apply(preview).Code.Should().Be("workspace.migrate_unverified");fixture.Service.Apply(preview).Code.Should().Be("workspace.migrate_unverified"); + using var fixture = new WorkspaceFixture(); + fixture.Write("a/morphir.json", "{name:'A',exposedModules:[]}"); + fixture.Write("b/morphir.json", "{name:'B',exposedModules:[],dependencies:[{env:'secret-reference'}]}"); + var blocked = fixture.Service.Plan(fixture.Root); blocked.CanApply.Should().BeFalse(); fixture.Service.Apply(blocked).Status.Should().Be(ConfigurationTransactionStatus.IoFailure); + Directory.GetFiles(fixture.Root, "*.toml", SearchOption.AllDirectories).Should().BeEmpty(); + fixture.Write("b/morphir.json", "{name:'B',exposedModules:[]}"); + var readOnly = new WorkspaceMigrationService(new FileSystemProjectTree(fixture.Root)); + var preview = readOnly.Plan(fixture.Root); preview.CanApply.Should().BeTrue(); + readOnly.Apply(preview).Code.Should().Be("workspace.migrate_unverified"); fixture.Service.Apply(preview).Code.Should().Be("workspace.migrate_unverified"); } [Test] public void ExistingEmptyVirtualWorkspaceIsUnchangedButUndeclaredEmptyRootCannotAdopt() { - using var fixture=new WorkspaceFixture(); + using var fixture = new WorkspaceFixture(); fixture.Service.Plan(fixture.Root).CanApply.Should().BeFalse(); - fixture.Write("morphir.toml","# keep\n[workspace]\nmembers=[]\n"); - var preview=fixture.Service.Plan(fixture.Root);preview.CanApply.Should().BeTrue();preview.Changed.Should().BeFalse(); + fixture.Write("morphir.toml", "# keep\n[workspace]\nmembers=[]\n"); + var preview = fixture.Service.Plan(fixture.Root); preview.CanApply.Should().BeTrue(); preview.Changed.Should().BeFalse(); fixture.Service.Apply(preview).Status.Should().Be(ConfigurationTransactionStatus.Unchanged); - File.ReadAllText(fixture.Root+"/morphir.toml").Should().Be("# keep\n[workspace]\nmembers=[]\n"); + File.ReadAllText(fixture.Root + "/morphir.toml").Should().Be("# keep\n[workspace]\nmembers=[]\n"); } } -internal sealed class WorkspaceFixture:IDisposable +internal sealed class WorkspaceFixture : IDisposable { - public string Root {get;}=ProjectPath.join(Path.GetTempPath(),"workspace-migration-"+Guid.NewGuid().ToString("N")); - public WorkspaceMigrationService Service {get;} - public WorkspaceFixture(){Directory.CreateDirectory(Root);Service=new(new FileSystemProjectTree(Root),new NativeConfigurationTransactionHost());} - public void Write(string path,string text){path=ProjectPath.join(Root,path);Directory.CreateDirectory(ProjectPath.parent(path));File.WriteAllText(path,text);} + public string Root { get; } = ProjectPath.join(Path.GetTempPath(), "workspace-migration-" + Guid.NewGuid().ToString("N")); + public WorkspaceMigrationService Service { get; } + public WorkspaceFixture() { Directory.CreateDirectory(Root); Service = new(new FileSystemProjectTree(Root), new NativeConfigurationTransactionHost()); } + public void Write(string path, string text) { path = ProjectPath.join(Root, path); Directory.CreateDirectory(ProjectPath.parent(path)); File.WriteAllText(path, text); } public void Dispose() { - if(Directory.EnumerateFileSystemEntries(Root,".morphir-edit-*",SearchOption.AllDirectories).Any())Console.Error.WriteLine("Retained workspace recovery evidence: "+Root); - else Directory.Delete(Root,true); + if (Directory.EnumerateFileSystemEntries(Root, ".morphir-edit-*", SearchOption.AllDirectories).Any()) Console.Error.WriteLine("Retained workspace recovery evidence: " + Root); + else Directory.Delete(Root, true); } } @@ -197,11 +207,11 @@ public sealed class WorkspaceMigrationBoundaryTests [Test] public void DiscoveredLinksAndReversedHostOrderDoNotChangeInventoryOrEscapeRoot() { - var files=new Dictionary{["/repo/a/morphir.json"]="{name:'A',exposedModules:[]}",["/repo/z/morphir.json"]="{name:'Z',exposedModules:[]}"}; - var first=new WorkspaceMigrationService(new OrderAndLinkTree(new InMemoryProjectTree("/",files),false)).Plan("/repo"); - var second=new WorkspaceMigrationService(new OrderAndLinkTree(new InMemoryProjectTree("/",files),true)).Plan("/repo"); - first.CanApply.Should().BeTrue();second.CanApply.Should().BeTrue();second.Inventory.Should().Equal(first.Inventory);second.Members.Should().Equal(first.Members); - first.Inventory.Should().Contain(i=>i.RelativePath=="linked"&&i.Kind==WorkspaceInventoryKind.LinkSkipped); + var files = new Dictionary { ["/repo/a/morphir.json"] = "{name:'A',exposedModules:[]}", ["/repo/z/morphir.json"] = "{name:'Z',exposedModules:[]}" }; + var first = new WorkspaceMigrationService(new OrderAndLinkTree(new InMemoryProjectTree("/", files), false)).Plan("/repo"); + var second = new WorkspaceMigrationService(new OrderAndLinkTree(new InMemoryProjectTree("/", files), true)).Plan("/repo"); + first.CanApply.Should().BeTrue(); second.CanApply.Should().BeTrue(); second.Inventory.Should().Equal(first.Inventory); second.Members.Should().Equal(first.Members); + first.Inventory.Should().Contain(i => i.RelativePath == "linked" && i.Kind == WorkspaceInventoryKind.LinkSkipped); } [Test] [Arguments("../outside")] @@ -209,57 +219,60 @@ public void DiscoveredLinksAndReversedHostOrderDoNotChangeInventoryOrEscapeRoot( [Arguments("")] public void InvalidFiltersAreStructuredAndCannotBroadenTheBoundary(string filter) { - var plan=new WorkspaceMigrationService(new InMemoryProjectTree("/repo",new Dictionary{["/repo/a/morphir.json"]="{name:'A',exposedModules:[]}"})).Plan("/repo",[filter]); - plan.CanApply.Should().BeFalse();plan.Diagnostics.Should().Contain(d=>d.Code=="workspace.migrate_filter"); + var plan = new WorkspaceMigrationService(new InMemoryProjectTree("/repo", new Dictionary { ["/repo/a/morphir.json"] = "{name:'A',exposedModules:[]}" })).Plan("/repo", [filter]); + plan.CanApply.Should().BeFalse(); plan.Diagnostics.Should().Contain(d => d.Code == "workspace.migrate_filter"); } [Test] public void ExcludedRootClassicStillSuppliesActualBaseSettingsForAdoptedMembers() { - var plan=new WorkspaceMigrationService(new InMemoryProjectTree("/repo",new Dictionary{ - ["/repo/morphir.json"]="{name:'Root',exposedModules:[],pipeline:{backend:'root-specific'}}", - ["/repo/app/morphir.json"]="{name:'App',exposedModules:[]}" - })).Plan("/repo",["app"]); + var plan = new WorkspaceMigrationService(new InMemoryProjectTree("/repo", new Dictionary + { + ["/repo/morphir.json"] = "{name:'Root',exposedModules:[],pipeline:{backend:'root-specific'}}", + ["/repo/app/morphir.json"] = "{name:'App',exposedModules:[]}" + })).Plan("/repo", ["app"]); plan.CanApply.Should().BeFalse("retained root settings will inherit after adoption even if its conversion is filtered out"); - plan.Diagnostics.Should().Contain(d=>d.Code=="workspace.migrate_adoption_value"); + plan.Diagnostics.Should().Contain(d => d.Code == "workspace.migrate_adoption_value"); } [Test] public void ConfiguredOutputDirectoriesAreExcludedWithoutReadingTheirContents() { - var tree=new InMemoryProjectTree("/repo",new Dictionary{ - ["/repo/morphir.toml"]="[workspace]\nmembers=[]\nout_dir='generated-artifacts'\n", - ["/repo/app/morphir.json"]="{name:'App',exposedModules:[]}", - ["/repo/generated-artifacts/fake/morphir.json"]="{name:'Generated',exposedModules:[]}" + var tree = new InMemoryProjectTree("/repo", new Dictionary + { + ["/repo/morphir.toml"] = "[workspace]\nmembers=[]\nout_dir='generated-artifacts'\n", + ["/repo/app/morphir.json"] = "{name:'App',exposedModules:[]}", + ["/repo/generated-artifacts/fake/morphir.json"] = "{name:'Generated',exposedModules:[]}" }); - var plan=new WorkspaceMigrationService(tree).Plan("/repo"); - plan.CanApply.Should().BeTrue();plan.Members.Should().Equal("app"); - tree.ObservedPaths.Should().NotContain(p=>p.StartsWith("/repo/generated-artifacts/",StringComparison.Ordinal)); - plan.Inventory.Should().Contain(i=>i.RelativePath=="generated-artifacts"&&i.Kind==WorkspaceInventoryKind.Excluded); + var plan = new WorkspaceMigrationService(tree).Plan("/repo"); + plan.CanApply.Should().BeTrue(); plan.Members.Should().Equal("app"); + tree.ObservedPaths.Should().NotContain(p => p.StartsWith("/repo/generated-artifacts/", StringComparison.Ordinal)); + plan.Inventory.Should().Contain(i => i.RelativePath == "generated-artifacts" && i.Kind == WorkspaceInventoryKind.Excluded); } [Test] public void MissingDeclaredDefaultIsInvalidRatherThanSuccessfulNoop() { - var plan=new WorkspaceMigrationService(new InMemoryProjectTree("/repo",new Dictionary{ - ["/repo/morphir.toml"]="[workspace]\nmembers=[]\ndefault_member='absent'\n" + var plan = new WorkspaceMigrationService(new InMemoryProjectTree("/repo", new Dictionary + { + ["/repo/morphir.toml"] = "[workspace]\nmembers=[]\ndefault_member='absent'\n" })).Plan("/repo"); - plan.CanApply.Should().BeFalse();plan.Diagnostics.Should().Contain(d=>d.Code=="workspace.migrate_default"); + plan.CanApply.Should().BeFalse(); plan.Diagnostics.Should().Contain(d => d.Code == "workspace.migrate_default"); } [Test] public void GlobCharactersCannotPretendToBeAnExplicitLiteralMemberPath() { - var plan=new WorkspaceMigrationService(new InMemoryProjectTree("/repo",new Dictionary{["/repo/a?/morphir.json"]="{name:'A',exposedModules:[]}"})).Plan("/repo"); - plan.CanApply.Should().BeFalse();plan.Diagnostics.Should().Contain(d=>d.Code=="workspace.migrate_member_path"); + var plan = new WorkspaceMigrationService(new InMemoryProjectTree("/repo", new Dictionary { ["/repo/a?/morphir.json"] = "{name:'A',exposedModules:[]}" })).Plan("/repo"); + plan.CanApply.Should().BeFalse(); plan.Diagnostics.Should().Contain(d => d.Code == "workspace.migrate_member_path"); } - private sealed class OrderAndLinkTree(IProjectTree inner,bool reverse):IProjectTree + private sealed class OrderAndLinkTree(IProjectTree inner, bool reverse) : IProjectTree { - public string Boundary=>inner.Boundary; + public string Boundary => inner.Boundary; public ProjectEntry Observe(string path) { - if(!ProjectPath.contains("/repo",path))throw new InvalidOperationException("Scanned outside supplied root"); - if(path=="/repo/linked")return ProjectEntry.Link; - var entry=inner.Observe(path); - if(entry is not ProjectEntry.Directory directory)return entry; - var children=directory.children.AsEnumerable();if(path=="/repo")children=children.Append("/repo/linked"); - if(reverse)children=children.Reverse();return ProjectEntry.NewDirectory(Microsoft.FSharp.Collections.ListModule.OfSeq(children)); + if (!ProjectPath.contains("/repo", path)) throw new InvalidOperationException("Scanned outside supplied root"); + if (path == "/repo/linked") return ProjectEntry.Link; + var entry = inner.Observe(path); + if (entry is not ProjectEntry.Directory directory) return entry; + var children = directory.children.AsEnumerable(); if (path == "/repo") children = children.Append("/repo/linked"); + if (reverse) children = children.Reverse(); return ProjectEntry.NewDirectory(Microsoft.FSharp.Collections.ListModule.OfSeq(children)); } } } @@ -269,20 +282,20 @@ public sealed class WorkspaceMigrationRecoveryPreviewTests [Test] public void PendingJournalMakesPreviewRecoveryRequiredBeforeAnyNewWrite() { - using var fixture=new WorkspaceFixture(); - fixture.Write("a/morphir.json","{name:'A',exposedModules:[]}"); - var target=fixture.Root+"/a/morphir.toml"; - var initial=new ConfigurationTransactionPlan([ConfigurationTransaction.Observe(fixture.Root)], - [new ConfigurationFileChange(target,ConfigurationTransaction.Observe(target),System.Text.Encoding.UTF8.GetBytes("[project]\nname='A'\n"))]); - var failed=new ConfigurationTransaction(fault:(phase,_)=>{if(phase is ConfigurationTransactionPhase.AfterInstall or ConfigurationTransactionPhase.BeforeRollback)throw new IOException("injected interrupted batch");}).Apply(initial); + using var fixture = new WorkspaceFixture(); + fixture.Write("a/morphir.json", "{name:'A',exposedModules:[]}"); + var target = fixture.Root + "/a/morphir.toml"; + var initial = new ConfigurationTransactionPlan([ConfigurationTransaction.Observe(fixture.Root)], + [new ConfigurationFileChange(target, ConfigurationTransaction.Observe(target), System.Text.Encoding.UTF8.GetBytes("[project]\nname='A'\n"))]); + var failed = new ConfigurationTransaction(fault: (phase, _) => { if (phase is ConfigurationTransactionPhase.AfterInstall or ConfigurationTransactionPhase.BeforeRollback) throw new IOException("injected interrupted batch"); }).Apply(initial); failed.Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); - var journal=failed.Artifacts.First(a=>a.Kind==ConfigurationArtifactKind.Journal).Path; + var journal = failed.Artifacts.First(a => a.Kind == ConfigurationArtifactKind.Journal).Path; try { - var preview=fixture.Service.Plan(fixture.Root); + var preview = fixture.Service.Plan(fixture.Root); preview.CanApply.Should().BeFalse("pending recovery makes the plan unavailable even in dry-run"); - preview.Diagnostics.Should().Contain(d=>d.Code=="workspace.migrate_recovery_required"); - File.Exists(fixture.Root+"/morphir.toml").Should().BeFalse(); + preview.Diagnostics.Should().Contain(d => d.Code == "workspace.migrate_recovery_required"); + File.Exists(fixture.Root + "/morphir.toml").Should().BeFalse(); } finally { @@ -296,21 +309,23 @@ public sealed class WorkspaceMigrationDeclaredControlTests [Test] public void AdoptionCannotHideChangedProfileDeclarationsBehindDisabledActivation() { - var plan=new WorkspaceMigrationService(new InMemoryProjectTree("/repo",new Dictionary{ - ["/repo/morphir.json"]="{name:'Root',exposedModules:[],profiles:{default:['root-only']}}", - ["/repo/app/morphir.json"]="{name:'App',exposedModules:[]}" + var plan = new WorkspaceMigrationService(new InMemoryProjectTree("/repo", new Dictionary + { + ["/repo/morphir.json"] = "{name:'Root',exposedModules:[],profiles:{default:['root-only']}}", + ["/repo/app/morphir.json"] = "{name:'App',exposedModules:[]}" })).Plan("/repo"); plan.CanApply.Should().BeFalse("disabled activation must not hide newly inherited declaration controls"); - plan.Diagnostics.Should().Contain(d=>d.Code=="workspace.migrate_adoption_value"); + plan.Diagnostics.Should().Contain(d => d.Code == "workspace.migrate_adoption_value"); } [Test] public void EveryUnicodeKeySegmentStillUsesTheProjectLossLedger() { - var plan=new WorkspaceMigrationService(new InMemoryProjectTree("/repo",new Dictionary{ - ["/repo/a/morphir.json"]="{name:'A',exposedModules:[],custom:{'\\ud800':{nested:'retained'}}}" + var plan = new WorkspaceMigrationService(new InMemoryProjectTree("/repo", new Dictionary + { + ["/repo/a/morphir.json"] = "{name:'A',exposedModules:[],custom:{'\\ud800':{nested:'retained'}}}" })).Plan("/repo"); - plan.CanApply.Should().BeFalse();plan.Projects.Should().ContainSingle(); - plan.Projects[0].Presentation!.Ledger.ToArray().Should().Contain(f=>f.Disposition=="blocking"); + plan.CanApply.Should().BeFalse(); plan.Projects.Should().ContainSingle(); + plan.Projects[0].Presentation!.Ledger.ToArray().Should().Contain(f => f.Disposition == "blocking"); } } @@ -321,15 +336,16 @@ public sealed class WorkspaceMigrationLayoutTests [Arguments(false)] public void AggregateReusesExistingSupportedTomlLayoutRatherThanCreatingAmbiguity(bool rootProject) { - var prefix=rootProject?"":"app/"; - var files=new Dictionary{ - ["/repo/"+prefix+".morphir/morphir.json"]="{name:'A',exposedModules:[]}", - ["/repo/"+prefix+"morphir.toml"]="# keep\n[project]\nname='A'\n" + var prefix = rootProject ? "" : "app/"; + var files = new Dictionary + { + ["/repo/" + prefix + ".morphir/morphir.json"] = "{name:'A',exposedModules:[]}", + ["/repo/" + prefix + "morphir.toml"] = "# keep\n[project]\nname='A'\n" }; - if(!rootProject)files["/repo/morphir.toml"]="[workspace]\nmembers=['app']\n"; - var plan=new WorkspaceMigrationService(new InMemoryProjectTree("/repo",files)).Plan("/repo"); - plan.CanApply.Should().BeTrue(string.Join(",",plan.Diagnostics.Select(d=>d.Code))); - plan.Projects.Should().ContainSingle().Which.TargetPath.Should().Be("/repo/"+prefix+"morphir.toml"); + if (!rootProject) files["/repo/morphir.toml"] = "[workspace]\nmembers=['app']\n"; + var plan = new WorkspaceMigrationService(new InMemoryProjectTree("/repo", files)).Plan("/repo"); + plan.CanApply.Should().BeTrue(string.Join(",", plan.Diagnostics.Select(d => d.Code))); + plan.Projects.Should().ContainSingle().Which.TargetPath.Should().Be("/repo/" + prefix + "morphir.toml"); plan.Projects[0].ProjectPath.Should().Be(rootProject ? "/repo" : "/repo/app"); } } @@ -339,9 +355,9 @@ public sealed class WorkspaceMigrationClassicSourcePolicyTests [Test] public void LegacyJson5PrimaryCannotBeMisreportedAsAlreadyMigrated() { - var plan=new WorkspaceMigrationService(new InMemoryProjectTree("/repo",new Dictionary{["/repo/a/morphir.json5"]="{name:'A',exposedModules:[]}"})).Plan("/repo"); + var plan = new WorkspaceMigrationService(new InMemoryProjectTree("/repo", new Dictionary { ["/repo/a/morphir.json5"] = "{name:'A',exposedModules:[]}" })).Plan("/repo"); plan.CanApply.Should().BeFalse("batch cannot bypass the single-project source contract by calling legacy JSON5 migrated"); - plan.Diagnostics.Should().Contain(d=>d.Code=="migration.classic_required"); - plan.Inventory.Should().Contain(i=>i.RelativePath=="a"&&i.Kind==WorkspaceInventoryKind.Candidate); + plan.Diagnostics.Should().Contain(d => d.Code == "migration.classic_required"); + plan.Inventory.Should().Contain(i => i.RelativePath == "a" && i.Kind == WorkspaceInventoryKind.Candidate); } } From a4d2ee36888dc022a255a8808b497186aaf9897c Mon Sep 17 00:00:00 2001 From: Damian Reeves <957246+DamianReeves@users.noreply.github.com> Date: Tue, 3 Nov 2026 17:55:26 -0600 Subject: [PATCH 06/14] docs: qualify and archive workspace migration Record final executable, coverage and native-host evidence. Keep parent review, landing and cleanup gates explicit; preserve unresolved fault-fixture recovery evidence. --- docs/content/docs/cli/_index.md | 2 +- docs/content/docs/cli/config.md | 2 +- docs/content/docs/cli/troubleshooting.md | 2 +- .../docs/contributing/design/prds/_index.md | 2 +- .../design/prds/layered-configuration.md | 2 +- .../prds/workspace-migration-delivery.md | 158 ++++++++++++++++++ .../docs/getting-started/configuration.md | 2 +- docs/content/guides/configuration.md | 6 +- .../.openspec.yaml | 0 .../design.html | 2 +- .../proposal.md | 6 +- .../specs/workspace-migration/spec.md | 0 .../tasks.md | 2 +- openspec/specs/workspace-migration/spec.md | 96 +++++++++++ 14 files changed, 268 insertions(+), 14 deletions(-) rename openspec/changes/{add-workspace-migration => archive/2026-11-03-add-workspace-migration}/.openspec.yaml (100%) rename openspec/changes/{add-workspace-migration => archive/2026-11-03-add-workspace-migration}/design.html (98%) rename openspec/changes/{add-workspace-migration => archive/2026-11-03-add-workspace-migration}/proposal.md (89%) rename openspec/changes/{add-workspace-migration => archive/2026-11-03-add-workspace-migration}/specs/workspace-migration/spec.md (100%) rename openspec/changes/{add-workspace-migration => archive/2026-11-03-add-workspace-migration}/tasks.md (98%) create mode 100644 openspec/specs/workspace-migration/spec.md diff --git a/docs/content/docs/cli/_index.md b/docs/content/docs/cli/_index.md index 80d8bf80..90fd8579 100644 --- a/docs/content/docs/cli/_index.md +++ b/docs/content/docs/cli/_index.md @@ -85,7 +85,7 @@ These exit codes are designed for CI/CD integration and scripting. The [configuration guide]({{< relref "/guides/configuration" >}}) distinguishes the compatibility TOML cache loader from the additive shared configuration and project system. -The [morphir config read commands]({{< relref "/docs/cli/config" >}}) inspect effective/scoped values, source reports and redacted explanations. JSON output follows the published schema; success, missing keys and usage/loading failures use exit codes 0/1/2. Shared `config set/unset` editing and [project migrate]({{< relref "/docs/cli/project-migrate" >}}) are available; workspace migration remains planned. Start with the [workspace configuration walkthrough]({{< relref "/docs/getting-started/configuration" >}}). +The [morphir config read commands]({{< relref "/docs/cli/config" >}}) inspect effective/scoped values, source reports and redacted explanations. JSON output follows the published schema; success, missing keys and usage/loading failures use exit codes 0/1/2. Shared `config set/unset` editing, [project migrate]({{< relref "/docs/cli/project-migrate" >}}) and bounded [workspace migrate]({{< relref "/docs/cli/workspace-migrate" >}}) are available. Start with the [workspace configuration walkthrough]({{< relref "/docs/getting-started/configuration" >}}). ## Troubleshooting diff --git a/docs/content/docs/cli/config.md b/docs/content/docs/cli/config.md index 6753bf42..55885a6d 100644 --- a/docs/content/docs/cli/config.md +++ b/docs/content/docs/cli/config.md @@ -240,7 +240,7 @@ Native filesystem access keeps the existing no-follow policy. On macOS, the conv ## Project migration -Use [project migrate]({{< relref "/docs/cli/project-migrate" >}}) to adopt one classic project through loss-checked shared planning and guarded application. Workspace migration remains planned. Editing does not remove classic manifests, perform migration equivalence checks or adopt workspace membership automatically. +Use [project migrate]({{< relref "/docs/cli/project-migrate" >}}) to adopt one classic project, or [workspace migrate]({{< relref "/docs/cli/workspace-migrate" >}}) for a bounded repository. Both use loss-checked shared planning and guarded application. Editing does not remove classic manifests, perform migration equivalence checks or adopt workspace membership automatically. ## Related documentation diff --git a/docs/content/docs/cli/troubleshooting.md b/docs/content/docs/cli/troubleshooting.md index db8b5cae..94bd8c14 100644 --- a/docs/content/docs/cli/troubleshooting.md +++ b/docs/content/docs/cli/troubleshooting.md @@ -155,7 +155,7 @@ Message: Malformed JSON: 'i' is an invalid start of a value. LineNumber: 6 ## Configuration -The shared libraries and `morphir config get/list/show/path/explain` provide multi-format resolution, bounded discovery and inspection. Configuration editing uses shared plan/apply services; classic project migration is available and workspace migration remains planned. Existing artifact/cache tooling retains TOML cache settings and Git-first discovery. See the [guide]({{< relref "/guides/configuration" >}}) for the library APIs and the [command reference]({{< relref "/docs/cli/config" >}}) for read/write options and recovery semantics. +The shared libraries and `morphir config get/list/show/path/explain` provide multi-format resolution, bounded discovery and inspection. Configuration editing, classic project migration and bounded workspace migration use shared plan/apply services. Existing artifact/cache tooling retains TOML cache settings and Git-first discovery. See the [guide]({{< relref "/guides/configuration" >}}) for the library APIs and the [command reference]({{< relref "/docs/cli/config" >}}) for read/write options and recovery semantics. ### Wrong workspace or missing project diff --git a/docs/content/docs/contributing/design/prds/_index.md b/docs/content/docs/contributing/design/prds/_index.md index cb648967..f087282f 100644 --- a/docs/content/docs/contributing/design/prds/_index.md +++ b/docs/content/docs/contributing/design/prds/_index.md @@ -15,7 +15,7 @@ Product Requirements Documents (PRDs) track feature requirements, design decisio | [IR JSON Schema Verification BDD](ir-json-schema-verification-bdd) | 🚧 In Progress | BDD scenarios for schema verification | | [Deployment Architecture Refactor](deployment-architecture-refactor) | 📋 Draft | Build and deployment improvements | | [F# backend / Myriad delivery](https://github.com/finos/morphir-dotnet/blob/main/docs/design/PRD-fsharp-backend.md) | In Progress | Classic v1-v3 to F#, per-module Myriad generation and car rental tutorial | -| [Layered Configuration](layered-configuration) | In Progress | Campaigns 1 through 5 landed and cleaned; workspace migration in progress under #479 | +| [Layered Configuration](layered-configuration) | In Progress | Campaigns 1 through 5 landed and cleaned; workspace migration locally qualified and archived under #479; parent review/CI/landing/cleanup pending | | [Product Manager Skill](product-manager-skill) | 📋 Draft | AI skill for product management | ## Status Legend diff --git a/docs/content/docs/contributing/design/prds/layered-configuration.md b/docs/content/docs/contributing/design/prds/layered-configuration.md index 88c42158..10e8a97a 100644 --- a/docs/content/docs/contributing/design/prds/layered-configuration.md +++ b/docs/content/docs/contributing/design/prds/layered-configuration.md @@ -323,4 +323,4 @@ Campaign 6 tracks [#479](https://github.com/finos/morphir-dotnet/issues/479). The [delivery journal](workspace-migration-delivery.md) records ownership, prerequisite ancestry, task progress and qualification. PR #477 landed at `e3b9a053120a49e94c70c30cd4f62ef783672c64`; its receipt confirms campaign 5 -cleanup. Parent owns campaign 6 review, CI watch, landing and cleanup. +cleanup. Implementation is locally qualified (all nine tasks), with the OpenSpec change synchronized and archived at `openspec/changes/archive/2026-11-03-add-workspace-migration/`. Parent owns campaign 6 review, CI watch, landing and cleanup; the campaign remains incomplete. diff --git a/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md b/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md index 7425b567..6e7ee40e 100644 --- a/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md +++ b/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md @@ -212,3 +212,161 @@ retain evidence and durable overlap guards. Inspection exposes unresolved paths after process-local result loss. Deliberately unresolved fault fixtures remain under the owned external artifact root for parent inspection; they are not silently cleaned or represented as successful recovery. + +## Final runtime qualification + +Runtime checkpoint: `de44fa5` (`fix: verify aggregate context and artifact ownership`), +based on exact main `e3b9a053120a49e94c70c30cd4f62ef783672c64`. +`qualified-source-checkpoint.json` records hashes of every changed file. Subsequent +delivery edits are documentation/specification only. All following commands run +from the owned checkout, one heavy operation at a time, with +`DOTNET_PROCESSOR_COUNT=2`, `BuildInParallel=false`, `RestoreDisableParallel=true` +and `TMPDIR`, `TEMP`, `TMP` set to `/private/tmp/morphir-workspace-migration/tmp`. + +- Locked solution restore: `dotnet restore Morphir.slnx --locked-mode -m:1 -p:BuildInParallel=false -p:RestoreDisableParallel=true`. +- Final build/analyzers: `dotnet build Morphir.slnx --no-restore -m:1 -p:BuildInParallel=false`; zero errors, ten existing package/generated warnings. Actual CLI rebuilt before subprocess qualification. +- Changed C# formatter: `dotnet format Morphir.slnx --no-restore --include --verbosity quiet`; exact expanded argv is in `qualification-first-summary.json`. Full formatter verification uses `dotnet format Morphir.slnx --no-restore --verify-no-changes --verbosity quiet`. +- F# formatting: `dotnet fantomas --check src/Morphir.Configuration/Migration.fs src/Morphir.Configuration/ProjectDiscovery.fs`. +- Managed host: `dotnet tests/Morphir.Configuration.IO.Smoke/bin/Debug/net10.0/Morphir.Configuration.IO.Smoke.dll --fixtures tests/Fixtures/Configuration`. + +The final configured executable sweep passes all 17 runners, 3,887 tests and +smoke checks. Each invocation is `dotnet tests//bin/Debug/net10.0/.dll` +with the flag below. The process-local PATH adds campaign `bin/dotnet` for serial +nested builds and existing PowerShell 7.5.3 for F# SDK package tests; +`DOTNET_HOST_PATH` points to that campaign wrapper. Global tools/caches are untouched. + +| Project | Runner flags | Passed | +|---|---|---:| +| Morphir.CodeGeneration.FSharp.Tests | `--sequenced` | 333 | +| Morphir.Configuration.IO.Tests | `--maximum-parallel-tests 1` | 429 | +| Morphir.Configuration.Inspection.Smoke | none | 71 | +| Morphir.Configuration.Microsoft.Tests | `--maximum-parallel-tests 1` | 22 | +| Morphir.Configuration.Tests | `--sequenced` | 177 | +| Morphir.Core.Tests | `--maximum-parallel-tests 1` | 237 | +| Morphir.IR.Pipeline.Plugins.Tests | `--sequenced` | 141 | +| Morphir.IR.Pipeline.Tests | `--sequenced` | 299 | +| Morphir.Internal.CodeGeneration.Tests | `--sequenced` | 25 | +| Morphir.Ion.Tests | `--sequenced` | 463 | +| Morphir.Models.Tests | `--sequenced` | 791 | +| Morphir.Myriad.Integration.Tests | `--maximum-parallel-tests 1` | 12 | +| Morphir.Myriad.Tests | `--sequenced` | 4 | +| Morphir.SDK.CSharp.Tests | `--maximum-parallel-tests 1` | 20 | +| Morphir.SDK.Conformance.Tests | `--maximum-parallel-tests 1` | 27 | +| Morphir.SDK.Tests | `--sequenced` | 403 | +| Morphir.Tooling.Tests | `--maximum-parallel-tests 1` | 433 | + +`validation/final-receipt.json` preserves expanded commands, durations and counts; +individual logs preserve actual runner discovery. IO includes shared/project/ +workspace BDD, full aggregate fault/recovery/ownership and separate-process +interruption/overlap tests. Tooling includes 31 real workspace subprocess BDD +scenarios and fresh-registry JSON-schema validation. Models/SDK runners retain +codec/roundtrip contracts. Plain `dotnet test` is not substituted for these runners. + +Coverage commands: + +```sh +dotnet tests/Morphir.Configuration.IO.Tests/bin/Debug/net10.0/Morphir.Configuration.IO.Tests.dll \ + --maximum-parallel-tests 1 --coverage --coverage-output-format cobertura \ + --coverage-output /private/tmp/morphir-workspace-migration/io.cobertura.xml +dotnet test tests/Morphir.Configuration.Tests/Morphir.Configuration.Tests.fsproj \ + --no-build --no-restore -m:1 -p:BuildInParallel=false \ + --collect:"XPlat Code Coverage" \ + --results-directory /private/tmp/morphir-workspace-migration/core-coverage +/usr/bin/python3 scripts/configuration/summarize-coverage.py \ + /private/tmp/morphir-workspace-migration/core-coverage/*/coverage.cobertura.xml \ + /private/tmp/morphir-workspace-migration/io.cobertura.xml +``` + +Final source coverage: core 4,188/4,345 (96.39%); IO 2,495/2,561 (97.42%). +Both exceed prerequisite source checkpoints. The core collector is supplementary; +177 sequenced Expecto tests establish actual test execution. No source exclusions +or warning suppressions were added to meet qualification. + +Managed, trimmed and native non-CLI batch hosts each pass 38 workspace checks, +32 project migration checks, 44 editing checks (9 editor, 22 shared, 13 transaction), +228 official JSON5 syntax cases and 26 pinned Morphir documents. They exercise +read-only/native preview, aggregate application, no-op bytes/timestamps, source +and generated-output retention, drift, redacted generated serialization, explicit +journal recovery and uncertain-directory preservation. Native output is a Mach-O +64-bit arm64 executable. Exact publish argv and exits are in `publish/summary.json`: + +```sh +dotnet publish tests/Morphir.Configuration.IO.Smoke/Morphir.Configuration.IO.Smoke.csproj \ + -c Release -r osx-arm64 -m:1 -p:BuildInParallel=false \ + -p:RestoreDisableParallel=true -p:PublishTrimmed= \ + -p:PublishAot= -p:ILLinkTreatWarningsAsErrors=false \ + -p:TrimmerSingleWarn=false -p:JsonSerializerIsReflectionEnabledByDefault=false \ + -o /private/tmp/morphir-workspace-migration/publish/ +/private/tmp/morphir-workspace-migration/publish//Morphir.Configuration.IO.Smoke \ + --fixtures /Users/damian/.t3/worktrees/morphir-dotnet/t3-f370bf10/tests/Fixtures/Configuration +``` + +The three modes run sequentially with the corresponding values above. Managed +publish has zero warning lines; trimmed retains 44 and native retains 47 pinned +CsToml/FSharp.Core analysis-warning lines. These remain visible upstream +limitations, not a strict warning-free trimming claim. No suppression or package +update was introduced. `publish-locks/restored.json` proves all 45 lockfiles were +restored byte-for-byte after publishing. Reports and stderr are retained separately. + +## Specification sync and handoff gates + +Full formatter verification and final locked restore pass. OpenSpec apply returned +`all_done`, 9/9, before archive. The current specs-rule snapshot was read before +writing the new main spec. Only `add-workspace-migration` was synchronized and +archived at `openspec/changes/archive/2026-11-03-add-workspace-migration/`. +`openspec/specs/workspace-migration/spec.md` preserves the approved Purpose and +all 11 requirement bodies/14 scenarios byte-for-byte under canonical main-spec +headings. `archive-verification.json` verifies exact content, nine checked tasks, +retained `.openspec.yaml`, balanced HTML and valid local design/proposal links. +The archive's prerequisite and PRD links are adjusted for archive depth. + +`MISE_TRUSTED_CONFIG_PATHS="$PWD" mise run openspec:verify` passes before and +after archive: managed html-design snapshot/schema, all 22 current change/spec +items, and both extension tests. `openspec validate --specs --strict --json` +passes 18/18 and `openspec validate --archived --strict --json` passes 11/11. +No archived path is passed as a direct CLI item name. Independent active plans +are unchanged against the required baseline. NOTICE and contribution policy +remain satisfied; no AI coauthor trailers or local `docs/superpowers/` artifacts +are staged. + +The implementation is locally qualified and archived, but this sixth campaign +is incomplete. Parent owns independent final review, current-head CI watching, +authorized landing and verified owned checkout/local-branch/remote-branch/artifact +cleanup. This worker opens one focused PR with issue #479 and exact head/base, +then hands back without watching, merging, deleting sources/branches/checkouts, +releasing or deploying. PR head and forge state belong in the PR/issue handoff +receipt; this committed runtime checkpoint is not a claimed review/CI approval. + +### Owned external artifact and recovery inventory + +All external artifacts remain beneath `/private/tmp/morphir-workspace-migration/`. +`artifact-inventory.json` lists every owned file and `artifact-inventory-summary.json` +lists retained recovery paths without configuration values. At final qualification +the inventory contains 2,739 files (320,872,941 bytes), excluding the inventory's +own output files; final PR/receipt files added afterward are also owned beneath +that same root. Major groups are: + +- `tmp/`: physical test/subprocess temporary roots and deliberately retained fault + fixtures, including injected user-change and ownership evidence. +- `transactions/`, `cli/`, `discovery-verification/`: immutable red/green/probe, + interruption and explicit-recovery receipts. Two deliberately malformed parser + samples have no real destination operation. +- `validation/`: final 17-runner logs/expanded commands/counts; + `validation-initial/` is explicitly an earlier checkpoint. +- `publish/`: three published hosts, build logs, separate JSON results/stderr; + `publish-locks/`: 45 original lockfiles and byte-identical restoration proof. +- Coverage XMLs/summary, format/build/restore/OpenSpec receipts, source hashes and + campaign-local orchestration scripts/wrapper. Checkout `bin/obj/TestResults` + are reproducible local build artifacts, not new external cleanup roots. + +There are 114 retained `.morphir-edit-*` evidence files across 37 containing +directories: 112 from generated fault fixtures and two synthetic malformed parser +samples. These are intentionally unresolved/altered test evidence, not production +operations. The exact paths and transaction IDs are in the inventory. Earlier +helper receipts reporting no unresolved journals describe their earlier checkpoint; +the later ownership/cleanup regressions intentionally retain new evidence. +No file was removed or manufactured to improve the cleanup claim. Preserve these +journals/backups and their surrounding user-change fixtures until explicitly +inspected/recovered or dispositioned by the parent. Owned cleanup therefore +remains an explicit parent obligation. Global caches/tools and unrelated +campaign artifacts are excluded from ownership. diff --git a/docs/content/docs/getting-started/configuration.md b/docs/content/docs/getting-started/configuration.md index c8f0ba94..93ef453c 100644 --- a/docs/content/docs/getting-started/configuration.md +++ b/docs/content/docs/getting-started/configuration.md @@ -7,7 +7,7 @@ description: "Workspace discovery, inspection, safe TOML editing and .NET consum # Configure a workspace -The shared libraries read configuration and expose bounded workspace/member discovery, source loading and pure inspection. The `morphir config get/list/show/path/explain` commands expose the same queries. `config set/unset` plan and apply one TOML edit. `project migrate` adopts one classic project through loss-checked shared services. Workspace migration remains planned. See the [editing delivery journal]({{< relref "/docs/contributing/design/prds/config-editing-delivery" >}}) for local validation and parent delivery gates. +The shared libraries read configuration and expose bounded workspace/member discovery, source loading and pure inspection. The `morphir config get/list/show/path/explain` commands expose the same queries. `config set/unset` plan and apply one TOML edit. `project migrate` adopts one classic project through loss-checked shared services. `workspace migrate` adopts a bounded repository through aggregate verification and recoverable application. See the [workspace delivery journal]({{< relref "/docs/contributing/design/prds/workspace-migration-delivery" >}}) for local validation and parent delivery gates. ## Declare the workspace diff --git a/docs/content/guides/configuration.md b/docs/content/guides/configuration.md index 201f088b..dfe0b1ca 100644 --- a/docs/content/guides/configuration.md +++ b/docs/content/guides/configuration.md @@ -7,7 +7,7 @@ description: "Morphir configuration formats, discovery, precedence, and shared h # Configuration -Morphir configuration describes projects, workspace membership, and tooling settings. Shared libraries provide typed documents, source resolution, and project discovery to native and embedded hosts. Shared queries, CLI inspection and an opt-in Microsoft configuration adapter are available. Shared TOML editing and classic project migration are available; workspace migration remains planned. +Morphir configuration describes projects, workspace membership, and tooling settings. Shared libraries provide typed documents, source resolution, and project discovery to native and embedded hosts. Shared queries, CLI inspection and an opt-in Microsoft configuration adapter are available. Shared TOML editing and classic project migration are available; [bounded workspace migration]({{< relref "/docs/cli/workspace-migrate" >}}) is available. ## Implementation status @@ -18,7 +18,7 @@ Campaign 3 landed [shared queries]({{< relref "/docs/configuration/inspection" > `morphir config get/list/show/path/explain` and the [Microsoft adapter]({{< relref "/docs/configuration/microsoft" >}}). See the [inspection journal]({{< relref "/docs/contributing/design/prds/config-inspection-delivery" >}}) for its verified validation and landing/cleanup receipt. -Campaign 4 adds [shared editing]({{< relref "/docs/configuration/editing" >}}) and `morphir config set/unset`, including dry-run, typed inputs, effective explanations, optimistic conflicts and recovery. Its [delivery journal]({{< relref "/docs/contributing/design/prds/config-editing-delivery" >}}) records local qualification and parent delivery gates. [Classic project migration]({{< relref "/docs/cli/project-migrate" >}}) is available; workspace migration remains planned. +Campaign 4 adds [shared editing]({{< relref "/docs/configuration/editing" >}}) and `morphir config set/unset`, including dry-run, typed inputs, effective explanations, optimistic conflicts and recovery. Its [delivery journal]({{< relref "/docs/contributing/design/prds/config-editing-delivery" >}}) records local qualification and parent delivery gates. [Classic project migration]({{< relref "/docs/cli/project-migrate" >}}) is available; [bounded workspace migration]({{< relref "/docs/cli/workspace-migrate" >}}) is available. Existing cache-only tooling retains Git-first discovery and its separate CI overlay. It does not opt into the new policy automatically. Follow the [layered configuration PRD]({{< relref "/docs/contributing/design/prds/layered-configuration" >}}), the [walkthrough]({{< relref "/docs/getting-started/configuration" >}}), and the [CLI reference]({{< relref "/docs/cli/config" >}}) for their respective status. @@ -318,4 +318,4 @@ only to project files, and isolation omits all machine reads. ## Classic project adoption -Use [project migrate]({{< relref "/docs/cli/project-migrate" >}}) for one classic manifest. The [shared migration APIs]({{< relref "/docs/configuration/project-migration" >}}) support read-only hosts and guarded application without the CLI. Workspace migration remains planned. +Use [project migrate]({{< relref "/docs/cli/project-migrate" >}}) for one classic manifest. The [shared migration APIs]({{< relref "/docs/configuration/project-migration" >}}) support read-only hosts and guarded application without the CLI. The [workspace API]({{< relref "/docs/configuration/workspace-migration" >}}) composes the same conversion checks with bounded inventory, aggregate verification and explicit recovery. diff --git a/openspec/changes/add-workspace-migration/.openspec.yaml b/openspec/changes/archive/2026-11-03-add-workspace-migration/.openspec.yaml similarity index 100% rename from openspec/changes/add-workspace-migration/.openspec.yaml rename to openspec/changes/archive/2026-11-03-add-workspace-migration/.openspec.yaml diff --git a/openspec/changes/add-workspace-migration/design.html b/openspec/changes/archive/2026-11-03-add-workspace-migration/design.html similarity index 98% rename from openspec/changes/add-workspace-migration/design.html rename to openspec/changes/archive/2026-11-03-add-workspace-migration/design.html index 2721dbc8..e84af073 100644 --- a/openspec/changes/add-workspace-migration/design.html +++ b/openspec/changes/archive/2026-11-03-add-workspace-migration/design.html @@ -35,7 +35,7 @@

add-workspace-migration

-

OpenSpec change | 2026-11-02 | Planning only; implementation pending

+

OpenSpec change | 2026-11-03 | Implementation locally qualified; parent review, CI, landing and cleanup pending

Read the proposal, workspace-migration, and the tasks.

Context

diff --git a/openspec/changes/add-workspace-migration/proposal.md b/openspec/changes/archive/2026-11-03-add-workspace-migration/proposal.md similarity index 89% rename from openspec/changes/add-workspace-migration/proposal.md rename to openspec/changes/archive/2026-11-03-add-workspace-migration/proposal.md index e04be309..ed792c12 100644 --- a/openspec/changes/add-workspace-migration/proposal.md +++ b/openspec/changes/archive/2026-11-03-add-workspace-migration/proposal.md @@ -24,15 +24,15 @@ None. Existing publication and virtual-file requirements remain unchanged. ## Tracking -- [Layered configuration PRD](../../../docs/content/docs/contributing/design/prds/layered-configuration.md). -- Implementation prerequisites: [`add-classic-project-migration`](../archive/2026-11-03-add-classic-project-migration/proposal.md). +- [Layered configuration PRD](../../../../docs/content/docs/contributing/design/prds/layered-configuration.md). +- Implementation prerequisites: [`add-classic-project-migration`](../2026-11-03-add-classic-project-migration/proposal.md). - GitHub tracking issue: [#479](https://github.com/finos/morphir-dotnet/issues/479). ## Impact Add shared inventory/aggregate planner and workspace CLI commands; extend the transaction adapter to coordinate a bounded batch with journaled recovery. Cover filesystem and in-memory hosts; do not alter the generated-output publication capability. -Planning only. No runtime behavior or existing public API changes in this proposal commit. +Implementation locally qualified; parent review, CI, landing and cleanup remain pending. See the [delivery journal](../../../../docs/content/docs/contributing/design/prds/workspace-migration-delivery.md). ## Campaign completion diff --git a/openspec/changes/add-workspace-migration/specs/workspace-migration/spec.md b/openspec/changes/archive/2026-11-03-add-workspace-migration/specs/workspace-migration/spec.md similarity index 100% rename from openspec/changes/add-workspace-migration/specs/workspace-migration/spec.md rename to openspec/changes/archive/2026-11-03-add-workspace-migration/specs/workspace-migration/spec.md diff --git a/openspec/changes/add-workspace-migration/tasks.md b/openspec/changes/archive/2026-11-03-add-workspace-migration/tasks.md similarity index 98% rename from openspec/changes/add-workspace-migration/tasks.md rename to openspec/changes/archive/2026-11-03-add-workspace-migration/tasks.md index a855dd90..ca0e0b3b 100644 --- a/openspec/changes/add-workspace-migration/tasks.md +++ b/openspec/changes/archive/2026-11-03-add-workspace-migration/tasks.md @@ -22,7 +22,7 @@ Read [the design](design.html) and change-local specs before starting. Existing ## 4. Workspace CLI and integration - [x] 4.1 Add red-then-green workspace migrate subprocess scenarios for bounded directory/current roots, repeated include/exclude controls, --dry-run/--json, blocked/no-op results, and absence of config migrate; verify thin command registration and the workspace reference. -- [ ] 4.2 Run format/build/analyzers, shared/project/workspace/tooling suites, aggregate fault/concurrent-process and contract/coverage checks, and a non-CLI batch preview; verify repeat runs preserve bytes/timestamps and sources, then record results. +- [x] 4.2 Run format/build/analyzers, shared/project/workspace/tooling suites, aggregate fault/concurrent-process and contract/coverage checks, and a non-CLI batch preview; verify repeat runs preserve bytes/timestamps and sources, then record results. ## Workflow follow-up diff --git a/openspec/specs/workspace-migration/spec.md b/openspec/specs/workspace-migration/spec.md new file mode 100644 index 00000000..2a61b144 --- /dev/null +++ b/openspec/specs/workspace-migration/spec.md @@ -0,0 +1,96 @@ +# workspace-migration Specification + +## Purpose + +Adopts a repository of classic Morphir projects into the newer workspace system with an explicit bounded inventory, coordinated project plans, and recoverable application. + +## Requirements + +### Requirement: Workspace migration works before a workspace exists +The CLI SHALL expose workspace migrate [directory], backed by shared inventory, planning, and application services. The supplied directory or current directory SHALL bound the operation even when it has no declared Morphir workspace. Migration SHALL NOT scan ancestors for additional projects to adopt. + +#### Scenario: Repository of classic projects +- **WHEN** the selected directory contains two classic member projects and no workspace primary +- **THEN** migration can propose a workspace and individual project conversions within that directory + +### Requirement: Inventory respects explicit boundaries and exclusions +Candidate inventory SHALL remain within the selected root, skip discovered links, stop at nested declared workspaces, and exclude version-control/dependency/build output directories. Repeatable root-relative include/exclude patterns SHALL refine candidates. Existing declared exclusions SHALL remain effective. + +#### Scenario: Nested independent workspace +- **WHEN** a child directory declares its own workspace +- **THEN** inventory reports the boundary and does not adopt its members into the outer workspace + +#### Scenario: Excluded classic candidate +- **WHEN** an excluded directory contains morphir.json +- **THEN** it is reported as excluded and no project edit is planned for it + +### Requirement: Membership proposals preserve existing ownership +Plans SHALL use explicit root-relative member paths, preserve existing membership/settings, and include already migrated members without rewriting them. A root with a project SHALL remain a root project; otherwise the root SHALL be virtual. Duplicate names SHALL be diagnosed without conflating project paths. + +#### Scenario: Root classic project +- **WHEN** the selected root itself contains a classic project plus child projects +- **THEN** the plan retains its root-project identity and adds workspace membership without making it a glob member + +#### Scenario: Existing default member +- **WHEN** a valid workspace.default_member exists before migration +- **THEN** the plan preserves it rather than choosing another default + +### Requirement: Project migration behavior is reused +Each selected classic project SHALL be planned and independently verified using the single-project migration contract. Workspace migration SHALL NOT apply a looser mapping or omit unsupported fields to make a batch pass. + +#### Scenario: One blocked member +- **WHEN** one selected project has an unmapped nonempty dependency field +- **THEN** the aggregate plan is blocked and no member or workspace destination is replaced + +### Requirement: Aggregate verification checks the resulting workspace +Before application, migration SHALL verify discovery, membership, selection, and each conversion in the same proposed workspace context, excluding converted classic sources and ambient overlays. Source-path meaning and non-adoption settings SHALL be preserved. Existing workspace settings SHALL remain effective except for explicit membership additions. + +#### Scenario: Workspace change alters a member +- **WHEN** classic and converted versions of a project would discover different source paths under the same proposed workspace +- **THEN** aggregate verification reports the mismatch and blocks application + +### Requirement: Workspace adoption effects are explicit +The plan SHALL report intentional changes to ownership, membership, default selection, and shared output roots separately from configuration conversion. It SHALL NOT move existing generated artifacts. These adoption effects SHALL NOT excuse unrelated value or source-path changes. + +#### Scenario: Standalone output roots become workspace output +- **WHEN** standalone projects with member-local default output roots are adopted into a new workspace +- **THEN** preview reports their new shared workspace output context, conversion is compared within that context, and existing generated files are untouched + +### Requirement: Workspace previews enumerate the full operation +Workspace migrate SHALL support --dry-run, --json, and include/exclude controls. Reports SHALL list candidates, exclusions, boundaries, already migrated projects, member edits, workspace edits, blockers, and verification. Preview SHALL write nothing. + +#### Scenario: Repeatable inventory +- **WHEN** identical bounded trees are enumerated in different directory orders +- **THEN** the reported plan has the same project/path ordering and membership + +### Requirement: Apply is coordinated and recoverable +Application SHALL validate and stage the entire plan before any destination replacement and recheck all relevant source/target preconditions. It SHALL coordinate overlapping writers, journal replacements, and attempt rollback on commit failure. It SHALL NOT claim cross-file crash atomicity or delete classic sources. + +#### Scenario: Member changes before commit +- **WHEN** a selected source changes after planning +- **THEN** the batch reports a conflict and no destination is replaced + +#### Scenario: Commit failure +- **WHEN** replacement fails after an earlier destination changed +- **THEN** rollback is attempted and the result identifies restored and unresolved paths + +### Requirement: Recovery state prevents false completion +Success and complete rollback SHALL clean temporary transaction artifacts. Incomplete rollback or crash recovery SHALL retain an identifiable journal and report an explicit recovery-required outcome. A pending recovery SHALL block overlapping new writes until resolved. + +#### Scenario: Interrupted batch +- **WHEN** an interrupted migration leaves a journal with unresolved replacements +- **THEN** the next overlapping write reports recovery required instead of treating the batch as complete + +### Requirement: Repetition preserves completed work +An equivalent repeat SHALL report unchanged files without rewriting them. Already migrated projects SHALL remain in membership. A valid existing workspace with no further conversions SHALL be a successful no-op; an undeclared root with no eligible projects SHALL report that no adoption can be planned. + +#### Scenario: Second successful run +- **WHEN** the same workspace migration is applied twice +- **THEN** the second run leaves file bytes and timestamps unchanged + +### Requirement: Results remain host-independent and scriptable +Non-CLI hosts SHALL access the same aggregate plan and outcomes. CLI success/applicable preview/no-op SHALL exit 0; blocked, invalid, failed, or recovery-required operations SHALL exit 2 with structured JSON available and diagnostic text confined to stderr. + +#### Scenario: Mounted host preview +- **WHEN** an embedded host inventories the same tree as the CLI +- **THEN** it receives the same project plans, membership proposal, and blockers From ff5347fa4c1b98710b842a5274ef5f0cee7d0430 Mon Sep 17 00:00:00 2001 From: Damian Reeves <957246+DamianReeves@users.noreply.github.com> Date: Tue, 3 Nov 2026 17:56:38 -0600 Subject: [PATCH 07/14] docs: record current-main integration boundary --- .../design/prds/workspace-migration-delivery.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md b/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md index 6e7ee40e..3e6e3992 100644 --- a/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md +++ b/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md @@ -370,3 +370,20 @@ journals/backups and their surrounding user-change fixtures until explicitly inspected/recovered or dispositioned by the parent. Owned cleanup therefore remains an explicit parent obligation. Global caches/tools and unrelated campaign artifacts are excluded from ownership. + +### Current-main interaction at PR preparation + +GitHub main advanced during qualification to +`0542145a4bcc0edf9c9dd2fb3122df36c73e80d0` through independent C# generation +PRs #478, #480 and #482. The owned branch retains the explicitly required starting +baseline `e3b9a053120a49e94c70c30cd4f62ef783672c64`; PR #477 remains its merged +prerequisite. No unrelated generation work is copied or modified in this branch. +The intervening production changes add a separate generation backend/reference +and model package-release notes; they do not change configuration/project/workspace +services. README/troubleshooting edits are in different sections. +`git merge-tree --write-tree origin/main HEAD` succeeds without conflicts +(integration tree `a866f8115332651717e14fe1b0a7ec4b8db3e460` at the documentation +checkpoint). This is merge compatibility evidence, not execution of a merged +checkout. The 17-runner/runtime receipts qualify this branch at its required +baseline; parent-owned current-base PR CI remains necessary for the new backend +and additional independent suites. The PR records the actual newer base separately. From 19504a8f7b7e3eb26b57028c3fdd0c705539717e Mon Sep 17 00:00:00 2001 From: Damian Reeves <957246+DamianReeves@users.noreply.github.com> Date: Tue, 3 Nov 2026 18:16:31 -0600 Subject: [PATCH 08/14] fix: preserve workspace adoption semantics Existing workspace adoption must preserve original project values. Apply root conversion filters and report the prior context for new members. --- .../docs/configuration/workspace-migration.md | 11 +++ .../prds/workspace-migration-delivery.md | 33 +++++++++ .../WorkspaceMigrationService.cs | 33 +++++---- .../WorkspaceMigrationSmoke.cs | 20 ++++++ .../WorkspaceMigrationPolicyReviewTests.cs | 70 +++++++++++++++++++ .../WorkspaceMigrationTests.cs | 4 +- .../WorkspaceMigration.feature | 27 +++++++ .../WorkspaceMigrationSteps.cs | 27 +++++++ 8 files changed, 206 insertions(+), 19 deletions(-) create mode 100644 tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationPolicyReviewTests.cs diff --git a/docs/content/docs/configuration/workspace-migration.md b/docs/content/docs/configuration/workspace-migration.md index 65fa301b..0d58a08d 100644 --- a/docs/content/docs/configuration/workspace-migration.md +++ b/docs/content/docs/configuration/workspace-migration.md @@ -118,3 +118,14 @@ or backups to bypass a guard. Preserve user changes and unresolved evidence. CLI reports use the same plans and outcomes. See the [workspace command reference](../cli/workspace-migrate.md), walkthrough and troubleshooting for exit codes, previews and scriptable JSON. + +Supplied exclusions also control root-project conversion. For example, excluding +`.` leaves the classic root project in place while child adoption can still add +a workspace declaration. Root-project identity and existing ownership are +preserved. Excluding `**` selects no conversions, including at the root. + +Every newly adopted project is checked against its original non-adoption values, +including additions to an existing workspace. A root backend or other inherited +setting cannot silently change the new member. Retained members keep their prior +workspace context; new members report their original standalone ownership and +local output root before adoption. diff --git a/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md b/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md index 3e6e3992..a503bb27 100644 --- a/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md +++ b/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md @@ -387,3 +387,36 @@ checkpoint). This is merge compatibility evidence, not execution of a merged checkout. The 17-runner/runtime receipts qualify this branch at its required baseline; parent-owned current-base PR CI remains necessary for the new backend and additional independent suites. The PR records the actual newer base separately. + +## Parent policy review follow-up + +Independent review of `ff5347f` identified three concrete defects: existing +workspaces bypassed original-value checks for newly adopted projects, supplied +exclusions did not prevent root conversion, and new members were reported with +existing workspace ownership/output roots before adoption. All five new public +API cases fail in `parent-review/policy-red-02.log`. An earlier malformed filter +selected zero tests and is not red evidence. All three actual CLI regressions +fail against the original executable in `parent-review/policy-cli-red.log`. + +The shared coordinator now checks each newly adopted project against its original +non-adoption settings in both new and existing workspaces. Root exclusions apply +to conversion selection while root identity and necessary workspace composition +remain. Adoption effects use actual prior membership to distinguish retained and +standalone contexts. The native smoke includes these paths. + +The first full IO run passed 433 of 434 cases and exposed an older success +fixture relying on the newly rejected silent backend change. Its new project +now explicitly authors the matching backend, retaining a valid successful +preservation case. Its member-local profile activation declaration was also +removed so that fixture does not rely on control metadata changing authority +through adoption. The rebuilt CLI passes the three new cases. The refreshed +full IO runner passes 434 cases. Final full-suite, coverage, runtime, CI and +independent transaction-review results remain delivery gates. + +All 17 configured executable runners pass again after the parent policy fixes. +The full IO runner now passes 434 cases and tooling passes 436, including all +three new real CLI regressions; all other runner counts remain as above. +`validation/summary.json` and the refreshed individual runner logs retain exact +commands, duration and exits for this current-source checkpoint. The full +solution build passes with nine existing warnings and zero errors, and changed +C# files passed formatter processing before that build. diff --git a/src/Morphir.Configuration.IO/WorkspaceMigrationService.cs b/src/Morphir.Configuration.IO/WorkspaceMigrationService.cs index a4b96fac..3041c793 100644 --- a/src/Morphir.Configuration.IO/WorkspaceMigrationService.cs +++ b/src/Morphir.Configuration.IO/WorkspaceMigrationService.cs @@ -117,8 +117,8 @@ WorkspaceMigrationPreview Result(bool verified = false, bool changed = false, Co diagnostics.AddRange(path == root ? [] : location.Diagnostics); if (Has(location.Values, "project")) { - if (include.Length > 0 && !include.Any(p => Glob(p, relative))) - inventory.Add(new(path, relative, WorkspaceInventoryKind.Excluded, "Project does not match include filters.", Name(location))); + if (Excluded(relative, exclude) || include.Length > 0 && !include.Any(p => Glob(p, relative))) + inventory.Add(new(path, relative, WorkspaceInventoryKind.Excluded, "Project excluded by supplied include/exclude filters.", Name(location))); else { var classics = Classics(location).ToArray(); @@ -251,20 +251,18 @@ WorkspaceMigrationPreview Result(bool verified = false, bool changed = false, Co if (projects.Any(p => !p.CanApply) || diagnostics.Any(d => d.Severity == DiagnosticSeverity.Error)) return Result(); if (projects.Count > 1 && projects.Any(p => !Equals(p.Verification!.BaseWorkspace, projects[0].Verification!.BaseWorkspace))) { Error(root, "workspace.migrate_context", "Conversions do not share the same proposed base workspace context."); return Result(); } - if (!declared) + // Every newly adopted project keeps its original non-adoption settings, + // whether the workspace is new or already declared. + foreach (var (path, location) in locations.Where(p => !declared || !existingPaths.Contains(p.Key))) { - // Adoption may change topology/output roots, but never unrelated values. - foreach (var (path, location) in locations) - { - var original = BaseResolution(location.Sources, ProjectScope.Root, null); - var adoptedSources = path == root ? common.RootSources.Concat(location.Sources.Where(s => Classics(location).Any(c => c.Source.Path == s.Source.Path))) - : common.RootSources.Concat(location.Sources.Select(s => Rebind(s, SourceRole.MemberPrimary))); - var adopted = BaseResolution(adoptedSources, path == root ? ProjectScope.Root : ProjectScope.Member, common.Workspace); - var beforeValues = ComparableValues(original, location.Sources); - var afterValues = ComparableValues(adopted, adoptedSources); - if (!ProjectMigration.same(beforeValues, afterValues)) - Error(path, "workspace.migrate_adoption_value", "Adoption changes a non-adoption setting; review the project and proposed shared base settings."); - } + var original = BaseResolution(location.Sources, ProjectScope.Root, null); + var adoptedSources = path == root ? common.RootSources.Concat(location.Sources.Where(s => Classics(location).Any(c => c.Source.Path == s.Source.Path))) + : common.RootSources.Concat(location.Sources.Select(s => Rebind(s, SourceRole.MemberPrimary))); + var adopted = BaseResolution(adoptedSources, path == root ? ProjectScope.Root : ProjectScope.Member, common.Workspace); + var beforeValues = ComparableValues(original, location.Sources); + var afterValues = ComparableValues(adopted, adoptedSources); + if (!ProjectMigration.same(beforeValues, afterValues)) + Error(path, "workspace.migrate_adoption_value", "Adoption changes a non-adoption setting; review the project and proposed shared base settings."); } // Check the actual retained-source view as well as independent conversion discovery. var retained = ProjectSystem.Discover(overlay, root).Discovery; @@ -305,8 +303,9 @@ WorkspaceMigrationPreview Result(bool verified = false, bool changed = false, Co var output = candidate.Workspace?.Value.OutputDirectory?.Value; foreach (var path in locations.Keys) { - var oldOutput = declared ? existingDiscovery!.Workspace?.Value.OutputDirectory?.Value : ProjectPath.join(path, ".morphir/out"); - effects.Add(new(path, "ownership", declared ? existingDiscovery!.Workspace?.Value.Root : path, candidate.Workspace?.Value.Root)); + var previouslyOwned = declared && (existingPaths.Contains(path) || path == root); + var oldOutput = previouslyOwned ? existingDiscovery!.Workspace?.Value.OutputDirectory?.Value : ProjectPath.join(path, ".morphir/out"); + effects.Add(new(path, "ownership", previouslyOwned ? existingDiscovery!.Workspace?.Value.Root : path, candidate.Workspace?.Value.Root)); effects.Add(new(path, "membership", existingPaths.Contains(path) || path == root && declared ? "retained" : "standalone", path == root ? "root-project" : ProjectPath.relative(workspaceBase, path))); effects.Add(new(path, "shared-output-root", oldOutput, output)); } diff --git a/tests/Morphir.Configuration.IO.Smoke/WorkspaceMigrationSmoke.cs b/tests/Morphir.Configuration.IO.Smoke/WorkspaceMigrationSmoke.cs index 0379468c..2b16c417 100644 --- a/tests/Morphir.Configuration.IO.Smoke/WorkspaceMigrationSmoke.cs +++ b/tests/Morphir.Configuration.IO.Smoke/WorkspaceMigrationSmoke.cs @@ -27,6 +27,26 @@ internal static int Run() Check(!json.Contains("private-literal", StringComparison.Ordinal) && !json.Contains("PRIVATE_REFERENCE", StringComparison.Ordinal), "raw secret escaped redaction"); using (var parsed = JsonDocument.Parse(json)) Check(parsed.RootElement.GetProperty("verified").GetBoolean(), "generated report verification lost"); Check(readOnly.Apply(preview).Status == ConfigurationTransactionStatus.IoFailure, "read-only apply allowed"); + var inherited = new WorkspaceMigrationService(new InMemoryProjectTree("/repo", new Dictionary + { + ["/repo/morphir.toml"] = "[workspace]\nmembers=[]\n[pipeline]\nbackend='root-specific'\n", + ["/repo/app/morphir.toml"] = "[project]\nname='App'\n" + })).Plan("/repo"); + Check(!inherited.CanApply && inherited.Diagnostics.Any(d => d.Code == "workspace.migrate_adoption_value"), "existing workspace masked a new member value change"); + var adopted = new WorkspaceMigrationService(new InMemoryProjectTree("/repo", new Dictionary + { + ["/repo/morphir.toml"] = "[workspace]\nmembers=[]\nout_dir='shared'\n", + ["/repo/app/morphir.toml"] = "[project]\nname='App'\n" + })).Plan("/repo"); + Check(adopted.CanApply && adopted.AdoptionEffects.Any(e => e.Kind == "ownership" && e.Before == "/repo/app"), "original member ownership lost"); + Check(adopted.AdoptionEffects.Any(e => e.Kind == "shared-output-root" && e.Before == "/repo/app/.morphir/out"), "original member output lost"); + var excludedRoot = new WorkspaceMigrationService(new InMemoryProjectTree("/repo", new Dictionary + { + ["/repo/morphir.json"] = "{name:'Root',exposedModules:[]}", + ["/repo/app/morphir.json"] = "{name:'App',exposedModules:[]}" + })).Plan("/repo", excludes: ["."]); + Check(excludedRoot.CanApply && excludedRoot.RootIsProject && excludedRoot.Inventory.Any(i => i.RelativePath == "." && i.Kind == WorkspaceInventoryKind.Excluded) + && excludedRoot.Projects.All(p => p.ProjectPath != "/repo"), "root exclusion did not preserve conversion boundary"); var root = ProjectPath.join(Path.GetTempPath(), "workspace-smoke-" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(root); try diff --git a/tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationPolicyReviewTests.cs b/tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationPolicyReviewTests.cs new file mode 100644 index 00000000..2553b7b4 --- /dev/null +++ b/tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationPolicyReviewTests.cs @@ -0,0 +1,70 @@ +// Copyright 2026 FINOS +// SPDX-License-Identifier: Apache-2.0 +using FluentAssertions; +using Morphir.Configuration; +using Morphir.Configuration.IO; + +namespace Morphir.Configuration.IO.Tests; + +public sealed class WorkspaceMigrationPolicyReviewTests +{ + [Test] + [Arguments(false)] + [Arguments(true)] + public void NewlyAdoptedProjectsCannotGainUnrelatedExistingWorkspaceSettings(bool classic) + { + using var fixture = new WorkspaceFixture(); + fixture.Write("morphir.toml", "[workspace]\nmembers=[]\n[pipeline]\nbackend='root-specific'\n"); + fixture.Write(classic ? "app/morphir.json" : "app/morphir.toml", classic + ? "{name:'App',exposedModules:[]}" + : "[project]\nname='App'\nsource_directory='src'\nexposed_modules=[]\n"); + var original = File.ReadAllBytes(fixture.Root + "/morphir.toml"); + var preview = fixture.Service.Plan(fixture.Root); + preview.CanApply.Should().BeFalse("new members must retain non-adoption values even when the workspace exists"); + preview.Diagnostics.Should().Contain(d => d.Code == "workspace.migrate_adoption_value"); + fixture.Service.Apply(preview).Code.Should().Be("workspace.migrate_unverified"); + File.ReadAllBytes(fixture.Root + "/morphir.toml").Should().Equal(original); + } + + [Test] + public void NewAndRetainedMembersReportTheirActualOriginalOwnershipAndOutput() + { + var tree = new InMemoryProjectTree("/repo", new Dictionary + { + ["/repo/morphir.toml"] = "[workspace]\nmembers=['old']\nout_dir='shared'\n", + ["/repo/old/morphir.toml"] = "[project]\nname='Old'\n", + ["/repo/app/morphir.toml"] = "[project]\nname='App'\n" + }); + var preview = new WorkspaceMigrationService(tree).Plan("/repo"); + preview.CanApply.Should().BeTrue(); + preview.AdoptionEffects.Should().Contain(e => e.ProjectPath == "/repo/app" && e.Kind == "ownership" && e.Before == "/repo/app" && e.After == "/repo"); + preview.AdoptionEffects.Should().Contain(e => e.ProjectPath == "/repo/app" && e.Kind == "shared-output-root" && e.Before == "/repo/app/.morphir/out" && e.After == "/repo/shared"); + preview.AdoptionEffects.Should().Contain(e => e.ProjectPath == "/repo/old" && e.Kind == "ownership" && e.Before == "/repo" && e.After == "/repo"); + preview.AdoptionEffects.Should().Contain(e => e.ProjectPath == "/repo/old" && e.Kind == "shared-output-root" && e.Before == "/repo/shared" && e.After == "/repo/shared"); + } + + [Test] + [Arguments(".")] + [Arguments("**")] + public void SuppliedExclusionsPreventRootConversionWithoutRemovingItsIdentity(string pattern) + { + using var fixture = new WorkspaceFixture(); + const string rootSource = "{name:'Root',exposedModules:[]}"; + fixture.Write("morphir.json", rootSource); + fixture.Write("app/morphir.json", "{name:'App',exposedModules:[]}"); + var preview = fixture.Service.Plan(fixture.Root, excludes: [pattern]); + preview.RootIsProject.Should().BeTrue(); + preview.Inventory.Should().Contain(i => i.Path == fixture.Root && i.Kind == WorkspaceInventoryKind.Excluded); + preview.Projects.Should().NotContain(p => p.ProjectPath == fixture.Root); + if (pattern == ".") + { + preview.CanApply.Should().BeTrue(string.Join(",", preview.Diagnostics.Select(d => d.Code))); + preview.Projects.Should().ContainSingle().Which.ProjectPath.Should().Be(fixture.Root + "/app"); + fixture.Service.Apply(preview).Status.Should().Be(ConfigurationTransactionStatus.Written); + File.ReadAllText(fixture.Root + "/morphir.json").Should().Be(rootSource); + var target = ConfigurationReader.ReadText(fixture.Root + "/morphir.toml", File.ReadAllText(fixture.Root + "/morphir.toml")); + ConfigTree.tryFind(Microsoft.FSharp.Collections.ListModule.OfSeq(new[] { "project" }), ((SourceContent.Available)target.Content).Item.Raw).Should().BeNull("only workspace composition belongs in the root target"); + } + else preview.CanApply.Should().BeFalse("excluding every project leaves no adoption to plan"); + } +} diff --git a/tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationTests.cs b/tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationTests.cs index d252f63f..e65700b9 100644 --- a/tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationTests.cs +++ b/tests/Morphir.Configuration.IO.Tests/WorkspaceMigrationTests.cs @@ -43,9 +43,9 @@ public void FiltersAreRootRelativeAndExistingExclusionsRemainEffective() public sealed class WorkspaceMigrationPlanningTests { - private const string Classic = "{name:'A',sourceDirectory:'../source',exposedModules:[],profiles:{default:['dev']}}"; + private const string Classic = "{name:'A',sourceDirectory:'../source',exposedModules:[],pipeline:{backend:'shared'}}"; [Test] - public void ExistingWorkspaceSettingsDefaultsAndMigratedMembersRemainEffective() + public void MatchingNewProjectPreservesExistingWorkspaceSettingsDefaultsAndMigratedMembers() { var tree = new InMemoryProjectTree("/repo", new Dictionary { diff --git a/tests/Morphir.Tooling.Tests/Features/WorkspaceMigration/WorkspaceMigration.feature b/tests/Morphir.Tooling.Tests/Features/WorkspaceMigration/WorkspaceMigration.feature index a3760612..c42c073f 100644 --- a/tests/Morphir.Tooling.Tests/Features/WorkspaceMigration/WorkspaceMigration.feature +++ b/tests/Morphir.Tooling.Tests/Features/WorkspaceMigration/WorkspaceMigration.feature @@ -176,3 +176,30 @@ Feature: Adopt classic projects through one bounded workspace migration Given a workspace migration repository When I request the configuration migration alias Then configuration migration remains unavailable + + @ParentPolicyReview + Scenario: Existing workspace adoption cannot change unrelated new member values + Given an empty workspace migration repository + And workspace migration adoption fixture "incompatible" + When I migrate the workspace with "" + Then workspace migration exits 2 with outcome "blocked" + And workspace migration reports diagnostics on stderr + And workspace migration writes no destinations + + @ParentPolicyReview + Scenario: A root exclusion retains root identity without conversion + Given a workspace migration repository + And a workspace migration root project + When I migrate the workspace with "--exclude . --dry-run" + Then workspace migration exits 0 with outcome "preview" + And workspace migration reports excluded "." + And workspace migration retains root project identity + And workspace migration excludes root conversion + + @ParentPolicyReview + Scenario: A new canonical member reports its standalone original context + Given an empty workspace migration repository + And workspace migration adoption fixture "compatible" + When I migrate the workspace with "--dry-run" + Then workspace migration exits 0 with outcome "preview" + And workspace migration reports the new member original context diff --git a/tests/Morphir.Tooling.Tests/Features/WorkspaceMigration/WorkspaceMigrationSteps.cs b/tests/Morphir.Tooling.Tests/Features/WorkspaceMigration/WorkspaceMigrationSteps.cs index 1433b39f..72d032db 100644 --- a/tests/Morphir.Tooling.Tests/Features/WorkspaceMigration/WorkspaceMigrationSteps.cs +++ b/tests/Morphir.Tooling.Tests/Features/WorkspaceMigration/WorkspaceMigrationSteps.cs @@ -74,6 +74,33 @@ public void Existing() Write("existing/morphir.toml", ExistingMember); } + [Given("workspace migration adoption fixture {string}")] + public void AdoptionFixture(string mode) + { + Write("morphir.toml", "[workspace]\nmembers=['old']\nout_dir='shared'\n" + + (mode == "incompatible" ? "[pipeline]\nbackend='root-specific'\n" : "")); + Write("old/morphir.toml", "[project]\nname='Old'\n"); + Write("app/morphir.toml", "[project]\nname='App'\n"); + } + + [Then("workspace migration excludes root conversion")] + public void ExcludedRootConversion() + { + var root = ProjectPath.normalize(_root); + _report!.RootElement.GetProperty("projects").EnumerateArray() + .Should().NotContain(p => p.GetProperty("projectPath").GetString() == root); + } + + [Then("workspace migration reports the new member original context")] + public void OriginalAdoptionContext() + { + var root = ProjectPath.normalize(_root); + var effects = _report!.RootElement.GetProperty("adoptionEffects").EnumerateArray() + .Where(e => e.GetProperty("projectPath").GetString() == root + "/app").ToArray(); + effects.Single(e => e.GetProperty("kind").GetString() == "ownership").GetProperty("before").GetString().Should().Be(root + "/app"); + effects.Single(e => e.GetProperty("kind").GetString() == "shared-output-root").GetProperty("before").GetString().Should().Be(root + "/app/.morphir/out"); + } + [Given("a hidden workspace migration member primary")] public void HiddenMember() { From de005e0b34496cb83152b447dfe8f919eef786e8 Mon Sep 17 00:00:00 2001 From: Damian Reeves <957246+DamianReeves@users.noreply.github.com> Date: Tue, 3 Nov 2026 18:18:38 -0600 Subject: [PATCH 09/14] test: qualify workspace adoption review fixes --- .../design/prds/workspace-migration-delivery.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md b/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md index a503bb27..efbc2dbd 100644 --- a/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md +++ b/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md @@ -420,3 +420,18 @@ three new real CLI regressions; all other runner counts remain as above. commands, duration and exits for this current-source checkpoint. The full solution build passes with nine existing warnings and zero errors, and changed C# files passed formatter processing before that build. + +Final policy qualification after `19504a8` passes full formatter verification, +OpenSpec/extension verification, fresh IO coverage and all three published host +modes. The unchanged F# source maps reuse their core collector with the fresh IO +report: core 4188/4345, 96.39%; IO 2493/2559, 97.42%, recorded in +`parent-review/policy-coverage-summary.json`. Managed, trimmed and native hosts +each pass 42 workspace checks, 32 project migration checks, 44 editing checks, +228 JSON5 cases and 26 Morphir fixtures. Published stderr is empty; trimmed and +native dependency warning counts remain 44/47 with the documented explicit +warning-severity qualification. Lockfiles are restored byte-for-byte. Exact +commands and exits are in `parent-review/policy-qualification-summary.json` and +the refreshed publish summary. Original retained recovery evidence is untouched; +new parent validation fault-fixture evidence also remains for final disposition. +Independent transaction/recovery review, current-head CI, landing and audited +cleanup remain outstanding campaign gates. From 56eb44a2cd9c37e23c21f9cc65fbeb562104e64f Mon Sep 17 00:00:00 2001 From: Damian Reeves <957246+DamianReeves@users.noreply.github.com> Date: Tue, 3 Nov 2026 18:30:53 -0600 Subject: [PATCH 10/14] fix: avoid ancestor recovery discovery races Check directory kind without fingerprinting unrelated ancestor entries. Preserve no-follow checks and strict transaction preconditions while allowing unrelated sibling files to change. --- .../prds/workspace-migration-delivery.md | 25 +++++++ .../ConfigurationTransaction.cs | 22 ++++++- ...ConfigurationRecoveryDiscoveryRaceTests.cs | 65 +++++++++++++++++++ 3 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 tests/Morphir.Configuration.IO.Tests/ConfigurationRecoveryDiscoveryRaceTests.cs diff --git a/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md b/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md index efbc2dbd..2f6b3b1f 100644 --- a/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md +++ b/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md @@ -435,3 +435,28 @@ the refreshed publish summary. Original retained recovery evidence is untouched; new parent validation fault-fixture evidence also remains for final disposition. Independent transaction/recovery review, current-head CI, landing and audited cleanup remain outstanding campaign gates. + +### Ancestor recovery discovery race + +Linux CI on `ff5347f` fails the valid empty-workspace no-op CLI scenario with +`config.edit_io`. Its generic renderer does not expose the original exception, +so the retained CI log alone cannot identify the exact failing filesystem call. +A focused concurrent probe then reproduces a relevant native recovery race: +`FindRecoveryJournals` fingerprints every ancestor directory through `Observe`, +and an unrelated sibling disappearing during that fingerprint throws +`FileNotFoundException`. `parent-review/recovery-race-red.log` records this +failure before production changes. + +Journal discovery now checks directory kind and refuses linked ancestors without +fingerprinting unrelated contents. Source/target/listing observation and stale +plan guards retain their original strict behavior. The focused regression scans +while unrelated files churn; a companion test preserves linked-ancestor refusal. +Both pass in `parent-review/recovery-race-green.log`. The full refreshed IO runner +passes 436 cases. Current-head Linux CI still qualifies the original CI symptom. + +All 17 runners pass again after the recovery scan correction: IO 436, tooling +436, core configuration 177 and Microsoft 22; the other counts are unchanged. +The full solution build and changed-file formatter pass. The refreshed full +runner logs and summary retain current-source execution evidence. This is a +verified native discovery race fix; current-head CI remains required before +claiming the Linux no-op failure resolved on the runner. diff --git a/src/Morphir.Configuration.IO/ConfigurationTransaction.cs b/src/Morphir.Configuration.IO/ConfigurationTransaction.cs index 6dd24cf9..410320f8 100644 --- a/src/Morphir.Configuration.IO/ConfigurationTransaction.cs +++ b/src/Morphir.Configuration.IO/ConfigurationTransaction.cs @@ -483,12 +483,30 @@ public ConfigurationObservation Observe(string path, IReadOnlyCollection return new(path, ConfigurationObservationKind.Missing, ""); } + // Recovery enumeration needs directory kind, not a fingerprint of unrelated + // ancestor entries. Keep the same no-follow checks as native observations. + private static bool IsDirectoryWithoutLinks(string path) + { + var nativeRoot = Path.GetPathRoot(path)!; + var current = ProjectPath.normalize(nativeRoot); + var attributes = FileAttributes.Directory; + foreach (var component in path[nativeRoot.Length..].Split('/', StringSplitOptions.RemoveEmptyEntries)) + { + current = ProjectPath.join(current, component); + try { attributes = File.GetAttributes(current); } + catch (FileNotFoundException) { return false; } + catch (DirectoryNotFoundException) { return false; } + if ((attributes & FileAttributes.ReparsePoint) != 0) return false; + } + return (attributes & FileAttributes.Directory) != 0 && Directory.Exists(path); + } + public IReadOnlyList FindRecoveryJournals(IReadOnlyList paths) { var directories = new HashSet(StringComparer.Ordinal); foreach (var path in paths) { - var current = Observe(path).Kind == ConfigurationObservationKind.Directory ? Canonical(path) : ProjectPath.parent(Canonical(path)); + var current = IsDirectoryWithoutLinks(Canonical(path)) ? Canonical(path) : ProjectPath.parent(Canonical(path)); while (true) { directories.Add(current); @@ -500,7 +518,7 @@ public IReadOnlyList FindRecoveryJournals(IReadOnlyList paths) var result = new List(); foreach (var directory in directories.Order(StringComparer.Ordinal)) { - if (Observe(directory).Kind != ConfigurationObservationKind.Directory) continue; + if (!IsDirectoryWithoutLinks(directory)) continue; result.AddRange(Directory.EnumerateFileSystemEntries(directory, ".morphir-edit-*") .Select(Canonical).Where(RecoveryDocument.IsJournal)); } diff --git a/tests/Morphir.Configuration.IO.Tests/ConfigurationRecoveryDiscoveryRaceTests.cs b/tests/Morphir.Configuration.IO.Tests/ConfigurationRecoveryDiscoveryRaceTests.cs new file mode 100644 index 00000000..dfb9942f --- /dev/null +++ b/tests/Morphir.Configuration.IO.Tests/ConfigurationRecoveryDiscoveryRaceTests.cs @@ -0,0 +1,65 @@ +// Copyright 2026 FINOS +// SPDX-License-Identifier: Apache-2.0 +using FluentAssertions; +using Morphir.Configuration.IO; + +namespace Morphir.Configuration.IO.Tests; + +public sealed class ConfigurationRecoveryDiscoveryRaceTests +{ + [Test] + public void RecoveryDiscoveryNeverEnumeratesLinkedAncestors() + { + if (OperatingSystem.IsWindows()) return; + var root = ProjectPath.join(Path.GetTempPath(), "recovery-discovery-link-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root + "/physical"); + var alias = root + "/alias"; + Directory.CreateSymbolicLink(alias, root + "/physical"); + var journal = ".morphir-edit-" + new string('a', 32) + "-fixture-journal"; + File.WriteAllText(root + "/physical/" + journal, "must not inspect through link"); + try + { + new NativeConfigurationTransactionHost().FindRecoveryJournals([alias + "/morphir.toml"]) + .Should().NotContain(p => p.StartsWith(alias + "/", StringComparison.Ordinal)); + } + finally { Directory.Delete(root, true); } + } + + [Test] + public async Task UnrelatedAncestorEntriesCanChangeDuringRecoveryDiscovery() + { + var root = ProjectPath.join(Path.GetTempPath(), "recovery-discovery-race-" + Guid.NewGuid().ToString("N")); + var workspace = root + "/workspace"; + Directory.CreateDirectory(workspace); + var target = workspace + "/morphir.toml"; + await File.WriteAllTextAsync(target, "[workspace]\nmembers=[]\n"); + using var cancel = new CancellationTokenSource(); + using var started = new ManualResetEventSlim(); + var churn = Task.Run(() => + { + started.Set(); + while (!cancel.IsCancellationRequested) + for (var i = 0; i < 8; i++) + { + var path = root + "/unrelated-" + i; + File.WriteAllText(path, "unrelated"); + Thread.Yield(); + File.Delete(path); + } + }); + try + { + started.Wait(); + var host = new NativeConfigurationTransactionHost(); + for (var i = 0; i < 300; i++) + _ = host.FindRecoveryJournals([target]); + (await File.ReadAllTextAsync(target)).Should().Be("[workspace]\nmembers=[]\n"); + } + finally + { + cancel.Cancel(); + await churn; + Directory.Delete(root, true); + } + } +} From 9590b604b7b491433d06ccb17f44939476530c34 Mon Sep 17 00:00:00 2001 From: Damian Reeves <957246+DamianReeves@users.noreply.github.com> Date: Tue, 3 Nov 2026 18:33:21 -0600 Subject: [PATCH 11/14] test: qualify recovery discovery race fix --- .../design/prds/workspace-migration-delivery.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md b/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md index 2f6b3b1f..020ced86 100644 --- a/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md +++ b/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md @@ -460,3 +460,14 @@ The full solution build and changed-file formatter pass. The refreshed full runner logs and summary retain current-source execution evidence. This is a verified native discovery race fix; current-head CI remains required before claiming the Linux no-op failure resolved on the runner. + +Final qualification after `56eb44a` passes full formatter, OpenSpec/extension, +fresh IO coverage and all managed/trimmed/native runtime hosts. Core coverage +remains 4188/4345, 96.39%; IO is 2506/2573, 97.4%. +Each published mode passes 42 workspace, 32 project migration, 44 editing, 228 +JSON5 and 26 Morphir checks with empty stderr. Dependency warning counts remain +44/47 for trimmed/native and zero for managed; lockfiles are restored exactly. +The latest full rebuild reports eleven existing warnings and zero errors. +`parent-review/recovery-race-qualification-summary.json`, coverage summary and +publish summary retain exact commands and results. Current-head Linux CI and +independent transaction/recovery review remain open gates. From ff495a727de49a0c88e6e7258f32ad91ea64a726 Mon Sep 17 00:00:00 2001 From: Damian Reeves <957246+DamianReeves@users.noreply.github.com> Date: Tue, 3 Nov 2026 18:53:51 -0600 Subject: [PATCH 12/14] fix: retain durable recovery failure evidence Honor persisted commit witnesses before rollback. Preserve rejected artifact guards and return conservative metadata when storage disconnects during error handling. --- .../configuration/transaction-recovery.md | 17 +++ .../prds/workspace-migration-delivery.md | 37 +++++ .../ConfigurationRecovery.cs | 9 ++ .../ConfigurationTransaction.cs | 110 ++++++++++----- .../RecoveryReceiptSmoke.cs | 79 +++++++++++ .../WorkspaceMigrationSmoke.cs | 2 +- .../ConfigurationRecoveryReceiptTests.cs | 133 ++++++++++++++++++ 7 files changed, 348 insertions(+), 39 deletions(-) create mode 100644 tests/Morphir.Configuration.IO.Smoke/RecoveryReceiptSmoke.cs create mode 100644 tests/Morphir.Configuration.IO.Tests/ConfigurationRecoveryReceiptTests.cs diff --git a/docs/content/docs/configuration/transaction-recovery.md b/docs/content/docs/configuration/transaction-recovery.md index 0a8f9ddc..b4914819 100644 --- a/docs/content/docs/configuration/transaction-recovery.md +++ b/docs/content/docs/configuration/transaction-recovery.md @@ -154,3 +154,20 @@ registry is not a substitute. Recovery metadata uses version 2. Automatic recovery refuses unknown versions and invalid ownership. Preserve earlier journals for explicit review with their original host/version. Do not rewrite or silently delete them during an upgrade. + +A lost host acknowledgement does not prove a commit failed. If the committed +witness is present and validates against the attempted operation, application +preserves the candidate files and backups and returns recovery required for +explicit completion. If the witness is absent, rollback can proceed. + +Rejected transaction-shaped artifacts remain recovery blockers while present. +Recovery preserves the journals and reports their unresolved paths rather than +claiming complete rollback with metadata left behind. Any manual disposition +must first establish the artifact's origin and preserve user changes. Unrelated +foreign files are never cleanup authority. + +If the storage host disconnects while a failure is being handled, application +returns a typed recovery-required result with known or possible evidence +locations and unresolved targets. Its fallback does not perform further storage observations +or blind rollback. Reconnect storage and inspect the retained journal +before explicitly recovering. diff --git a/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md b/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md index 020ced86..2cedaff0 100644 --- a/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md +++ b/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md @@ -471,3 +471,40 @@ The latest full rebuild reports eleven existing warnings and zero errors. `parent-review/recovery-race-qualification-summary.json`, coverage summary and publish summary retain exact commands and results. Current-head Linux CI and independent transaction/recovery review remain open gates. + +### Independent transaction review follow-up + +Independent review of `ff5347f` reports three P2 failures with retained probes: +a durably created committed witness could be followed by rollback, a rejected +persisted journal could remain after a false complete-recovery result, and a +host observation failure inside error handling could escape without a typed +outcome. `parent-review/receipt-red.log` reproduces all six new cases before +production changes, including missing/persisted commit witnesses, rejected +journal/stage receipts and disconnection after stage/commit persistence. + +Application now validates a possibly persisted committed witness before choosing +rollback. A valid witness keeps candidate bytes and backups for explicit +completion; an absent witness permits rollback. Explicit recovery preserves +rejected transaction-shaped artifacts and reports recovery required while they +remain. The failure-handler fallback returns known or possible evidence and +unresolved targets without further storage observations or blind rollback. +All six regressions pass in `parent-review/receipt-green.log`; the full IO suite +passes 442 cases. Published smoke coverage adds these commit/receipt cases. + +The independent read-only audit classified the original 114 retained files +across 37 directories as generated test evidence: 28 target-drift, 28 backup-drift, +35 committed-cleanup, 14 partial-journal ownership, seven partial-journal cleanup +and two malformed-parser samples. No external/non-fixture references, links or +unique user data were found. The disposition inventory records paths, hashes, +provenance and inspection results. Original files remain untouched; parent-owned +fixture cleanup is authorized after the final review/landing gates. Additional +parent regression fixtures are retained separately under the same owned root. + +All 17 configured runners pass after the receipt/recovery corrections: IO 442, +tooling 436, core 177 and Microsoft 22; other counts are unchanged. The rebuilt +solution passes with eleven existing warnings and zero errors. Changed source +files passed formatting before the rebuild. The original disposition inventory +was rechecked after parent validation: all 149 recorded artifact/adjacent-file +hashes still match, recorded in `parent-review/original-evidence-hash-check.json`. +Both independent review scopes are complete; their six findings are addressed. +Final coverage, published hosts and current-head CI remain qualification gates. diff --git a/src/Morphir.Configuration.IO/ConfigurationRecovery.cs b/src/Morphir.Configuration.IO/ConfigurationRecovery.cs index 2723190d..087b22ac 100644 --- a/src/Morphir.Configuration.IO/ConfigurationRecovery.cs +++ b/src/Morphir.Configuration.IO/ConfigurationRecovery.cs @@ -181,6 +181,15 @@ ConfigurationTransactionResult Required(string code, IEnumerable? restor if (!SameOperation(initial, document)) return Required("config.recovery_changed"); if (document.Guards.Any(g => host.EnumerateTransactionArtifacts(g, document.TransactionId) is null)) return Required("config.recovery_host_capability"); + // Rejected artifact receipts are not cleanup authority. Keep durable + // metadata while any rejected transaction-shaped evidence still exists. + var rejected = document.UnverifiedPaths.Where(path => ProjectPath.isAbsolute(path) + && document.Guards.Contains(ProjectPath.parent(path), StringComparer.Ordinal) + && Path.GetFileName(path).StartsWith(".morphir-edit-" + document.TransactionId + "-", StringComparison.Ordinal) + && host.Observe(path).Kind != ConfigurationObservationKind.Missing).ToArray(); + if (rejected.Length > 0) + return Required("config.recovery_unverified_artifact") with + { UnresolvedPaths = inspection.UnresolvedPaths.Concat(rejected).Distinct(StringComparer.Ordinal).ToImmutableArray() }; var restored = new List(); var unresolved = new List(); foreach (var entry in document.Entries.Reverse()) diff --git a/src/Morphir.Configuration.IO/ConfigurationTransaction.cs b/src/Morphir.Configuration.IO/ConfigurationTransaction.cs index 410320f8..660dfc02 100644 --- a/src/Morphir.Configuration.IO/ConfigurationTransaction.cs +++ b/src/Morphir.Configuration.IO/ConfigurationTransaction.cs @@ -331,50 +331,84 @@ void RecordRecoveryEvidence() } catch (Exception error) when (IsIo(error)) { - if (error is ConfigurationArtifactFailureException artifactFailure) ReportArtifacts(artifactFailure); - foreach (var path in stageAttempts.Keys.Where(p => !artifacts.Any(a => a.Path == p))) - if (host.Observe(path).Kind != ConfigurationObservationKind.Missing) Unverified(path); - if (document is not null) - foreach (var directory in document.PendingDirectories) - if (!artifacts.Any(a => a.Kind == ConfigurationArtifactKind.Directory && a.Path == directory) - && host.Observe(directory).Kind != ConfigurationObservationKind.Missing) unresolved.Add(directory); - foreach (var (change, backup) in attempted.AsEnumerable().Reverse()) + try { - try + if (error is ConfigurationArtifactFailureException artifactFailure) ReportArtifacts(artifactFailure); + foreach (var path in stageAttempts.Keys.Where(p => !artifacts.Any(a => a.Path == p))) + if (host.Observe(path).Kind != ConfigurationObservationKind.Missing) Unverified(path); + if (document is not null) + foreach (var directory in document.PendingDirectories) + if (!artifacts.Any(a => a.Kind == ConfigurationArtifactKind.Directory && a.Path == directory) + && host.Observe(directory).Kind != ConfigurationObservationKind.Missing) unresolved.Add(directory); + if (document is { Committed: true }) { - var current = host.Observe(change.Path); - if (current == change.Expected) { restored.Add(change.Path); continue; } - var candidate = CandidateObservation(change); - if (current != candidate) { unresolved.Add(change.Path); continue; } - fault?.Invoke(ConfigurationTransactionPhase.BeforeRollback, change.Path); - if (host.Observe(change.Path) != candidate || backup is not null && host.Observe(backup) != - new ConfigurationObservation(backup, ConfigurationObservationKind.File, change.Expected.Identity)) - { unresolved.Add(change.Path); continue; } - if (backup is null) host.DeleteFile(change.Path); - else host.Install(backup, change.Path, true); - fault?.Invoke(ConfigurationTransactionPhase.AfterRollback, change.Path); - if (host.Observe(change.Path) == change.Expected) restored.Add(change.Path); - else unresolved.Add(change.Path); + var witnessPath = RecoveryDocument.ArtifactPath(document.MetadataDestination, transactionId, "journal-committed"); + if (host.Observe(witnessPath).Kind != ConfigurationObservationKind.Missing) + { + var witness = ReadRecovery(witnessPath); + if (!witness.Committed || !SameOperation(document, witness) + || !witness.Encode().AsSpan().SequenceEqual(document.Encode())) + throw new InvalidDataException("Commit witness does not match the attempted transaction"); + if (!artifacts.Any(a => a.Path == witnessPath)) + artifacts.Add(new(witnessPath, ConfigurationArtifactKind.Journal, transactionId)); + // Durable commit is authoritative even when its acknowledgement failed. + // Preserve candidate bytes and backups for explicit cleanup/recovery. + return new(ConfigurationTransactionStatus.RecoveryRequired, "config.edit_cleanup_required", targets, Retained(artifacts)) + { UnresolvedPaths = targets }; + } + document = document with { Committed = false }; } - catch (Exception recoveryError) when (IsIo(recoveryError)) + foreach (var (change, backup) in attempted.AsEnumerable().Reverse()) { - try { if (host.Observe(change.Path) == change.Expected) restored.Add(change.Path); else unresolved.Add(change.Path); } - catch (Exception observationError) when (IsIo(observationError)) { unresolved.Add(change.Path); } + try + { + var current = host.Observe(change.Path); + if (current == change.Expected) { restored.Add(change.Path); continue; } + var candidate = CandidateObservation(change); + if (current != candidate) { unresolved.Add(change.Path); continue; } + fault?.Invoke(ConfigurationTransactionPhase.BeforeRollback, change.Path); + if (host.Observe(change.Path) != candidate || backup is not null && host.Observe(backup) != + new ConfigurationObservation(backup, ConfigurationObservationKind.File, change.Expected.Identity)) + { unresolved.Add(change.Path); continue; } + if (backup is null) host.DeleteFile(change.Path); + else host.Install(backup, change.Path, true); + fault?.Invoke(ConfigurationTransactionPhase.AfterRollback, change.Path); + if (host.Observe(change.Path) == change.Expected) restored.Add(change.Path); + else unresolved.Add(change.Path); + } + catch (Exception recoveryError) when (IsIo(recoveryError)) + { + try { if (host.Observe(change.Path) == change.Expected) restored.Add(change.Path); else unresolved.Add(change.Path); } + catch (Exception observationError) when (IsIo(observationError)) { unresolved.Add(change.Path); } + } } + if (unverified.Count > 0) RecordRecoveryEvidence(); + if (unresolved.Count > 0) + return new(ConfigurationTransactionStatus.RecoveryRequired, "config.edit_recovery_required", targets, + Retained(artifacts)) + { RestoredPaths = restored.ToImmutableArray(), UnresolvedPaths = unresolved.Distinct().ToImmutableArray() }; + if (!TryCleanup(artifacts, false, document, artifactEvidence, partialArtifacts)) + return new(ConfigurationTransactionStatus.RecoveryRequired, "config.edit_cleanup_required", targets, Retained(artifacts)) + { RestoredPaths = restored.ToImmutableArray(), UnresolvedPaths = targets }; + return new(error is ConflictException ? ConfigurationTransactionStatus.Conflict + : attempted.Count > 0 ? ConfigurationTransactionStatus.RolledBack : ConfigurationTransactionStatus.IoFailure, + error is ConflictException ? "config.edit_conflict" + : attempted.Count > 0 ? "config.edit_rolled_back" : "config.edit_io", targets, []) + { RestoredPaths = restored.ToImmutableArray() }; + } + catch (Exception observationError) when (IsRecoveryError(observationError)) + { + // The host may be disconnected after a durable write. Report possible + // owned locations without querying it again or attempting blind rollback. + var possible = artifacts.Concat(stageAttempts.Select(p => new ConfigurationArtifact(p.Key, p.Value, transactionId))) + .Distinct().ToImmutableArray(); + return new(ConfigurationTransactionStatus.RecoveryRequired, "config.edit_recovery_io", targets, possible) + { + RestoredPaths = restored.ToImmutableArray(), + UnresolvedPaths = targets.Concat(unresolved).Concat(unverified).Concat(plannedDirectories) + .Distinct(StringComparer.Ordinal).ToImmutableArray() + }; } - if (unverified.Count > 0) RecordRecoveryEvidence(); - if (unresolved.Count > 0) - return new(ConfigurationTransactionStatus.RecoveryRequired, "config.edit_recovery_required", targets, - Retained(artifacts)) - { RestoredPaths = restored.ToImmutableArray(), UnresolvedPaths = unresolved.Distinct().ToImmutableArray() }; - if (!TryCleanup(artifacts, false, document, artifactEvidence, partialArtifacts)) - return new(ConfigurationTransactionStatus.RecoveryRequired, "config.edit_cleanup_required", targets, Retained(artifacts)) - { RestoredPaths = restored.ToImmutableArray(), UnresolvedPaths = targets }; - return new(error is ConflictException ? ConfigurationTransactionStatus.Conflict - : attempted.Count > 0 ? ConfigurationTransactionStatus.RolledBack : ConfigurationTransactionStatus.IoFailure, - error is ConflictException ? "config.edit_conflict" - : attempted.Count > 0 ? "config.edit_rolled_back" : "config.edit_io", targets, []) - { RestoredPaths = restored.ToImmutableArray() }; } finally { coordination?.Dispose(); } } diff --git a/tests/Morphir.Configuration.IO.Smoke/RecoveryReceiptSmoke.cs b/tests/Morphir.Configuration.IO.Smoke/RecoveryReceiptSmoke.cs new file mode 100644 index 00000000..e4b7dbf1 --- /dev/null +++ b/tests/Morphir.Configuration.IO.Smoke/RecoveryReceiptSmoke.cs @@ -0,0 +1,79 @@ +// Copyright 2026 FINOS +// SPDX-License-Identifier: Apache-2.0 +using System.Text; +using Morphir.Configuration; +using Morphir.Configuration.IO; + +internal static class RecoveryReceiptSmoke +{ + internal static int Run() + { + var checks = 0; + void Check(bool value, string message) { if (!value) throw new InvalidOperationException("Recovery receipt smoke: " + message); checks++; } + foreach (var mode in new[] { "missing", "persisted", "disconnected", "rejected" }) + { + var root = ProjectPath.join(Path.GetTempPath(), "receipt-smoke-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + var target = root + "/morphir.toml"; + var foreign = root + "/user.txt"; + File.WriteAllText(target, "original=true\n"); File.WriteAllText(foreign, "user=true\n"); + try + { + var host = new ReceiptHost(mode, foreign); + var result = new ConfigurationTransaction(host).Apply(new ConfigurationTransactionPlan([], + [new ConfigurationFileChange(target, ConfigurationTransaction.Observe(target), Encoding.UTF8.GetBytes("candidate=true\n"))])); + var committed = mode is "persisted" or "disconnected"; + Check(result.Status == (mode == "missing" ? ConfigurationTransactionStatus.RolledBack : ConfigurationTransactionStatus.RecoveryRequired), "receipt outcome lost"); + Check(File.ReadAllText(target) == (committed ? "candidate=true\n" : "original=true\n"), "commit boundary lost"); + var native = new ConfigurationTransaction(); + if (mode != "missing") + { + var journal = result.Artifacts.First(a => a.Kind == ConfigurationArtifactKind.Journal).Path; + if (mode == "rejected") + { + Check(native.Recover(journal).Status == ConfigurationTransactionStatus.RecoveryRequired && File.Exists(host.Rejected), "rejected evidence falsely completed"); + Check(ProjectPath.contains(root, host.Rejected!) && File.ReadAllBytes(host.Rejected!).SequenceEqual(host.Bytes!) + && File.ReadAllText(foreign) == "user=true\n", "test disposition ownership failed"); + File.Delete(host.Rejected!); + } + Check(native.Recover(journal).Status == (committed ? ConfigurationTransactionStatus.Written : ConfigurationTransactionStatus.RolledBack), "explicit recovery failed"); + } + Check(native.InspectPending([root]).IsEmpty, "receipt recovery left guards"); + } + finally + { + if (Directory.EnumerateFiles(root, ".morphir-edit-*").Any()) Console.Error.WriteLine("Retained receipt smoke evidence: " + root); + else Directory.Delete(root, true); + } + } + return checks; + } + + private sealed class ReceiptHost(string mode, string foreign) : IConfigurationTransactionHost + { + private readonly NativeConfigurationTransactionHost native = new(); + private bool disconnected; + public string? Rejected { get; private set; } + public byte[]? Bytes { get; private set; } + public bool SupportsDurableRecovery => true; + public ConfigurationObservation Observe(string path, IReadOnlyCollection? excludedPaths = null) => disconnected ? throw new IOException("disconnected") : native.Observe(path, excludedPaths); + public byte[] ReadFile(string path) => native.ReadFile(path); + public IDisposable Acquire(IReadOnlyList paths) => native.Acquire(paths); + public IReadOnlyList CreateParentDirectories(string path) => native.CreateParentDirectories(path); + public IReadOnlyList CreateParentDirectories(string path, string id) => native.CreateParentDirectories(path, id); + public string Stage(string destination, ReadOnlyMemory content, string id, string role) + { + if (mode == "missing" && role == "journal-committed") throw new IOException("not persisted"); + var path = native.Stage(destination, content, id, role); + if (mode == "rejected" && role == "journal") { Rejected = path; Bytes = content.ToArray(); return foreign; } + if (mode == "disconnected" && role == "journal-committed") { disconnected = true; throw new IOException("receipt lost"); } + if (mode == "persisted" && role == "journal-committed") throw new ConfigurationArtifactFailureException([new(path, ConfigurationArtifactKind.Journal, id)]); + return path; + } + public void Install(string stage, string destination, bool overwrite) => native.Install(stage, destination, overwrite); + public void DeleteFile(string path) => native.DeleteFile(path); + public void DeleteDirectory(string path) => native.DeleteDirectory(path); + public IReadOnlyList FindRecoveryJournals(IReadOnlyList paths) => native.FindRecoveryJournals(paths); + public IReadOnlyList? EnumerateTransactionArtifacts(string directory, string id) => native.EnumerateTransactionArtifacts(directory, id); + } +} diff --git a/tests/Morphir.Configuration.IO.Smoke/WorkspaceMigrationSmoke.cs b/tests/Morphir.Configuration.IO.Smoke/WorkspaceMigrationSmoke.cs index 2b16c417..c56d535f 100644 --- a/tests/Morphir.Configuration.IO.Smoke/WorkspaceMigrationSmoke.cs +++ b/tests/Morphir.Configuration.IO.Smoke/WorkspaceMigrationSmoke.cs @@ -123,7 +123,7 @@ internal static int Run() if (Directory.EnumerateFileSystemEntries(root, ".morphir-edit-*", SearchOption.AllDirectories).Any()) Console.Error.WriteLine("Retained workspace smoke recovery: " + root); else Directory.Delete(root, true); } - return checks; + return checks + RecoveryReceiptSmoke.Run(); } private sealed class InterruptedParentHost : IConfigurationTransactionHost diff --git a/tests/Morphir.Configuration.IO.Tests/ConfigurationRecoveryReceiptTests.cs b/tests/Morphir.Configuration.IO.Tests/ConfigurationRecoveryReceiptTests.cs new file mode 100644 index 00000000..80e388b6 --- /dev/null +++ b/tests/Morphir.Configuration.IO.Tests/ConfigurationRecoveryReceiptTests.cs @@ -0,0 +1,133 @@ +// Copyright 2026 FINOS +// SPDX-License-Identifier: Apache-2.0 +using System.Text; +using FluentAssertions; +using Morphir.Configuration; +using Morphir.Configuration.IO; + +namespace Morphir.Configuration.IO.Tests; + +public sealed class ConfigurationRecoveryReceiptTests +{ + [Test] + [Arguments(false)] + [Arguments(true)] + public void CommitReceiptFailureRespectsWhetherTheWitnessPersisted(bool persisted) + { + using var fixture = new Fixture(persisted ? "persisted-commit" : "missing-commit"); + var result = new ConfigurationTransaction(fixture.Host).Apply(fixture.Plan()); + if (persisted) + { + result.Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); + File.ReadAllText(fixture.Target).Should().Be(Fixture.Candidate, "a durable commit witness prevents rollback"); + var journal = result.Artifacts.First(a => a.Kind == ConfigurationArtifactKind.Journal).Path; + new ConfigurationTransaction().Recover(journal).Status.Should().Be(ConfigurationTransactionStatus.Written); + } + else + { + result.Status.Should().Be(ConfigurationTransactionStatus.RolledBack); + File.ReadAllText(fixture.Target).Should().Be(Fixture.Original); + } + new ConfigurationTransaction().InspectPending([fixture.Root]).Should().BeEmpty(); + fixture.Resolved = true; + } + + [Test] + [Arguments("journal")] + [Arguments("stage")] + public void RejectedPersistedArtifactsRemainRecoveryBlockersUntilDisposition(string role) + { + using var fixture = new Fixture("foreign-" + role); + var result = new ConfigurationTransaction(fixture.Host).Apply(fixture.Plan()); + result.Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); + var journal = result.Artifacts.First(a => a.Kind == ConfigurationArtifactKind.Journal).Path; + var recovery = new ConfigurationTransaction().Recover(journal); + recovery.Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired, "rejected evidence is still present"); + recovery.UnresolvedPaths.Should().Contain(fixture.Host.CreatedPath); + File.Exists(fixture.Host.CreatedPath).Should().BeTrue(); + new ConfigurationTransaction().Apply(fixture.Plan(Fixture.Original)).Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); + File.ReadAllText(fixture.Foreign).Should().Be("user=true\n"); + // Explicit test-owned disposition after proving the rejected file's exact origin and bytes. + ProjectPath.contains(fixture.Root, fixture.Host.CreatedPath!).Should().BeTrue(); + File.ReadAllBytes(fixture.Host.CreatedPath!).Should().Equal(fixture.Host.CreatedBytes!); + File.Delete(fixture.Host.CreatedPath!); + new ConfigurationTransaction().Recover(journal).Status.Should().Be(ConfigurationTransactionStatus.RolledBack); + new ConfigurationTransaction().InspectPending([fixture.Root]).Should().BeEmpty(); + fixture.Resolved = true; + } + + [Test] + [Arguments("stage")] + [Arguments("journal-committed")] + public void DisconnectionAfterDurableStageReturnsTypedRecoverableMetadata(string role) + { + using var fixture = new Fixture("disconnect-" + role); + var result = new ConfigurationTransaction(fixture.Host).Apply(fixture.Plan()); + result.Status.Should().Be(ConfigurationTransactionStatus.RecoveryRequired); + result.Artifacts.Should().Contain(a => a.Kind == ConfigurationArtifactKind.Journal); + result.UnresolvedPaths.Should().Contain(fixture.Target); + var journal = result.Artifacts.First(a => a.Kind == ConfigurationArtifactKind.Journal).Path; + var recovered = new ConfigurationTransaction().Recover(journal); + recovered.Status.Should().Be(role == "stage" ? ConfigurationTransactionStatus.RolledBack : ConfigurationTransactionStatus.Written); + File.ReadAllText(fixture.Target).Should().Be(role == "stage" ? Fixture.Original : Fixture.Candidate); + new ConfigurationTransaction().InspectPending([fixture.Root]).Should().BeEmpty(); + fixture.Resolved = true; + } + + private sealed class Fixture : IDisposable + { + public const string Original = "original=true\n"; + public const string Candidate = "candidate=true\n"; + public string Root { get; } = ProjectPath.join(Path.GetTempPath(), "recovery-receipt-" + Guid.NewGuid().ToString("N")); + public string Target => Root + "/morphir.toml"; + public string Foreign => Root + "/user.txt"; + public ReceiptHost Host { get; } + public bool Resolved { get; set; } + public Fixture(string mode) + { + Directory.CreateDirectory(Root); + File.WriteAllText(Target, Original); + File.WriteAllText(Foreign, "user=true\n"); + Host = new(mode, Foreign); + } + public ConfigurationTransactionPlan Plan(string content = Candidate) => new([], + [new ConfigurationFileChange(Target, ConfigurationTransaction.Observe(Target), Encoding.UTF8.GetBytes(content))]); + public void Dispose() + { + if (Resolved) Directory.Delete(Root, true); + else Console.Error.WriteLine("Retained receipt regression evidence: " + Root); + } + } + + private sealed class ReceiptHost(string mode, string foreign) : IConfigurationTransactionHost + { + private readonly NativeConfigurationTransactionHost native = new(); + private bool disconnected; + public string? CreatedPath { get; private set; } + public byte[]? CreatedBytes { get; private set; } + public bool SupportsDurableRecovery => true; + public ConfigurationObservation Observe(string path, IReadOnlyCollection? excludedPaths = null) => + disconnected ? throw new IOException("mount disconnected") : native.Observe(path, excludedPaths); + public byte[] ReadFile(string path) => native.ReadFile(path); + public IDisposable Acquire(IReadOnlyList paths) => native.Acquire(paths); + public IReadOnlyList CreateParentDirectories(string path) => native.CreateParentDirectories(path); + public IReadOnlyList CreateParentDirectories(string path, string id) => native.CreateParentDirectories(path, id); + public string Stage(string destination, ReadOnlyMemory content, string id, string role) + { + if (mode == "missing-commit" && role == "journal-committed") throw new IOException("commit not persisted"); + var path = native.Stage(destination, content, id, role); + if (mode == "foreign-" + role) + { CreatedPath = path; CreatedBytes = content.ToArray(); return foreign; } + if (mode == "disconnect-" + role) + { disconnected = true; throw new IOException("receipt lost after persistence"); } + if (mode == "persisted-commit" && role == "journal-committed") + throw new ConfigurationArtifactFailureException([new(path, ConfigurationArtifactKind.Journal, id)]); + return path; + } + public void Install(string stage, string target, bool overwrite) => native.Install(stage, target, overwrite); + public void DeleteFile(string path) => native.DeleteFile(path); + public void DeleteDirectory(string path) => native.DeleteDirectory(path); + public IReadOnlyList FindRecoveryJournals(IReadOnlyList paths) => native.FindRecoveryJournals(paths); + public IReadOnlyList? EnumerateTransactionArtifacts(string directory, string id) => native.EnumerateTransactionArtifacts(directory, id); + } +} From 7b448c6c232d93a70f4f1a21909a034a4239ae9f Mon Sep 17 00:00:00 2001 From: Damian Reeves <957246+DamianReeves@users.noreply.github.com> Date: Tue, 3 Nov 2026 18:55:29 -0600 Subject: [PATCH 13/14] test: qualify durable recovery review fixes --- .../design/prds/workspace-migration-delivery.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md b/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md index 2cedaff0..cab7910d 100644 --- a/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md +++ b/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md @@ -508,3 +508,17 @@ was rechecked after parent validation: all 149 recorded artifact/adjacent-file hashes still match, recorded in `parent-review/original-evidence-hash-check.json`. Both independent review scopes are complete; their six findings are addressed. Final coverage, published hosts and current-head CI remain qualification gates. + +Final receipt/recovery qualification after `ff495a7` passes full formatter, +OpenSpec/extension, fresh IO coverage and every published host mode. Core source +coverage is 4188/4345, 96.39%; IO is 2537/2606, 97.35%, still above the merged +prerequisite's 97.15%. No exclusions were added. Managed, trimmed and native +hosts each pass 59 workspace/recovery checks, 32 project migration checks, +44 editing checks, 228 JSON5 cases and 26 Morphir fixtures. Published stderr is +empty. Managed has no warnings; trimmed/native retain the same 44/47 dependency +warnings under explicit normal-warning qualification. Lockfiles are restored +byte-for-byte. Exact commands and outcomes are in +`parent-review/receipt-qualification-summary.json`, its coverage summary and the +publish summary. Both independent reviews are complete; all six findings plus +the independently reproduced ancestor-enumeration race are addressed and tested. +Current-head CI, landing and audited owned cleanup remain campaign gates. From af1412174aa6f564610f7985b5b8759c56eda2cd Mon Sep 17 00:00:00 2001 From: Damian Reeves <957246+DamianReeves@users.noreply.github.com> Date: Tue, 3 Nov 2026 19:06:17 -0600 Subject: [PATCH 14/14] test: normalize Windows recovery fixture paths --- .../prds/workspace-migration-delivery.md | 23 +++++++++++++++++++ .../ConfigurationArtifactOwnershipTests.cs | 2 +- .../ConfigurationTransactionTests.cs | 2 +- .../WorkspaceMigrationSteps.cs | 6 ++--- 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md b/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md index cab7910d..a17a6961 100644 --- a/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md +++ b/docs/content/docs/contributing/design/prds/workspace-migration-delivery.md @@ -522,3 +522,26 @@ byte-for-byte. Exact commands and outcomes are in publish summary. Both independent reviews are complete; all six findings plus the independently reproduced ancestor-enumeration race are addressed and tested. Current-head CI, landing and audited owned cleanup remain campaign gates. + +### Windows fixture path correction + +Development run `41747846560` on `9590b60` reports five Windows-only fixture +failures. The hidden-layout scenario moved the source successfully but failed +to remove its old dictionary entry because the original relative path contained +forward slashes and the removal used native separators. Four transaction tests +correctly retained tampered artifacts, then compared native enumerated paths +against the public API's normalized paths. The captured failing log is +`parent-review/ci-9590b60.clean.log`. + +The fixtures now normalize source dictionary keys and enumerated artifact paths +before comparing identities. Production behavior is unchanged. The full solution +rebuild passes with eleven existing warnings and zero errors. All 17 local +runners pass again with 3,903 tests/checks, including IO 442 and tooling 436. +Production source and published-host evidence remain unchanged; current-head +Windows CI remains the platform qualification gate. +Formatting and OpenSpec/extension verification also pass, recorded in +`parent-review/windows-fixture-postchecks.json`. The expanded read-only fixture +audit accounts for 326 retained recovery files after repeated validation, with +no references outside their individual fixtures, symlinks or unknown adjacent +file contents. The original 114-file audit remains intact. Explicit fixture +disposition and removal remain deferred until verified landing. diff --git a/tests/Morphir.Configuration.IO.Tests/ConfigurationArtifactOwnershipTests.cs b/tests/Morphir.Configuration.IO.Tests/ConfigurationArtifactOwnershipTests.cs index 88d3403c..3001997e 100644 --- a/tests/Morphir.Configuration.IO.Tests/ConfigurationArtifactOwnershipTests.cs +++ b/tests/Morphir.Configuration.IO.Tests/ConfigurationArtifactOwnershipTests.cs @@ -102,7 +102,7 @@ public void CleanupPreservesAnArtifactWhoseBytesChangedAfterStaging(string role) var result = new ConfigurationTransaction(fault: (phase, _) => { if (phase != ConfigurationTransactionPhase.BeforeCleanup) return; - changed = Directory.GetFiles(root, ".morphir-edit-*-" + role).Single(); + changed = ProjectPath.normalize(Directory.GetFiles(root, ".morphir-edit-*-" + role).Single()); original = File.ReadAllBytes(changed); // A valid JSON journal with added whitespace still has different user-owned bytes. if (role == "backup") File.WriteAllText(changed, changedText); diff --git a/tests/Morphir.Configuration.IO.Tests/ConfigurationTransactionTests.cs b/tests/Morphir.Configuration.IO.Tests/ConfigurationTransactionTests.cs index 07f8b650..18e9b310 100644 --- a/tests/Morphir.Configuration.IO.Tests/ConfigurationTransactionTests.cs +++ b/tests/Morphir.Configuration.IO.Tests/ConfigurationTransactionTests.cs @@ -437,7 +437,7 @@ public void StagingIntegrityIsCheckedBeforeInstallation(string tampered) var result = new ConfigurationTransaction(fault: (phase, _) => { if (phase != ConfigurationTransactionPhase.BeforeInstall) return; - artifactPath = Directory.GetFiles(directory.Path, ".morphir-edit-*-" + tampered).Single(); + artifactPath = ProjectPath.normalize(Directory.GetFiles(directory.Path, ".morphir-edit-*-" + tampered).Single()); original = File.ReadAllBytes(artifactPath); File.WriteAllText(artifactPath, "tampered = true\n"); }).Apply(plan); diff --git a/tests/Morphir.Tooling.Tests/Features/WorkspaceMigration/WorkspaceMigrationSteps.cs b/tests/Morphir.Tooling.Tests/Features/WorkspaceMigration/WorkspaceMigrationSteps.cs index 72d032db..636fb36b 100644 --- a/tests/Morphir.Tooling.Tests/Features/WorkspaceMigration/WorkspaceMigrationSteps.cs +++ b/tests/Morphir.Tooling.Tests/Features/WorkspaceMigration/WorkspaceMigrationSteps.cs @@ -50,7 +50,7 @@ private void Classic(string relative, string text) var path = Path.Combine(_root, relative, "morphir.json"); Directory.CreateDirectory(Path.GetDirectoryName(path)!); File.WriteAllText(path, text); - _sources[path] = File.ReadAllBytes(path); + _sources[ProjectPath.normalize(path)] = File.ReadAllBytes(path); } private void Write(string relative, string text) @@ -108,8 +108,8 @@ public void HiddenMember() var hidden = Path.Combine(_root, "apps", "a", ".config", "morphir", "config.json"); Directory.CreateDirectory(Path.GetDirectoryName(hidden)!); File.Move(original, hidden); - _sources.Remove(original); - _sources[hidden] = File.ReadAllBytes(hidden); + _sources.Remove(ProjectPath.normalize(original)); + _sources[ProjectPath.normalize(hidden)] = File.ReadAllBytes(hidden); } [Given("a declared empty migration workspace")]