From e4073e60a3f017eff665138b71a51a93ac9b3b0f Mon Sep 17 00:00:00 2001 From: Damian Reeves <957246+DamianReeves@users.noreply.github.com> Date: Sat, 31 Oct 2026 10:36:49 -0500 Subject: [PATCH 1/2] feat(cli): compile source trees with relative URIs and a claimed language Implement tasks 1.3, 2.1, 2.2, 2.4 and 2.5 of the OpenSpec change run-core-wasm-guests-from-any-language. - `--language` selects one of several claimed languages (SEL-3). Without it, several claimed languages give extension.input_invalid; an unclaimed language gives extension.capability_unavailable. The walk reads only the file extensions of the selected language. - Each document URI is the path relative to the source base: `--source-base`, else the current directory (SRC-1). An input outside the base is extension.input_invalid. A single file without `--source-base` keeps the earlier rule. - Each URI segment is percent-encoded as an RFC 3986 path segment, so names with `:`, `\` or a space are encoded, not refused (SRC-3). - Documents are sorted by the whole URI in byte order, a repeated URI is refused, and one file is bounded to 1 MiB. Limit messages name the limit and its value; link messages name the relative path; UTF-8 messages name the URI (SRC-2, FIT-1). - Tests prove byte-identical requests for the same tree and the same contextDigest for other trees. - Document the source base, the encoding, the order and the limits in apps/morphir/docs/extension.md. - Read the stop probe in the WASI entry, which clears an unused-field warning on wasm and wasm-gc. Checked with the released morphir-gleam 0.4.0 module (native and npm CLI). --- apps/morphir/cli/arguments.mbt | 9 +- apps/morphir/cli/extension.mbt | 376 +++++++++++++---- apps/morphir/cli/extension_session_wbtest.mbt | 282 ++++++++++++- apps/morphir/cli/extension_unhosted.mbt | 3 +- apps/morphir/cli/extension_wbtest.mbt | 393 ++++++++++++++++-- apps/morphir/cli/main.mbt | 2 +- apps/morphir/cli/mep_recording_wbtest.mbt | 12 +- apps/morphir/docs/extension.md | 130 +++++- .../tasks.md | 43 +- 9 files changed, 1087 insertions(+), 163 deletions(-) diff --git a/apps/morphir/cli/arguments.mbt b/apps/morphir/cli/arguments.mbt index ebd15ccd5..129b60a9f 100644 --- a/apps/morphir/cli/arguments.mbt +++ b/apps/morphir/cli/arguments.mbt @@ -248,7 +248,10 @@ fn command_schema(path : String) -> CliSchema { "extension describe" => options = extension_options "extension compile" => { options = extension_options + - ["package", "frontend-profile", "model-format", "output", "ir-version"] + [ + "package", "frontend-profile", "model-format", "output", "ir-version", + "language", "source-base", + ] positional = true } "extension generate" => { @@ -370,7 +373,9 @@ fn render_help(path : String) -> String { text += "A contract 0.2.0 pin names the artifact (path, sha256), its runtime and an optional descriptor.\n" text += "The host probes the worker, then runs one fresh session; no helper or Node runtime is used.\n" text += "Process and WASM runtimes run on the native and npm CLIs; WASI builds refuse them with extension.runtime_unavailable.\n" - text += "Sources and operations come from the extension's claims: compile reads the claimed file extensions.\n" + text += "Sources and operations come from the extension's claims: compile reads the file extensions of one claimed language.\n" + text += "--language selects that language when the extension claims several.\n" + text += "Each document URI is the file's path relative to --source-base (default: the current directory), percent-encoded.\n" text += "compile writes model-envelope.json and model.ionb or model.json, or morphir-ir.json on MEP 0.1.\n" text += "generate reads a model envelope or Morphir IR JSON and writes the returned artifacts.\n" text += "JSON returns morphir/command-result 0.1.0; diagnostics and guest stderr go to stderr.\n" diff --git a/apps/morphir/cli/extension.mbt b/apps/morphir/cli/extension.mbt index 6e14dfd8c..67f8b5d52 100644 --- a/apps/morphir/cli/extension.mbt +++ b/apps/morphir/cli/extension.mbt @@ -7,11 +7,12 @@ let extension_migration : String = "https://github.com/finos/morphir-moonbit/blo let extension_pin_contract : String = "0.2.0" ///| -/// The source snapshot bounds of `extension compile`: files read, bytes read -/// in total, directory entries listed and directory depth. Each bound is -/// inclusive. +/// The source snapshot bounds of `extension compile`: documents read, bytes +/// of one file, bytes read in total, directory entries listed and directory +/// depth. Each bound is inclusive. priv struct ExtensionSourceLimits { files : Int32 + file_bytes : Int32 bytes : Int32 entries : Int32 depth : Int32 @@ -20,6 +21,7 @@ priv struct ExtensionSourceLimits { ///| let extension_source_limits : ExtensionSourceLimits = { files: 1024, + file_bytes: 1048576, bytes: 2097152, entries: 16384, depth: 32, @@ -280,6 +282,11 @@ priv struct ExtensionInputs { ir_version : String target : String timeout_ms : Int32 + /// `--language`: the claimed language to compile, when the extension + /// claims several. + language : @ids.LanguageId? + /// `--source-base`: the directory that document URIs are relative to. + source_base : String? } ///| @@ -306,6 +313,8 @@ fn ExtensionInputs::read( ir_version: value(cli, "ir-version"), target: value(cli, "target"), timeout_ms, + language: None, + source_base: None, } match command { Describe => { @@ -355,7 +364,21 @@ fn ExtensionInputs::read( "the parent directory \{parent} of --output \{output} does not exist", ) } - { ..inputs, codec, output, } + let language = match value(cli, "language") { + "" => None + text => + Some( + @ids.LanguageId::parse(text) catch { + _ => + raise input_invalid("--language \{text} is not a language ID") + }, + ) + } + let source_base = match value(cli, "source-base") { + "" => None + base => Some(base) + } + { ..inputs, codec, output, language, source_base, } } } } @@ -388,8 +411,45 @@ priv struct ExtensionSource { } ///| -/// A clean relative path keeps its nesting; otherwise only the file name is -/// declared. Absolute host paths never enter the model's source provenance. +/// One source file that the walk found: its document URI, its path relative +/// to the source base for messages, and its path on the host. +priv struct ExtensionSourceFile { + uri : String + relative : String + path : String +} + +///| +/// Encodes one name as an RFC 3986 path segment. Unreserved characters +/// (letters, digits, `-`, `.`, `_` and `~`) stay as they are; every other +/// byte of the UTF-8 name is percent-encoded with uppercase digits, so a +/// URI never holds a raw `%`, `?`, `#`, `\`, `:`, space, control character +/// or non-ASCII character. +fn extension_uri_segment(name : String) -> String { + let digits : FixedArray[Char] = [ + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', + ] + let builder = StringBuilder() + for byte in @utf8.encode(name) { + match byte.to_char() { + 'a'..<='z' | 'A'..<='Z' | '0'..<='9' | '-' | '.' | '_' | '~' as char => + builder.write_char(char) + _ => { + let code = byte.to_int() + builder.write_char('%') + builder.write_char(digits[code / 16]) + builder.write_char(digits[code % 16]) + } + } + } + builder.to_string() +} + +///| +/// The URI of a single file without `--source-base`. A clean relative path +/// keeps its nesting; otherwise only the file name is declared. Absolute +/// host paths never enter the model's source provenance. Each segment is +/// percent-encoded. fn extension_document_uri(path : String) -> String { let parts = path .replace_all(old="\\", new="/") @@ -397,56 +457,123 @@ fn extension_document_uri(path : String) -> String { .map(part => part.to_owned()) .to_array() let name = parts[parts.length() - 1] - if mck_absolute(path) || + let kept = if mck_absolute(path) || parts.any(part => { part == "" || part == "." || part == ".." || part.contains(":") }) { - name + [name] } else { - parts.join("/") + parts } + kept.map(extension_uri_segment).join("/") +} + +///| +/// The directory that document URIs are relative to (SRC-1): `--source-base` +/// when the user gives it, else the current directory. +priv struct ExtensionSourceBase { + given : String? + cwd : String + style : @worker.PathStyle } ///| -/// The file extensions of every language that the extension claims. -fn extension_claimed_extensions(claims : @worker.Claims) -> Array[String] { - let extensions : Array[String] = [] - for language in claims.languages { - for extension in language.file_extensions { - let text = extension.to_string() - if !extensions.contains(text) { - extensions.push(text) +/// The names from the source base down to `path`, which must be the base or +/// below it. Both paths are made absolute against the current directory and +/// normalized without reading the file system. +fn ExtensionSourceBase::segments( + self : ExtensionSourceBase, + path : String, +) -> Array[String] raise ExtensionError { + let base = self.given.unwrap_or(self.cwd) + let relative = match self.style { + Posix => { + let absolute = (text : String) => { + let path = @path_posix.Path(text) + let path = if path.is_absolute() { + path + } else { + @path_posix.Path(self.cwd).join(path) + } + path.normalize() + } + let relative = absolute(path).relative(base=absolute(base)) + if relative.is_absolute() { + None + } else { + Some(relative.0) } } + Windows => { + let absolute = (text : String) => { + let path = @path_windows.Path(text) + let path = if path.is_absolute() { + path + } else { + @path_windows.Path(self.cwd).join(path) + } + path.normalize() + } + let relative = absolute(path).relative(base=absolute(base)) + if relative.is_absolute() { + None + } else { + Some(relative.0.replace_all(old="\\", new="/")) + } + } + } + let segments = match relative { + Some(text) => + text + .split("/") + .filter(part => part != "" && part != ".") + .map(part => part.to_owned()) + .collect() + None => [".."] } - extensions + if segments.contains("..") { + raise input_invalid( + "\{path} is not at or below the source base \{base}; give a --source-base that contains it", + ) + } + segments } ///| -/// The claimed language of the documents: the first language whose file -/// extensions cover every document. The CLI holds no list of languages. +/// The language of a compile (SEL-3): the only language that the extension +/// claims, or the claimed language that `--language` names. The CLI holds +/// no list of languages. fn extension_language( claims : @worker.Claims, - documents : Array[ExtensionSource], -) -> (@ids.LanguageId, Array[ExtensionSource]) raise ExtensionError { - if documents.is_empty() { - raise input_invalid( - "no source file has a file extension that \{claims.extension_id} claims (\{extension_claimed_extensions(claims).join(", ")})", - ) - } - for language in claims.languages { - let covers = documents.all(document => { - language.file_extensions.any(extension => { - document.uri.has_suffix(extension.to_string()) - }) - }) - if covers { - return (language.language_id, documents) - } + selected : @ids.LanguageId?, +) -> @worker.LanguageClaim raise ExtensionError { + let names = claims.languages + .map(language => language.language_id.to_string()) + .join(", ") + match selected { + Some(id) => + match claims.languages.search_by(language => language.language_id == id) { + Some(index) => claims.languages[index] + None => + raise refused( + Worker(CapabilityUnavailable), + "\{claims.extension_id} does not claim the language \{id}; it claims \{if names == "" { "no language" } else { names }}", + ) + } + None => + match claims.languages { + [only] => only + [] => + raise refused( + Worker(CapabilityUnavailable), + "\{claims.extension_id} claims no source language", + ) + _ => + raise input_invalid( + "\{claims.extension_id} claims the languages \{names}: select one with --language", + ) + } } - raise input_invalid( - "the sources mix languages; \{claims.extension_id} compiles one language at a time", - ) } ///| @@ -466,8 +593,8 @@ fn extension_document_count( } ///| -/// Name order by UTF-16 code unit, so that `app` comes before `nested`. -/// `String` compares lengths first. +/// Order by UTF-16 code unit, so that `app` comes before `nested`. `String` +/// compares lengths first. A URI is ASCII, so this is its byte order. fn extension_name_order(left : String, right : String) -> Int32 { let length = if left.length() < right.length() { left.length() @@ -483,17 +610,14 @@ fn extension_name_order(left : String, right : String) -> Int32 { } ///| -/// Why a directory entry name cannot become part of a document URI, or -/// `None` when it can. A name with a separator, a drive colon or a NUL, or a -/// name that is empty, `.` or `..`, would make the URI absolute or -/// traversing. +/// Why a directory entry name cannot become a URI segment, or `None` when it +/// can. An empty name, `.` or `..` would make the URI traverse, and a name +/// with `/` or NUL is not one entry. Every other name is percent-encoded. fn extension_source_name(name : String) -> String? { if name == "" || name == "." || name == ".." { Some("is empty, . or ..") - } else if name.contains("/") || name.contains("\\") { + } else if name.contains("/") { Some("contains a path separator") - } else if name.contains(":") { - Some("contains a colon") } else if name.contains_char('\u0000') { Some("contains NUL") } else { @@ -501,88 +625,102 @@ fn extension_source_name(name : String) -> String? { } } +///| +/// Sorts the files by the whole URI in byte order and refuses two files +/// with the same URI. +fn extension_source_order( + files : Array[ExtensionSourceFile], +) -> Array[ExtensionSourceFile] raise ExtensionError { + let sorted = files.copy() + sorted.sort_by((left, right) => extension_name_order(left.uri, right.uri)) + for index in 1.. Unit raise ExtensionError, + base~ : ExtensionSourceBase, limits~ : ExtensionSourceLimits = extension_source_limits, ) -> Array[ExtensionSource] raise ExtensionError { - let documents : Array[ExtensionSource] = [] - let mut total = 0 + let files : Array[ExtensionSourceFile] = [] let mut entries = 0 let claimed = (name : String) => { extensions.any(extension => name.has_suffix(extension)) } - fn read(file : String, uri : String) -> Unit raise ExtensionError { - check() - if documents.length() >= limits.files { - raise input_invalid("the sources have more than \{limits.files} files") - } - let bytes = @platform.read_input(file, limits.bytes) catch { - error => - raise input_invalid( - "cannot read \{file}: \{@host.error_message(error)}", - ) - } - total += bytes.length() - if total > limits.bytes { - raise input_invalid("the sources have more than \{limits.bytes} bytes") - } - let text = @utf8.decode(bytes[:]) catch { - _ => raise input_invalid("\{file} is not UTF-8 text") - } - documents.push({ uri, text, }) - } - fn walk( directory : String, - relative : String, + names : Array[String], depth : Int32, ) -> Unit raise ExtensionError { check() if depth > limits.depth { raise input_invalid( - "the source directory is deeper than \{limits.depth} levels", + "the source directory \{names.join("/")} is deeper than the limit of \{limits.depth} directories", ) } - let names = @platform.list_bounded(directory, 4096) catch { + let listed = @platform.list_bounded(directory, 4096) catch { error => raise input_invalid( "cannot list \{directory}: \{@host.error_message(error)}", ) } - names.sort_by(extension_name_order) - for name in names { + listed.sort_by(extension_name_order) + for name in listed { entries += 1 if entries > limits.entries { raise input_invalid( - "the source directory has more than \{limits.entries} entries", + "the source directory has more entries than the limit of \{limits.entries} entries", ) } + let relative = [..names, name] if extension_source_name(name) is Some(reason) { raise input_invalid( - "the source entry \{repr(name)} in \{directory} \{reason}", + "the source entry \{repr(name)} in \{names.join("/")} \{reason}", ) } let full = directory + "/" + name - let uri = if relative == "" { name } else { relative + "/" + name } match (@platform.kind(full) catch { _ => -1 }) { - 1 => if claimed(name) { read(full, uri) } + 1 => + if claimed(name) { + files.push({ + uri: relative.map(extension_uri_segment).join("/"), + relative: relative.join("/"), + path: full, + }) + } 2 => if !name.has_prefix(".") && !extension_skipped_directories.contains(name) { - walk(full, uri, depth + 1) + walk(full, relative, depth + 1) } - _ => raise input_invalid("\{full} is a link or a special file") + _ => + raise input_invalid( + "\{relative.join("/")} is a link or a special file", + ) } } } + // An explicit source base is checked before the input is read. + let explicit = match base.given { + Some(_) => Some(base.segments(path)) + None => None + } match (@platform.kind(path) catch { _ => -1 }) { 1 => { if !claimed(path) { @@ -590,11 +728,65 @@ fn extension_sources( "\{path} does not have a claimed file extension (\{extensions.join(", ")})", ) } - read(path, extension_document_uri(path)) + let file : ExtensionSourceFile = match explicit { + None => { uri: extension_document_uri(path), relative: path, path, } + Some([]) => + raise input_invalid( + "\{path} is the source base itself; give a --source-base above it", + ) + Some(names) => + { + uri: names.map(extension_uri_segment).join("/"), + relative: names.join("/"), + path, + } + } + files.push(file) + } + 2 => { + let names = match explicit { + Some(names) => names + None => base.segments(path) + } + walk(path, names, 0) } - 2 => walk(path, "", 0) _ => raise input_invalid("\{path} is not a regular file or a directory") } + if files.is_empty() { + raise input_invalid( + "no source file in \{path} has a claimed file extension (\{extensions.join(", ")})", + ) + } + if files.length() > limits.files { + raise input_invalid( + "the sources have \{files.length()} documents, more than the limit of \{limits.files} documents", + ) + } + let documents : Array[ExtensionSource] = [] + let mut total = 0 + for file in extension_source_order(files) { + check() + let bytes = @platform.read_input(file.path, limits.file_bytes) catch { + @platform.IOError("input.limit") => + raise input_invalid( + "the source file \{file.uri} is larger than the limit of \{limits.file_bytes} bytes for one file", + ) + error => + raise input_invalid( + "cannot read \{file.relative}: \{@host.error_message(error)}", + ) + } + total += bytes.length() + if total > limits.bytes { + raise input_invalid( + "the sources are larger than the limit of \{limits.bytes} bytes of source text in total", + ) + } + let text = @utf8.decode(bytes[:]) catch { + _ => raise input_invalid("the source file \{file.uri} is not UTF-8 text") + } + documents.push({ uri: file.uri, text, }) + } documents } @@ -1458,12 +1650,18 @@ async fn extension_execute( ) Model(_) => None } + let language = extension_language(claims, inputs.language) let documents = extension_sources( inputs.path, - extension_claimed_extensions(claims), + language.file_extensions.map(extension => extension.to_string()), check, + base={ + given: inputs.source_base, + cwd: @platform.cwd(), + style: run.hosting.path_style(), + }, ) - let (language, documents) = extension_language(claims, documents) + let language = language.language_id extension_document_count(claims, documents) ( offer, diff --git a/apps/morphir/cli/extension_session_wbtest.mbt b/apps/morphir/cli/extension_session_wbtest.mbt index a92b7d6df..8dd483d6b 100644 --- a/apps/morphir/cli/extension_session_wbtest.mbt +++ b/apps/morphir/cli/extension_session_wbtest.mbt @@ -20,6 +20,8 @@ priv struct ScriptedLauncher { /// The bounds that the channel states, as a WASM host's channel does. bounds : @worker.EnforcedBounds? offers : Array[Json] + /// The encoded bytes of each operation call, as the worker received them. + requests : Array[Bytes] } ///| @@ -68,6 +70,7 @@ async fn fake_serve( methods : Array[String], calls : Array[Json], offers : Array[Json], + requests : Array[Bytes], ) -> Unit { worker.write_stderr(@utf8.encode(guest.stderr)) while true { @@ -93,6 +96,7 @@ async fn fake_serve( "morphir.shutdown" => Some(guest.shutdown) _ => { calls.push(params) + requests.push(message.to_bytes()) (guest.reply)(name, params) } } @@ -118,7 +122,14 @@ async fn fake_serve( impl @worker.Launcher for ScriptedLauncher with fn launch(self, _spec, _limits) { let (channel, worker) = @worker.MemoryChannel::pair(bounds?=self.bounds) self.group.spawn_bg(() => { - fake_serve(worker, self.guest, self.methods, self.calls, self.offers) + fake_serve( + worker, + self.guest, + self.methods, + self.calls, + self.offers, + self.requests, + ) }) channel } @@ -131,6 +142,7 @@ priv struct FakeRun { methods : Array[String] calls : Array[Json] offers : Array[Json] + requests : Array[Bytes] } ///| @@ -150,6 +162,7 @@ async fn fake_run( calls: [], bounds, offers: [], + requests: [], } let hosting = if hosted { WorkerHosting::new( @@ -166,6 +179,7 @@ async fn fake_run( methods: launcher.methods, calls: launcher.calls, offers: launcher.offers, + requests: launcher.requests, }) }) guard finished.val is Some(run) else { abort("the run did not finish") } @@ -189,6 +203,7 @@ async fn fake_command( calls: [], bounds: None, offers: [], + requests: [], } let context = { ..command_context(mode), @@ -596,6 +611,8 @@ async test "Python 0.4.1 with two modules: a frontend without multiDocument is r "extension", "compile", sources, + "--source-base", + workspace.root, "--package", "acme/loans", "--output", @@ -671,6 +688,8 @@ async test "a frontend without model methods compiles on MEP 0.1 from the claime "extension", "compile", sources, + "--source-base", + workspace.root, "--package", "acme/app", "--output", @@ -694,7 +713,7 @@ async test "a frontend without model methods compiles on MEP 0.1 from the claime fail("no documents") } assert_eq(documents.map(document => fake_field(document, ["uri"])), [ - "app.gleam", "nested/lib.gleam", + "src/app.gleam", "src/nested/lib.gleam", ]) assert_eq(fake_field(request, ["options", "irVersion"]), "3") assert_eq(fake_file(output + "/morphir-ir.json"), fake_legacy_ir.stringify()) @@ -1187,6 +1206,7 @@ async test "an async cancellation of the command exits 130, not host_failed" { calls: [], bounds: None, offers: [], + requests: [], } let hosting = WorkerHosting::new( Native(@worker.Machine::of(X86_64UnknownLinuxGnu)), @@ -1479,3 +1499,261 @@ async test "compile reports the WASM bounds that the worker's host enforced in r } assert_false(fields.contains("memoryCeiling")) } + +///| +let fake_gleam_language : Json = { "id": "gleam", "fileExtensions": [".gleam"] } + +///| +/// The claims of a Gleam frontend like morphir-gleam 0.4.0: several +/// documents in one request, on the JSON model tuple. +fn fake_gleam_describe( + languages~ : Array[Json] = [fake_gleam_language], +) -> Json { + extension_test_claims( + frontend={ + "languages": Json::array(languages), + "irVersions": ["4.0.0"], + "compile": false, + "multiDocument": true, + }, + tuples=[ + extension_test_tuple("morphir.frontend.compileModel", "morphir-json"), + ], + ) +} + +///| +fn fake_gleam_guest( + languages~ : Array[Json] = [fake_gleam_language], +) -> FakeGuest { + fake_guest(describe=fake_gleam_describe(languages~), reply=(_, _) => { + Some(Ok(fake_json_model_result())) + }) +} + +///| +/// A Gleam project in `root`: modules in two directories under `src` that +/// import one another, and files of other languages. +fn fake_gleam_tree(root : String) -> Unit raise { + @platform.mkdir(root + "/src") + @platform.mkdir(root + "/src/app") + @platform.mkdir(root + "/src/shared") + @platform.write( + root + "/src/app/money.gleam", + "import shared/numbers\npub type Money = numbers.Number\n", + ) + @platform.write( + root + "/src/app/orders.gleam", + "import app/money\npub type Order = money.Money\n", + ) + @platform.write(root + "/src/shared/numbers.gleam", "pub type Number = Int\n") + @platform.write(root + "/src/README.md", "not a source\n") + @platform.write(root + "/src/main.elm", "module Main exposing (..)\n") +} + +///| +/// The arguments of a compile of `input` in the workspace, with `options` +/// before the shared ones. +fn fake_tree_compile( + workspace : FakeWorkspace, + input : String, + output : String, + options : Array[String], +) -> Array[String] { + [ + "extension", + "compile", + input, + ..options, + "--package", + "acme/shop", + "--output", + workspace.root + "/" + output, + "--extension-pin", + workspace.pin, + "--json", + ] +} + +///| +fn fake_uris(request : Json) -> Array[Json] { + guard fake_field(request, ["sources", "documents"]) is Array(documents) else { + return [] + } + documents.map(document => fake_field(document, ["uri"])) +} + +///| +async test "language from claims: a guest with several languages needs --language" { + let workspace = fake_workspace() + fake_gleam_tree(workspace.root) + let elm : Json = { "id": "elm", "fileExtensions": [".elm"] } + let base = ["--source-base", workspace.root] + let source = workspace.root + "/src" + // Several claimed languages and no --language: refused before any + // operation session, and the message names both languages. + let run = fake_run( + fake_tree_compile(workspace, source, "none", base), + fake_gleam_guest(languages=[elm, fake_gleam_language]), + ) + assert_eq(run.report.exit_code(), 2, msg=run.json.stringify()) + assert_eq(fake_error(run), "extension.input_invalid") + let message = fake_field(run.json, ["error", "message"]).stringify() + assert_true(message.contains("elm") && message.contains("gleam"), msg=message) + assert_eq(run.methods, ["morphir.extension.describe", "morphir.exit"]) + assert_false(fake_exists(workspace.root + "/none")) + // An unclaimed language is a capability refusal, also before any session. + let run = fake_run( + fake_tree_compile( + workspace, + source, + "python", + [..base, "--language", "python"], + ), + fake_gleam_guest(languages=[elm, fake_gleam_language]), + ) + assert_eq(run.report.exit_code(), 2, msg=run.json.stringify()) + assert_eq(fake_error(run), "extension.capability_unavailable") + assert_eq(run.methods, ["morphir.extension.describe", "morphir.exit"]) + // --language gleam sends gleam and reads only the .gleam files. + let run = fake_run( + fake_tree_compile( + workspace, + source, + "gleam", + [..base, "--language", "gleam"], + ), + fake_gleam_guest(languages=[elm, fake_gleam_language]), + ) + assert_eq(run.report.exit_code(), 0, msg=run.json.stringify()) + let request = run.calls[0] + assert_eq(fake_field(request, ["languageId"]), "gleam") + assert_eq(fake_uris(request), [ + "src/app/money.gleam", "src/app/orders.gleam", "src/shared/numbers.gleam", + ]) + guard fake_field(request, ["sources", "documents"]) is Array(documents) else { + fail("no documents") + } + for document in documents { + assert_eq(fake_field(document, ["languageId"]), "gleam") + } +} + +///| +async test "relative document URIs: a tree is named from the source base, with no sources.root" { + let workspace = fake_workspace() + fake_gleam_tree(workspace.root) + // The guest claims one language, so no --language is needed. + let run = fake_run( + fake_tree_compile(workspace, workspace.root + "/src", "model", [ + "--source-base", + workspace.root, + ]), + fake_gleam_guest(), + ) + assert_eq(run.report.exit_code(), 0, msg=run.json.stringify()) + let request = run.calls[0] + assert_eq(fake_field(request, ["languageId"]), "gleam") + assert_eq(fake_uris(request), [ + "src/app/money.gleam", "src/app/orders.gleam", "src/shared/numbers.gleam", + ]) + guard fake_field(request, ["sources"]) is Object(sources) else { + fail("no sources") + } + assert_eq(sources.keys().collect(), ["documents"]) + assert_false(request.stringify().contains(workspace.root)) + // A deeper input keeps the URIs from the source base. + let run = fake_run( + fake_tree_compile(workspace, workspace.root + "/src/app", "app", [ + "--source-base", + workspace.root, + ]), + fake_gleam_guest(), + ) + assert_eq(fake_uris(run.calls[0]), [ + "src/app/money.gleam", "src/app/orders.gleam", + ]) + // An input outside the source base is refused before any operation + // session, and the message names the source base. + let run = fake_run( + fake_tree_compile(workspace, workspace.root + "/src", "outside", [ + "--source-base", + workspace.root + "/other", + ]), + fake_gleam_guest(), + ) + assert_eq(run.report.exit_code(), 2, msg=run.json.stringify()) + assert_eq(fake_error(run), "extension.input_invalid") + assert_true( + fake_field(run.json, ["error", "message"]) + .stringify() + .contains(workspace.root + "/other"), + msg=run.json.stringify(), + ) + assert_eq(run.methods, ["morphir.extension.describe", "morphir.exit"]) + assert_false(fake_exists(workspace.root + "/outside")) +} + +///| +async test "same tree twice gives byte-identical compileModel requests; other trees keep the contextDigest" { + let workspace = fake_workspace() + fake_gleam_tree(workspace.root) + let base = ["--source-base", workspace.root] + let first = fake_run( + fake_tree_compile(workspace, workspace.root + "/src", "first", base), + fake_gleam_guest(), + ) + let second = fake_run( + fake_tree_compile(workspace, workspace.root + "/src", "second", base), + fake_gleam_guest(), + ) + assert_eq(first.report.exit_code(), 0, msg=first.json.stringify()) + assert_eq(first.requests.length(), 1) + assert_eq(second.requests.length(), 1) + assert_true(first.requests[0] == second.requests[0]) + // The same package and options with another tree: other documents, the + // same contextDigest. + let other = fake_workspace() + @platform.mkdir(other.root + "/src") + @platform.write(other.root + "/src/ledger.gleam", "pub type Ledger = Int\n") + let third = fake_run( + fake_tree_compile(workspace, other.root + "/src", "third", [ + "--source-base", + other.root, + ]), + fake_gleam_guest(), + ) + assert_eq(third.report.exit_code(), 0, msg=third.json.stringify()) + assert_eq(fake_uris(third.calls[0]), ["src/ledger.gleam"]) + assert_eq( + fake_field(third.calls[0], ["contextDigest"]), + fake_field(first.calls[0], ["contextDigest"]), + ) + assert_true(fake_field(first.calls[0], ["contextDigest"]) is String(_)) +} + +///| +async test "a tree of 1,025 documents is refused before any operation session" { + let workspace = fake_workspace() + let many = workspace.root + "/many" + @platform.mkdir(many) + for index in 0..<1025 { + @platform.write(many + "/m\{index}.gleam", "pub type T = Int\n") + } + let run = fake_run( + fake_tree_compile(workspace, many, "model", [ + "--source-base", + workspace.root, + ]), + fake_gleam_guest(), + ) + assert_eq(run.report.exit_code(), 2, msg=run.json.stringify()) + assert_eq(fake_error(run), "extension.input_invalid") + assert_true( + fake_field(run.json, ["error", "message"]) + .stringify() + .contains("limit of 1024 documents"), + msg=run.json.stringify(), + ) + assert_eq(run.methods, ["morphir.extension.describe", "morphir.exit"]) +} diff --git a/apps/morphir/cli/extension_unhosted.mbt b/apps/morphir/cli/extension_unhosted.mbt index c08c39e78..3ed33827e 100644 --- a/apps/morphir/cli/extension_unhosted.mbt +++ b/apps/morphir/cli/extension_unhosted.mbt @@ -14,8 +14,9 @@ pub fn WorkerHosting::none() -> WorkerHosting { /// `extension.runtime_unavailable` and exit 2, in the report of the hosted /// targets. fn extension_command(cli : Cli, context : CommandContext) -> Int32 { - // The WASI entry passes no launchers. + // The WASI entry passes no launchers and starts no worker to stop. ignore(context.workers) + ignore(context.stop_requested) let message = "this CLI build has no worker host; use the native or npm CLI" let report : Json = { "profile": "morphir/command-result", diff --git a/apps/morphir/cli/extension_wbtest.mbt b/apps/morphir/cli/extension_wbtest.mbt index 9bf42e015..38eb99f15 100644 --- a/apps/morphir/cli/extension_wbtest.mbt +++ b/apps/morphir/cli/extension_wbtest.mbt @@ -218,13 +218,30 @@ fn extension_test_tuple(operation : String, codec : String) -> Json { } } +///| +/// A source base for the tests, with POSIX paths. Without `base` the current +/// directory is the input itself, so a directory gives URIs relative to it +/// and a single file follows the rule without `--source-base`. +fn extension_test_base(path : String, base? : String) -> ExtensionSourceBase { + { given: base, cwd: path, style: Posix, } +} + ///| /// Reads a source snapshot and returns its URIs, or the error code. fn extension_test_sources( path : String, + base? : String, limits~ : ExtensionSourceLimits = extension_source_limits, ) -> Result[Array[String], String] { - try extension_sources(path, [".gleam"], () => (), limits~) catch { + try + extension_sources( + path, + [".gleam"], + () => (), + base=extension_test_base(path, base?), + limits~, + ) + catch { ExtensionError(code~, message~, ..) => Err(code.name() + ": " + message) } noraise { documents => Ok(documents.map(document => document.uri)) @@ -236,10 +253,28 @@ fn extension_test_input_invalid(result : Result[Array[String], String]) -> Bool result is Err(message) && message.has_prefix("extension.input_invalid: ") } +///| +/// The refusal of `action`: its code, message and exit status. +fn extension_test_error( + action : () -> Unit raise ExtensionError, +) -> (String, String, Int32) { + try action() catch { + ExtensionError(status~, code~, message~) => + (code.name(), message, status.exit_code()) + } noraise { + _ => ("accepted", "", 0) + } +} + ///| test "compile selects sources by the claimed file extensions and sends the claimed language" { let claims = @worker.Claims::from_describe(extension_test_claims()) - assert_eq(extension_claimed_extensions(claims), [".gleam"]) + let language = extension_language(claims, None) + assert_eq(language.language_id.to_string(), "gleam") + let extensions = language.file_extensions.map(extension => { + extension.to_string() + }) + assert_eq(extensions, [".gleam"]) let root = @platform.temporary() @platform.mkdir(root + "/src") @platform.mkdir(root + "/src/nested") @@ -249,15 +284,14 @@ test "compile selects sources by the claimed file extensions and sends the claim @platform.write(root + "/src/README.md", "not a source") let documents = extension_sources( root + "/src", - extension_claimed_extensions(claims), + extensions, () => (), + base=extension_test_base(root + "/src"), ) assert_eq(documents.map(document => document.uri), [ "app.gleam", "lib.gleam", "nested/deep.gleam", ]) - let (language, selected) = extension_language(claims, documents) - assert_eq(language.to_string(), "gleam") - assert_eq(selected.map(document => document.text), [ + assert_eq(documents.map(document => document.text), [ "pub fn main() { 1 }", "pub fn lib() { 2 }", "pub fn deep() { 3 }", ]) // A single file keeps its clean relative path; a file without a claimed @@ -268,13 +302,235 @@ test "compile selects sources by the claimed file extensions and sends the claim extension_test_sources(root + "/src/README.md"), ), ) - let refusal = try { - ignore(extension_language(claims, [])) - "" - } catch { - ExtensionError(code~, ..) => code.name() + // A directory with no claimed file gives no document and is refused. + @platform.mkdir(root + "/docs") + @platform.write(root + "/docs/guide.md", "only text") + match extension_test_sources(root + "/docs") { + Err(message) => + assert_true( + message.has_prefix("extension.input_invalid: ") && + message.contains(".gleam"), + msg=message, + ) + Ok(uris) => fail("accepted \{uris.join(", ")}") + } +} + +///| +test "language from claims: the only language, the one that --language names, or a refusal" { + let claims = (languages : Array[Json]) => { + @worker.Claims::from_describe( + extension_test_claims(frontend={ + "languages": Json::array(languages), + "irVersions": ["4"], + "compile": true, + }), + ) + } + let gleam : Json = { "id": "gleam", "fileExtensions": [".gleam"] } + let elm : Json = { "id": "elm", "fileExtensions": [".elm"] } + let language = (text : String) => Some(@ids.LanguageId::parse(text)) + let several = claims([elm, gleam]) + let python = language("python") + // One claimed language: the CLI uses it. + assert_eq( + extension_language(claims([gleam]), None).language_id.to_string(), + "gleam", + ) + // Several claimed languages and no choice: input_invalid, exit 2, and the + // message names every claimed language. + let (code, message, status) = extension_test_error(() => { + ignore(extension_language(several, None)) + }) + assert_eq((code, status), ("extension.input_invalid", 2)) + assert_true( + message.contains("elm") && + message.contains("gleam") && + message.contains("--language"), + msg=message, + ) + // --language selects one of them, with only its file extensions. + let selected = extension_language(several, language("gleam")) + assert_eq(selected.language_id.to_string(), "gleam") + assert_eq(selected.file_extensions.map(extension => extension.to_string()), [ + ".gleam", + ]) + // A language that the extension does not claim is a capability refusal. + let (code, message, status) = extension_test_error(() => { + ignore(extension_language(several, python)) + }) + assert_eq((code, status), ("extension.capability_unavailable", 2)) + assert_true(message.contains("python"), msg=message) +} + +///| +test "the source base gives each URI its path relative to the base" { + let base = (given : String?, cwd : String) => ExtensionSourceBase::{ + given, + cwd, + style: Posix, + } + // Run from the project directory: the base is the current directory. + assert_eq(base(None, "/work/shop").segments("src"), ["src"]) + assert_eq(base(None, "/work/shop").segments("src/app/"), ["src", "app"]) + assert_eq(base(None, "/work/shop").segments("."), []) + // An explicit base gives the same segments for an absolute input. + assert_eq(base(Some("/work/shop"), "/tmp").segments("/work/shop/src"), ["src"]) + // A relative base is relative to the current directory. + assert_eq(base(Some("shop"), "/work").segments("shop/src/../src"), ["src"]) + // An input outside the base is refused, and the message names the base. + for + (refused, path) in [ + (base(Some("/work/shop"), "/tmp"), "/other/src"), + (base(None, "/work/shop/src"), ".."), + (base(Some("/work/shop"), "/tmp"), "/work/shopping/src"), + ] { + let (code, message, status) = extension_test_error(() => { + ignore(refused.segments(path)) + }) + assert_eq((code, status), ("extension.input_invalid", 2), msg=path) + assert_true( + message.contains(refused.given.unwrap_or(refused.cwd)), + msg=message, + ) + } + // Windows paths compare drive letters without case. + let windows = ExtensionSourceBase::{ + given: Some("C:\\work\\shop"), + cwd: "C:\\", + style: Windows, + } + assert_eq(windows.segments("c:\\work\\shop\\src\\app"), ["src", "app"]) + assert_eq( + extension_test_error(() => ignore(windows.segments("D:\\work\\shop"))).0, + "extension.input_invalid", + ) +} + +///| +test "a tree compiled from the project directory or with --source-base has URIs from the base" { + let root = @platform.temporary() + @platform.mkdir(root + "/src") + @platform.mkdir(root + "/src/app") + @platform.mkdir(root + "/src/shared") + @platform.write(root + "/src/app/money.gleam", "import shared/numbers\n") + @platform.write(root + "/src/app/orders.gleam", "import app/money\n") + @platform.write(root + "/src/shared/numbers.gleam", "pub type Number = Int\n") + let expected = [ + "src/app/money.gleam", "src/app/orders.gleam", "src/shared/numbers.gleam", + ] + // The current directory is the project directory. + let from_project = extension_sources(root + "/src", [".gleam"], () => (), base={ + given: None, + cwd: root, + style: Posix, + }) catch { + ExtensionError(message~, ..) => fail("refused: \{message}") + } + assert_eq(from_project.map(document => document.uri), expected) + // An explicit source base gives the same URIs. + assert_eq(extension_test_sources(root + "/src", base=root), Ok(expected)) + // A deeper input keeps the URIs from the base, so module names do not + // depend on the directory that the user names. + assert_eq( + extension_test_sources(root + "/src/app", base=root), + Ok(expected[:2].to_owned()), + ) + // A single file with --source-base takes the directory rule; without it, + // the earlier rule gives the name of an absolute path. + assert_eq( + extension_test_sources(root + "/src/app/money.gleam", base=root), + Ok(["src/app/money.gleam"]), + ) + assert_eq( + extension_test_sources(root + "/src/app/money.gleam"), + Ok(["money.gleam"]), + ) + // An input outside the source base is refused and names the base. + match extension_test_sources(root + "/src", base=root + "/other") { + Err(message) => + assert_true( + message.has_prefix("extension.input_invalid: ") && + message.contains(root + "/other"), + msg=message, + ) + Ok(uris) => fail("accepted \{uris.join(", ")}") } - assert_eq(refusal, "extension.input_invalid") +} + +///| +test "URI segments are percent-encoded and plain names stay the same" { + let plain = "AZaz09-._~" + assert_eq(extension_uri_segment(plain), plain) + assert_eq(extension_uri_segment("money.gleam"), "money.gleam") + for + (name, encoded) in [ + ("my module.gleam", "my%20module.gleam"), + ("a:b", "a%3Ab"), + ("100%", "100%25"), + ("why?", "why%3F"), + ("#1", "%231"), + ("a\\b", "a%5Cb"), + ("tab\t", "tab%09"), + ("caf\u{e9}", "caf%C3%A9"), + ("\u{65e5}", "%E6%97%A5"), + ("a+b@c", "a%2Bb%40c"), + ] { + assert_eq(extension_uri_segment(name), encoded, msg=name) + } + assert_eq( + extension_document_uri("src/my module.gleam"), + "src/my%20module.gleam", + ) + // Names that a URI cannot carry as a segment are refused; any other name + // is encoded. + assert_eq(extension_source_name("lib.gleam"), None) + assert_eq(extension_source_name(".hidden.gleam"), None) + assert_eq(extension_source_name("a:b.gleam"), None) + assert_eq(extension_source_name("a\\b.gleam"), None) + for name in ["", ".", "..", "a/b.gleam", "/abs.gleam", "nul\u0000.gleam"] { + assert_true(extension_source_name(name) is Some(_), msg=name) + } + let root = @platform.temporary() + @platform.mkdir(root + "/src") + @platform.write(root + "/src/my module.gleam", "pub fn main() { 1 }") + @platform.write(root + "/src/a:b.gleam", "pub fn main() { 2 }") + assert_eq( + extension_test_sources(root + "/src", base=root), + Ok(["src/a%3Ab.gleam", "src/my%20module.gleam"]), + ) +} + +///| +test "documents are sorted by the whole URI in byte order, and a repeated URI is refused" { + let root = @platform.temporary() + @platform.mkdir(root + "/a") + @platform.write(root + "/a/b.gleam", "1") + @platform.write(root + "/a-c.gleam", "2") + @platform.write(root + "/B.gleam", "3") + @platform.write(root + "/caf\u{e9}.gleam", "4") + // `-` (0x2D) comes before `/` (0x2F), so `a-c.gleam` is before + // `a/b.gleam`, though the directory `a` sorts before the file `a-c.gleam`. + assert_eq( + extension_test_sources(root), + Ok(["B.gleam", "a-c.gleam", "a/b.gleam", "caf%C3%A9.gleam"]), + ) + let file = (uri : String) => ExtensionSourceFile::{ + uri, + relative: uri, + path: "/tree/" + uri, + } + assert_eq( + extension_source_order([file("b"), file("a-c"), file("a/b")]).map(file => { + file.uri + }), + ["a-c", "a/b", "b"], + ) + let (code, message, status) = extension_test_error(() => { + ignore(extension_source_order([file("x.gleam"), file("x.gleam")])) + }) + assert_eq((code, status), ("extension.input_invalid", 2)) + assert_true(message.contains("x.gleam"), msg=message) } ///| @@ -293,23 +549,6 @@ test "the source walk skips hidden, build and dependency directories" { assert_eq(extension_test_sources(root), Ok(["app.gleam", "build/kept.gleam"])) } -///| -test "directory names that would make a URI unsafe are refused" { - assert_eq(extension_source_name("lib.gleam"), None) - assert_eq(extension_source_name(".hidden.gleam"), None) - for - name in [ - "", ".", "..", "a\\b.gleam", "c:x.gleam", "a:b", "a/b.gleam", "/abs.gleam", - "nul\u0000.gleam", - ] { - assert_true(extension_source_name(name) is Some(_), msg=name) - } - // A walked name with a colon is refused with input_invalid. - let root = @platform.temporary() - @platform.write(root + "/a:b.gleam", "pub fn main() { 1 }") - assert_true(extension_test_input_invalid(extension_test_sources(root))) -} - ///| test "the source snapshot enforces its file, byte, entry and depth limits" { let root = @platform.temporary() @@ -334,14 +573,18 @@ test "the source snapshot enforces its file, byte, entry and depth limits" { Ok(uris) => fail("accepted \{uris.join(", ")}") } } - refused({ ..limits, files: 2, }, "more than 2 files") - refused({ ..limits, bytes: 8, }, "more than 8 bytes") - refused({ ..limits, entries: 4, }, "more than 4 entries") - refused({ ..limits, depth: 1, }, "deeper than 1 levels") + // Each message names the limit and its value. + refused({ ..limits, files: 2, }, "limit of 2 documents") + refused({ ..limits, bytes: 8, }, "limit of 8 bytes") + refused({ ..limits, file_bytes: 3, }, "one.gleam") + refused({ ..limits, file_bytes: 3, }, "limit of 3 bytes for one file") + refused({ ..limits, entries: 4, }, "limit of 4 entries") + refused({ ..limits, depth: 1, }, "limit of 1 directories") // Each limit is inclusive. assert_true( extension_test_sources(root, limits={ files: 3, + file_bytes: 4, bytes: 9, entries: 5, depth: 2, @@ -350,6 +593,54 @@ test "the source snapshot enforces its file, byte, entry and depth limits" { ) } +///| +test "the default limits refuse 1,025 documents and a file larger than 1 MiB" { + assert_eq(extension_source_limits.files, 1024) + assert_eq(extension_source_limits.file_bytes, 1048576) + assert_eq(extension_source_limits.bytes, 2097152) + assert_eq(extension_source_limits.entries, 16384) + assert_eq(extension_source_limits.depth, 32) + let many = @platform.temporary() + for index in 0..<1025 { + @platform.write(many + "/m\{index}.gleam", "") + } + match extension_test_sources(many) { + Err(message) => + assert_true(message.contains("limit of 1024 documents"), msg=message) + Ok(uris) => fail("accepted \{uris.length()} documents") + } + let large = @platform.temporary() + @platform.write(large + "/large.gleam", "x".repeat(1048577)) + match extension_test_sources(large) { + Err(message) => + assert_true( + message.contains("large.gleam") && + message.contains("limit of 1048576 bytes for one file"), + msg=message, + ) + Ok(_) => fail("accepted a file larger than 1 MiB") + } + @platform.write(large + "/large.gleam", "x".repeat(1048576)) + assert_eq(extension_test_sources(large), Ok(["large.gleam"])) +} + +///| +test "text that is not UTF-8 is refused with the URI of the file" { + let root = @platform.temporary() + @platform.mkdir(root + "/src") + @platform.write(root + "/src/good.gleam", "pub fn main() { 1 }") + @platform.write_bytes(root + "/src/my latin.gleam", b"caf\xe9") + match extension_test_sources(root + "/src", base=root) { + Err(message) => + assert_true( + message.has_prefix("extension.input_invalid: ") && + message.contains("src/my%20latin.gleam"), + msg=message, + ) + Ok(uris) => fail("accepted \{uris.join(", ")}") + } +} + ///| #cfg(target="native") async fn extension_test_symlink(target : String, path : String) -> Unit { @@ -371,12 +662,32 @@ fn extension_test_symlink(target : String, path : String) -> Unit { } ///| -async test "a symbolic link in the sources is refused" { +async test "a symbolic link in the sources is refused with its relative path" { let root = @platform.temporary() - @platform.write(root + "/app.gleam", "pub fn main() { 1 }") - extension_test_symlink(root + "/app.gleam", root + "/link.gleam") - assert_true(extension_test_input_invalid(extension_test_sources(root))) + @platform.mkdir(root + "/src") + @platform.mkdir(root + "/other") + @platform.write(root + "/src/app.gleam", "pub fn main() { 1 }") + @platform.write(root + "/other/lib.gleam", "pub fn lib() { 2 }") + extension_test_symlink(root + "/src/app.gleam", root + "/src/link.gleam") + match extension_test_sources(root + "/src", base=root) { + Err(message) => + assert_true( + message.has_prefix("extension.input_invalid: ") && + message.contains("src/link.gleam") && + !message.contains(root), + msg=message, + ) + Ok(uris) => fail("accepted \{uris.join(", ")}") + } + // A link to a directory is refused and not followed. + @platform.remove_file(root + "/src/link.gleam") + extension_test_symlink(root + "/other", root + "/src/vendor") + match extension_test_sources(root + "/src", base=root) { + Err(message) => assert_true(message.contains("src/vendor"), msg=message) + Ok(uris) => fail("accepted \{uris.join(", ")}") + } // A link given as the source itself is refused too. + extension_test_symlink(root + "/src/app.gleam", root + "/link.gleam") assert_true( extension_test_input_invalid(extension_test_sources(root + "/link.gleam")), ) @@ -595,11 +906,19 @@ test "extension command syntax keeps its own options" { assert_eq(value(cli, "path"), "src/main.mbt") assert_eq(value(cli, "ir-version"), "4") assert_true(flag(cli, "json")) + let cli = parse_cli([ + "extension", "compile", "src", "--extension-pin", "pin.json", "--package", "my/pkg", + "--output", "out", "--language", "gleam", "--source-base", "/work/shop", + ]) + assert_eq(value(cli, "language"), "gleam") + assert_eq(value(cli, "source-base"), "/work/shop") for args in [ ["extension", "describe", "--package", "x"], ["extension"], ["extension", "describe", "--services-pin", "pin.json"], + ["extension", "describe", "--language", "gleam"], + ["extension", "generate", "model.json", "--source-base", "."], ] { assert_true( try { diff --git a/apps/morphir/cli/main.mbt b/apps/morphir/cli/main.mbt index 2b86e3c12..6306d3b9f 100644 --- a/apps/morphir/cli/main.mbt +++ b/apps/morphir/cli/main.mbt @@ -48,7 +48,7 @@ fn usage(mode : OutputMode, help_text : String) -> Unit { "--mck-helper", "--extension-pin", "--extension-helper", "--input", "--version", "--types-only", "--services-pin", "--services-helper", "--repository", "--executable", "--sha256", "--extension-id", "--short-id", - "--platform", "--ir-version", + "--platform", "--ir-version", "--source-base", ].map(Json::string), ), ), diff --git a/apps/morphir/cli/mep_recording_wbtest.mbt b/apps/morphir/cli/mep_recording_wbtest.mbt index 3e80499c9..b0f1e418f 100644 --- a/apps/morphir/cli/mep_recording_wbtest.mbt +++ b/apps/morphir/cli/mep_recording_wbtest.mbt @@ -113,12 +113,14 @@ async test "record native MEP host and guest sessions" { } let claims = description.claims let offer = extension_compile_offer(claims, inputs.codec) - let (language, documents) = extension_language( - claims, - extension_sources(inputs.path, extension_claimed_extensions(claims), () => { - () - }), + let language = extension_language(claims, inputs.language) + let documents = extension_sources( + inputs.path, + language.file_extensions.map(extension => extension.to_string()), + () => (), + base={ given: None, cwd: @platform.cwd(), style: Posix, }, ) + let language = language.language_id let request = @worker.CompileRequest::new( extension_compile_params(inputs, claims, offer, language, documents, None), ) diff --git a/apps/morphir/docs/extension.md b/apps/morphir/docs/extension.md index b69a48248..6d4eef617 100644 --- a/apps/morphir/docs/extension.md +++ b/apps/morphir/docs/extension.md @@ -133,20 +133,24 @@ Consider `extension compile main.mbt --package acme/rules --output model`: `morphir.frontend.compile` compiles on MEP `0.1`. An operation that the extension does not claim is refused with `extension.capability_unavailable` and exit status 2. -4. **Sources.** The CLI reads a bounded snapshot of the source file or - directory. It reads only the files with the file extensions that the - extension claims, in name order, and sends the claimed language ID. A - single file keeps its clean relative path as its URI; an absolute path or a - path with `..` keeps only its file name. Files in a directory keep their - path relative to that directory. A frontend that does not claim - `multiDocument` compiles one document for each request, so a snapshot with - more than one source file is refused with `extension.capability_unavailable` - and exit status 2 before the session. The walk skips hidden directories (names +4. **Sources.** The CLI selects the language from the claims. When the + extension claims one language, the CLI uses it. When it claims several, + `--language` must name one of them. Without `--language`, the command is + refused with `extension.input_invalid`, and the message names the claimed + languages. A `--language` that the extension does not claim is refused with + `extension.capability_unavailable`. Then the CLI reads a bounded snapshot of + the source file or directory. It reads only the files with the file + extensions of the selected language, and it sends that language ID at the + top of the request and in each document. Each document URI is the file's + path relative to the source base, with percent-encoded segments, and the + documents are in URI byte order (see + [Source trees and document URIs](#source-trees-and-document-uris)). A + frontend that does not claim `multiDocument` compiles one document for each + request, so a snapshot with more than one source file is refused with + `extension.capability_unavailable`. The walk skips hidden directories (names that start with `.`) and the build and dependency directories `.git`, - `.mooncakes`, `_build`, `node_modules` and `.venv`. It refuses, with - `extension.input_invalid`, a symbolic link, a special file, and an entry - name that contains `\`, `/`, `:` or NUL or is `.` or `..`, because such a - name would make a document URI absolute or traversing. + `.mooncakes`, `_build`, `node_modules` and `.venv`. Every refusal in this + step exits with status 2 and starts no operation session. 5. **Session.** It starts a fresh worker from the same verified bytes and sends `morphir.initialize`. It requires the returned identity and capabilities to equal the probed claims, then sends `morphir.initialized` and the operation. @@ -161,12 +165,110 @@ Consider `extension compile main.mbt --package acme/rules --output model`: (with `envelopeVersion`) is sent on MEP `0.2.0-draft.1` with the envelope's tuple; plain Morphir IR JSON is sent to `morphir.backend.generate` on MEP `0.1`. +### Source trees and document URIs + +A Gleam project keeps its modules in a tree under `src`, and the modules import +one another by their path below `src`: + +```text +/work/shop/ + gleam.toml + src/ + app/money.gleam import shared/numbers + app/orders.gleam import app/money + shared/numbers.gleam +``` + +Run the compile from the project directory and name `src`: + +```sh +cd /work/shop +morphir extension compile src --package acme/shop --output model --extension-pin pin.json +``` + +The request holds the whole tree as three documents. Each URI is the file's path +relative to the **source base**, which is the current directory by default: + +```json +{ + "languageId": "gleam", + "sources": { + "documents": [ + { "uri": "src/app/money.gleam", "languageId": "gleam", "version": 0, "text": "..." }, + { "uri": "src/app/orders.gleam", "languageId": "gleam", "version": 0, "text": "..." }, + { "uri": "src/shared/numbers.gleam", "languageId": "gleam", "version": 0, "text": "..." } + ] + } +} +``` + +The Gleam guest finds the `src` segment and names the modules `app/money`, +`app/orders` and `shared/numbers`. The module names do not depend on the +directory that you name: `compile src/app` from `/work/shop` sends +`src/app/money.gleam` and `src/app/orders.gleam`. Thus, name `src` (or the +project directory) from the project directory. The request has no +`sources.root` and no absolute host path, so the model's source provenance is +the same on every machine. + +From another directory, give the source base with `--source-base`. This command +sends the same URIs: + +```sh +morphir extension compile /work/shop/src --source-base /work/shop --package acme/shop --output model --extension-pin pin.json +``` + +The input must be the source base or a path below it. The CLI compares the two +paths after it makes them absolute and removes `.` and `..`; it does not resolve +symbolic links. An input outside the source base, such as `/other/src` with +`--source-base /work/shop`, is refused with `extension.input_invalid`, and the +message names the source base. A single file without `--source-base` keeps the +earlier rule, so a MoonBit request does not change: a clean relative path such +as `src/main.mbt` keeps its nesting, and an absolute path or a path with `..` +keeps only its file name. A single file with `--source-base` takes its path +relative to the source base. + +Each URI segment is encoded as an RFC 3986 path segment. Letters, digits, `-`, +`.`, `_` and `~` stay as they are, so a plain path does not change. Each other +byte of the UTF-8 name is written as `%` and two uppercase hexadecimal digits: +`src/my module.gleam` becomes `src/my%20module.gleam`, and `café.gleam` becomes +`caf%C3%A9.gleam`. Thus a URI never holds a raw `%`, `?`, `#`, `\`, `:`, space, +control character or non-ASCII character. An entry name that is empty, `.` or +`..` is refused, and two files that give the same URI are refused, both with +`extension.input_invalid`. + +The documents are sorted by the whole URI in byte order, so `a-c.gleam` comes +before `a/b.gleam` (`-` is `0x2D` and `/` is `0x2F`). Two compiles of the same +tree with the same options send byte-identical `compileModel` requests. The +`contextDigest` is the SHA-256 of the package name and the options only, so +other trees with the same package and options have the same `contextDigest`. + +The CLI checks these limits before it starts the operation session. Each refusal +has the code `extension.input_invalid`, exit status 2, and a message that names +the limit and its value, a relative path or a URI: + +| Check | Limit | Message | +| --- | --- | --- | +| Documents | 1,024 | `the sources have 1025 documents, more than the limit of 1024 documents` | +| One file | 1 MiB | `the source file src/big.gleam is larger than the limit of 1048576 bytes for one file` | +| All files | 2 MiB | `the sources are larger than the limit of 2097152 bytes of source text in total` | +| Directory entries | 16,384 | `the source directory has more entries than the limit of 16384 entries` | +| Directory depth | 32 | `the source directory src/a/... is deeper than the limit of 32 directories` | +| Links and special files | none allowed | `src/vendor is a link or a special file` (the path relative to the source base) | +| UTF-8 text | every file | `the source file src/my%20latin.gleam is not UTF-8 text` (the URI) | +| No document | at least one | `no source file in docs has a claimed file extension (.gleam)` | + +The walk does not follow symbolic links. After these checks, the framed request +must still fit in 4 MiB, because JSON escaping can make a request larger than its +sources; a larger request is refused with `extension.request_limit`. + ## Compile and generate options | Option | Commands | Meaning | | --- | --- | --- | | `--extension-pin FILE` | all | The pin; or `MORPHIR_EXTENSION_PIN` | | `` | compile | A source file, or a directory snapshot of claimed source files | +| `--language ID` | compile | One of the claimed languages; required when the extension claims more than one | +| `--source-base DIR` | compile | The directory that document URIs are relative to; the current directory by default. The source must be this directory or below it | | `` | generate | A model envelope, or plain Morphir IR JSON | | `--package NAME` | compile | The package name; required | | `--model-format ion\|json` | compile | The one model tuple that the CLI offers on the model line. Without it, the CLI offers Ion, then JSON, and uses the first tuple that the extension agrees on. MEP `0.1` compiles return Morphir JSON and refuse `ion` | @@ -267,7 +369,7 @@ process tree. A failure leaves no output directory and no | Frame header | 8 KiB | | Offered model limits | 2,796,204 encoded bytes, 2 MiB decoded, depth 128, 100,000 nodes; the worker's smaller values apply | | Generate input | 4 MiB, depth 128, 100,000 nodes | -| Compile sources | 1,024 files, 2 MiB in total, 16,384 directory entries, depth 32 | +| Compile sources | 1,024 files, 1 MiB for one file, 2 MiB in total, 16,384 directory entries, depth 32 | | Guest stderr kept | The last 64 KiB | | Deadline | 60,000 ms by default, at most 594,000 ms, for the probe and the session together | | Grace after `exit` | 2 seconds, then the host stops the tree | diff --git a/openspec/changes/run-core-wasm-guests-from-any-language/tasks.md b/openspec/changes/run-core-wasm-guests-from-any-language/tasks.md index 8eac0d0d3..84b3d5c61 100644 --- a/openspec/changes/run-core-wasm-guests-from-any-language/tasks.md +++ b/openspec/changes/run-core-wasm-guests-from-any-language/tasks.md @@ -20,12 +20,15 @@ the remaining descriptor checks. Group 4 runs the real Gleam module through all model line publishes the model and reports the session` (`extension_session_wbtest.mbt:362`, method order) and `probe of a guest that does not describe: describe_failed and no claims` (`host_driver_test.mbt:127`). The spec uses main's code `extension.describe_failed` (FIT-1) -- [ ] 1.3 Select the language from the claims (SEL-3): add `--language` through the command-line parsing library, +- [x] 1.3 Select the language from the claims (SEL-3): add `--language` through the command-line parsing library, refuse several claimed languages without it with `extension.input_invalid` (exit 2, the message names the languages; no new code, FIT-1), refuse an unclaimed language with `extension.capability_unavailable`, and collect only the file extensions of the selected language. Replace the first-covering-language rule of `extension_language` (`apps/morphir/cli/extension.mbt:428`) and the all-languages list of `extension_claimed_extensions` (`:412`); - verify the "Language from claims" scenarios in `extension_wbtest.mbt` and `extension_session_wbtest.mbt` + verify the "Language from claims" scenarios in `extension_wbtest.mbt` and `extension_session_wbtest.mbt`. + Done: `extension_language` and `ExtensionInputs::read` in `apps/morphir/cli/extension.mbt`; tests `language from + claims: the only language, the one that --language names, or a refusal` (`extension_wbtest.mbt`) and `language + from claims: a guest with several languages needs --language` (`extension_session_wbtest.mbt`) - [ ] 1.4 Add tests for the target selection that `extension_target` (`apps/morphir/cli/extension.mbt:789`) already does (SEL-3): several claimed targets without `--target` are refused, and an unclaimed `--target` exits 2 with `extension.capability_unavailable` and starts no operation session; verify the "Unclaimed target" scenario @@ -99,7 +102,7 @@ the remaining descriptor checks. Group 4 runs the real Gleam module through all ## 2. Source trees as relative-URI documents (morphir-ukvn.5.64.37.2) -- [ ] 2.1 Give `extension_sources` (`apps/morphir/cli/extension.mbt:509`) the refusals and the order of the spec +- [x] 2.1 Give `extension_sources` (`apps/morphir/cli/extension.mbt:509`) the refusals and the order of the spec (SRC-2), all with main's code `extension.input_invalid` and exit 2 (FIT-1, decided by Damian Reeves, 2026-10-31; no new source codes): a new 1 MiB bound for one file, a message that names each limit and its value, the relative path in the message for a link or a special file, a duplicate URI refused, the URI in the message for @@ -107,23 +110,38 @@ the remaining descriptor checks. Group 4 runs the real Gleam module through all keep main's skip list (FIT-4) and the 4 MiB framed request check of `Call::make` (`pkgs/morphir-host/worker/negotiation.mbt:320-337`) last; verify the updated tests in `extension_wbtest.mbt` ("Tree limits", "Links and special files", "UTF-8 source text", "Skipped directories" and "Deterministic order" - scenarios) with generated trees in a temporary directory, including a first test for text that is not UTF-8 -- [ ] 2.2 Make each URI relative to the source base, add `--source-base` through the command-line parsing library, + scenarios) with generated trees in a temporary directory, including a first test for text that is not UTF-8. + Done: `extension_sources` and `extension_source_order`; tests `the source snapshot enforces its file, byte, entry + and depth limits`, `the default limits refuse 1,025 documents and a file larger than 1 MiB`, `text that is not + UTF-8 is refused with the URI of the file`, `a symbolic link in the sources is refused with its relative path`, + `documents are sorted by the whole URI in byte order, and a repeated URI is refused` and `the source walk skips + hidden, build and dependency directories` (`extension_wbtest.mbt`), and `a tree of 1,025 documents is refused + before any operation session` (`extension_session_wbtest.mbt`) +- [x] 2.2 Make each URI relative to the source base, add `--source-base` through the command-line parsing library, and encode each segment as an RFC 3986 path segment instead of refusing names with `:` or `\` (SRC-1, SRC-3); keep the single-file rule of `extension_document_uri` (`apps/morphir/cli/extension.mbt:393`) without - `--source-base`; verify the "Relative document URIs", "Safe URI segments" and "Single file input" scenarios + `--source-base`; verify the "Relative document URIs", "Safe URI segments" and "Single file input" scenarios. + Done: `ExtensionSourceBase::segments`, `extension_uri_segment` and `extension_document_uri`; tests `the source base + gives each URI its path relative to the base`, `a tree compiled from the project directory or with --source-base + has URIs from the base`, `URI segments are percent-encoded and plain names stay the same` and `document + identities keep relative nesting and never expose absolute host paths` (`extension_wbtest.mbt`), and `relative + document URIs: a tree is named from the source base, with no sources.root` (`extension_session_wbtest.mbt`) - [x] 2.3 Accept a directory for `compile` and refuse more than one document without the multi-document claim. Done on main: `extension_sources` (`apps/morphir/cli/extension.mbt:509-605`) and `extension_document_count` (`:456`); tests `compile selects sources by the claimed file extensions and sends the claimed language` (`extension_wbtest.mbt:240`) and `Python 0.4.1 with two modules: a frontend without multiDocument is refused` (`extension_session_wbtest.mbt:588`) -- [ ] 2.4 Add tests that two compiles of the same tree give byte-identical `compileModel` requests, and that the same +- [x] 2.4 Add tests that two compiles of the same tree give byte-identical `compileModel` requests, and that the same package and options with different trees give the same `contextDigest` (the digest code is `extension_compile_params`, `apps/morphir/cli/extension.mbt:853-863`); verify the "Same tree twice" and "Same - options, different trees" scenarios -- [ ] 2.5 Document the source base, the URI encoding, the limits and their messages, the ignore files of task - 2.6, and the advice to name `src` from the project directory in `apps/morphir/docs/extension.md` ("What one - command does", step 4); verify the examples with the in-memory workers + options, different trees" scenarios. Done: test `same tree twice gives byte-identical compileModel requests; + other trees keep the contextDigest` (`extension_session_wbtest.mbt`), which compares the encoded request bytes +- [x] 2.5 Document the source base, the URI encoding, the limits and their messages, and the advice to name `src` + from the project directory in `apps/morphir/docs/extension.md` ("What one command does", step 4); verify the + examples with the in-memory workers. The documentation of the ignore files moved to task 2.6. Done: step 4 and + the section "Source trees and document URIs"; the examples are the trees of `relative document URIs: a tree is + named from the source base, with no sources.root` and `a tree compiled from the project directory or with + --source-base has URIs from the base` - [ ] 2.6 Honor ignore files in the source walk (FIT-4 and IGN-1 to IGN-5, decided by Damian Reeves, 2026-10-31). Depends on moonrockz/moonx-tools#16: use its ignore-file library (in place of the earlier candidate `moonbit-community/ignore`) and do not start the walk change until a release of it builds for the four targets. @@ -136,7 +154,8 @@ the remaining descriptor checks. Group 4 runs the real Gleam module through all special-file check. Filter only the entries below the input: read a named file, and walk a named directory, even when a pattern matches it. Add `--ignore-mode ` (contained reads only the ignore files in the walked tree, in or outside a work tree) and the repeatable `--skip-ignore-layer `, - with `extension.input_invalid`, exit 2, for an unknown value, and document both in the command help. Verify the + with `extension.input_invalid`, exit 2, for an unknown value, and document both in the command help and the + ignore files in `apps/morphir/docs/extension.md` ("Source trees and document URIs"). Verify the scenarios of "Ignore files in the source walk", "Precedence of ignore files", "Git ignore layers", "Ignore layer switches", "Contained ignore mode", "Ignore files filter only the walk", the ignore part of "Directory input" and "Two machines in contained mode" with generated trees in `extension_wbtest.mbt` (a temporary work tree, a From 12163ba0229223bbda27f5d721787e42b253113a Mon Sep 17 00:00:00 2001 From: Damian Reeves <957246+DamianReeves@users.noreply.github.com> Date: Sat, 31 Oct 2026 21:35:19 -0500 Subject: [PATCH 2/2] fix(cli): make a directory input its own default source base Without --source-base, the document URIs of a directory compile are relative to that directory again, as before this change. The current directory default broke the published Elm 0.5.0-preview.2 and Python 0.4.2 extensions in the extension qualification suite: compile generated/src sent generated/src/Acme/Rules/Main.elm, and compile src named the Python module src.acme.rules. The Gleam 0.4.0 guest gives the same module names for compile src and compile . with this default. --source-base still overrides the base, and a single file keeps its earlier rule. Damian Reeves chose this option (B) on 2026-11-01. The spec, design (SRC-1), tasks, proposal, CLI help and documentation now describe it. --- apps/morphir/cli/arguments.mbt | 2 +- apps/morphir/cli/extension.mbt | 25 ++++---- apps/morphir/cli/extension_session_wbtest.mbt | 13 +++- apps/morphir/cli/extension_wbtest.mbt | 54 +++++++++------- apps/morphir/docs/extension.md | 61 +++++++++++-------- .../design.html | 59 ++++++++++++------ .../proposal.md | 8 ++- .../specs/extensions/source-documents/spec.md | 23 +++++-- .../tasks.md | 25 ++++---- 9 files changed, 171 insertions(+), 99 deletions(-) diff --git a/apps/morphir/cli/arguments.mbt b/apps/morphir/cli/arguments.mbt index 129b60a9f..56680e0fb 100644 --- a/apps/morphir/cli/arguments.mbt +++ b/apps/morphir/cli/arguments.mbt @@ -375,7 +375,7 @@ fn render_help(path : String) -> String { text += "Process and WASM runtimes run on the native and npm CLIs; WASI builds refuse them with extension.runtime_unavailable.\n" text += "Sources and operations come from the extension's claims: compile reads the file extensions of one claimed language.\n" text += "--language selects that language when the extension claims several.\n" - text += "Each document URI is the file's path relative to --source-base (default: the current directory), percent-encoded.\n" + text += "Each document URI is the file's path relative to --source-base (default: the input directory), percent-encoded.\n" text += "compile writes model-envelope.json and model.ionb or model.json, or morphir-ir.json on MEP 0.1.\n" text += "generate reads a model envelope or Morphir IR JSON and writes the returned artifacts.\n" text += "JSON returns morphir/command-result 0.1.0; diagnostics and guest stderr go to stderr.\n" diff --git a/apps/morphir/cli/extension.mbt b/apps/morphir/cli/extension.mbt index 67f8b5d52..0d6dcd01c 100644 --- a/apps/morphir/cli/extension.mbt +++ b/apps/morphir/cli/extension.mbt @@ -470,7 +470,8 @@ fn extension_document_uri(path : String) -> String { ///| /// The directory that document URIs are relative to (SRC-1): `--source-base` -/// when the user gives it, else the current directory. +/// when the user gives it, else the input directory itself. The current +/// directory makes relative paths absolute. priv struct ExtensionSourceBase { given : String? cwd : String @@ -480,12 +481,13 @@ priv struct ExtensionSourceBase { ///| /// The names from the source base down to `path`, which must be the base or /// below it. Both paths are made absolute against the current directory and -/// normalized without reading the file system. +/// normalized without reading the file system. Without `--source-base` the +/// input is its own base, so there are no names. fn ExtensionSourceBase::segments( self : ExtensionSourceBase, path : String, ) -> Array[String] raise ExtensionError { - let base = self.given.unwrap_or(self.cwd) + guard self.given is Some(base) else { return [] } let relative = match self.style { Posix => { let absolute = (text : String) => { @@ -646,10 +648,11 @@ fn extension_source_order( ///| /// Reads a bounded snapshot of the source file or directory. Only files with /// a claimed extension are read. Each URI is the path relative to the source -/// base, with percent-encoded segments; a single file without -/// `--source-base` keeps the rule of `extension_document_uri`. Hidden, build -/// and dependency directories are skipped; a link, a special entry or an -/// unsafe name is refused. The documents are in URI byte order. +/// base, with percent-encoded segments. Without `--source-base`, a directory +/// is its own base and a single file keeps the rule of +/// `extension_document_uri`. Hidden, build and dependency directories are +/// skipped; a link, a special entry or an unsafe name is refused. The +/// documents are in URI byte order. fn extension_sources( path : String, extensions : Array[String], @@ -743,13 +746,7 @@ fn extension_sources( } files.push(file) } - 2 => { - let names = match explicit { - Some(names) => names - None => base.segments(path) - } - walk(path, names, 0) - } + 2 => walk(path, explicit.unwrap_or([]), 0) _ => raise input_invalid("\{path} is not a regular file or a directory") } if files.is_empty() { diff --git a/apps/morphir/cli/extension_session_wbtest.mbt b/apps/morphir/cli/extension_session_wbtest.mbt index 8dd483d6b..2056b7504 100644 --- a/apps/morphir/cli/extension_session_wbtest.mbt +++ b/apps/morphir/cli/extension_session_wbtest.mbt @@ -1640,7 +1640,7 @@ async test "language from claims: a guest with several languages needs --languag } ///| -async test "relative document URIs: a tree is named from the source base, with no sources.root" { +async test "relative document URIs: a tree is named from the input directory or the source base, with no sources.root" { let workspace = fake_workspace() fake_gleam_tree(workspace.root) // The guest claims one language, so no --language is needed. @@ -1673,6 +1673,17 @@ async test "relative document URIs: a tree is named from the source base, with n assert_eq(fake_uris(run.calls[0]), [ "src/app/money.gleam", "src/app/orders.gleam", ]) + // Without --source-base the input directory is the base, wherever the + // current directory is. + let run = fake_run( + fake_tree_compile(workspace, workspace.root + "/src", "own", []), + fake_gleam_guest(), + ) + assert_eq(run.report.exit_code(), 0, msg=run.json.stringify()) + assert_eq(fake_uris(run.calls[0]), [ + "app/money.gleam", "app/orders.gleam", "shared/numbers.gleam", + ]) + assert_false(run.calls[0].stringify().contains(workspace.root)) // An input outside the source base is refused before any operation // session, and the message names the source base. let run = fake_run( diff --git a/apps/morphir/cli/extension_wbtest.mbt b/apps/morphir/cli/extension_wbtest.mbt index 38eb99f15..34e366aeb 100644 --- a/apps/morphir/cli/extension_wbtest.mbt +++ b/apps/morphir/cli/extension_wbtest.mbt @@ -219,9 +219,9 @@ fn extension_test_tuple(operation : String, codec : String) -> Json { } ///| -/// A source base for the tests, with POSIX paths. Without `base` the current -/// directory is the input itself, so a directory gives URIs relative to it -/// and a single file follows the rule without `--source-base`. +/// A source base for the tests, with POSIX paths. Without `base` the input +/// directory is the base, so a directory gives URIs relative to it and a +/// single file follows the rule without `--source-base`. fn extension_test_base(path : String, base? : String) -> ExtensionSourceBase { { given: base, cwd: path, style: Posix, } } @@ -370,10 +370,14 @@ test "the source base gives each URI its path relative to the base" { cwd, style: Posix, } - // Run from the project directory: the base is the current directory. - assert_eq(base(None, "/work/shop").segments("src"), ["src"]) - assert_eq(base(None, "/work/shop").segments("src/app/"), ["src", "app"]) - assert_eq(base(None, "/work/shop").segments("."), []) + // Without --source-base the input is its own base, wherever the current + // directory is. + assert_eq(base(None, "/work/shop").segments("src/app/"), []) + assert_eq(base(None, "/tmp").segments("/work/shop/src"), []) + // A base of the current directory names the path from it. + assert_eq(base(Some("."), "/work/shop").segments("src"), ["src"]) + assert_eq(base(Some("."), "/work/shop").segments("src/app/"), ["src", "app"]) + assert_eq(base(Some("."), "/work/shop").segments("."), []) // An explicit base gives the same segments for an absolute input. assert_eq(base(Some("/work/shop"), "/tmp").segments("/work/shop/src"), ["src"]) // A relative base is relative to the current directory. @@ -382,17 +386,14 @@ test "the source base gives each URI its path relative to the base" { for (refused, path) in [ (base(Some("/work/shop"), "/tmp"), "/other/src"), - (base(None, "/work/shop/src"), ".."), + (base(Some("/work/shop/src"), "/work/shop/src"), ".."), (base(Some("/work/shop"), "/tmp"), "/work/shopping/src"), ] { let (code, message, status) = extension_test_error(() => { ignore(refused.segments(path)) }) assert_eq((code, status), ("extension.input_invalid", 2), msg=path) - assert_true( - message.contains(refused.given.unwrap_or(refused.cwd)), - msg=message, - ) + assert_true(message.contains(refused.given.unwrap()), msg=message) } // Windows paths compare drive letters without case. let windows = ExtensionSourceBase::{ @@ -408,7 +409,7 @@ test "the source base gives each URI its path relative to the base" { } ///| -test "a tree compiled from the project directory or with --source-base has URIs from the base" { +test "a tree has URIs from the input directory, or from --source-base when given" { let root = @platform.temporary() @platform.mkdir(root + "/src") @platform.mkdir(root + "/src/app") @@ -419,16 +420,27 @@ test "a tree compiled from the project directory or with --source-base has URIs let expected = [ "src/app/money.gleam", "src/app/orders.gleam", "src/shared/numbers.gleam", ] - // The current directory is the project directory. - let from_project = extension_sources(root + "/src", [".gleam"], () => (), base={ - given: None, - cwd: root, - style: Posix, - }) catch { + // Without --source-base the input directory is the base, also when the + // current directory is elsewhere: `compile src` names `app/money.gleam` + // and `compile .` names `src/app/money.gleam`. + let own = (path : String, cwd : String) => { + extension_sources(path, [".gleam"], () => (), base={ + given: None, + cwd, + style: Posix, + }).map(document => document.uri) + } + let uris = own(root + "/src", "/elsewhere") catch { + ExtensionError(message~, ..) => fail("refused: \{message}") + } + assert_eq(uris, [ + "app/money.gleam", "app/orders.gleam", "shared/numbers.gleam", + ]) + let uris = own(root, "/elsewhere") catch { ExtensionError(message~, ..) => fail("refused: \{message}") } - assert_eq(from_project.map(document => document.uri), expected) - // An explicit source base gives the same URIs. + assert_eq(uris, expected) + // An explicit source base names each file from the base. assert_eq(extension_test_sources(root + "/src", base=root), Ok(expected)) // A deeper input keeps the URIs from the base, so module names do not // depend on the directory that the user names. diff --git a/apps/morphir/docs/extension.md b/apps/morphir/docs/extension.md index 6d4eef617..8c4ad360e 100644 --- a/apps/morphir/docs/extension.md +++ b/apps/morphir/docs/extension.md @@ -187,45 +187,58 @@ morphir extension compile src --package acme/shop --output model --extension-pin ``` The request holds the whole tree as three documents. Each URI is the file's path -relative to the **source base**, which is the current directory by default: +relative to the **source base**. Without `--source-base`, the source base is the +directory that you name, here `src`: ```json { "languageId": "gleam", "sources": { "documents": [ - { "uri": "src/app/money.gleam", "languageId": "gleam", "version": 0, "text": "..." }, - { "uri": "src/app/orders.gleam", "languageId": "gleam", "version": 0, "text": "..." }, - { "uri": "src/shared/numbers.gleam", "languageId": "gleam", "version": 0, "text": "..." } + { "uri": "app/money.gleam", "languageId": "gleam", "version": 0, "text": "..." }, + { "uri": "app/orders.gleam", "languageId": "gleam", "version": 0, "text": "..." }, + { "uri": "shared/numbers.gleam", "languageId": "gleam", "version": 0, "text": "..." } ] } } ``` -The Gleam guest finds the `src` segment and names the modules `app/money`, -`app/orders` and `shared/numbers`. The module names do not depend on the -directory that you name: `compile src/app` from `/work/shop` sends -`src/app/money.gleam` and `src/app/orders.gleam`. Thus, name `src` (or the -project directory) from the project directory. The request has no -`sources.root` and no absolute host path, so the model's source provenance is -the same on every machine. - -From another directory, give the source base with `--source-base`. This command -sends the same URIs: +The Gleam guest names the modules `app/money`, `app/orders` and +`shared/numbers` from these paths. It also removes a leading path up to a `src` +segment, so `compile .` from `/work/shop` sends `src/app/money.gleam` and gives +the same module names. The current directory has no effect on the URIs: +`compile /work/shop/src` from any directory sends the same request. The request +has no `sources.root` and no absolute host path, so the model's source +provenance is the same on every machine. + +The default suits the published Elm and Python extensions, which also name a +module from its path below the source directory. For example, the Elm +extension requires the module `Acme.Rules.Main` in `Acme/Rules/Main.elm`, so +`compile generated/src` must send `Acme/Rules/Main.elm`, not +`generated/src/Acme/Rules/Main.elm`. + +When you name a directory below the root of the module paths, give that root +with `--source-base`. Then each URI is the path relative to the source base, and +the module names do not depend on the directory that you name. This command +sends `src/app/money.gleam` and `src/app/orders.gleam`, so the Gleam guest names +the modules `app/money` and `app/orders`, as for `compile src`: ```sh -morphir extension compile /work/shop/src --source-base /work/shop --package acme/shop --output model --extension-pin pin.json +morphir extension compile /work/shop/src/app --source-base /work/shop --package acme/shop --output model --extension-pin pin.json ``` +Without `--source-base`, the same input sends `money.gleam` and `orders.gleam`, +and the modules are `money` and `orders`. + The input must be the source base or a path below it. The CLI compares the two -paths after it makes them absolute and removes `.` and `..`; it does not resolve -symbolic links. An input outside the source base, such as `/other/src` with -`--source-base /work/shop`, is refused with `extension.input_invalid`, and the -message names the source base. A single file without `--source-base` keeps the -earlier rule, so a MoonBit request does not change: a clean relative path such -as `src/main.mbt` keeps its nesting, and an absolute path or a path with `..` -keeps only its file name. A single file with `--source-base` takes its path -relative to the source base. +paths after it makes them absolute against the current directory and removes +`.` and `..`; it does not resolve symbolic links. An input outside the source +base, such as `/other/src` with `--source-base /work/shop`, is refused with +`extension.input_invalid`, and the message names the source base. A single +file without `--source-base` keeps the earlier rule, so a MoonBit request does +not change: a clean relative path such as `src/main.mbt` keeps its nesting, and +an absolute path or a path with `..` keeps only its file name. A single file +with `--source-base` takes its path relative to the source base. Each URI segment is encoded as an RFC 3986 path segment. Letters, digits, `-`, `.`, `_` and `~` stay as they are, so a plain path does not change. Each other @@ -268,7 +281,7 @@ sources; a larger request is refused with `extension.request_limit`. | `--extension-pin FILE` | all | The pin; or `MORPHIR_EXTENSION_PIN` | | `` | compile | A source file, or a directory snapshot of claimed source files | | `--language ID` | compile | One of the claimed languages; required when the extension claims more than one | -| `--source-base DIR` | compile | The directory that document URIs are relative to; the current directory by default. The source must be this directory or below it | +| `--source-base DIR` | compile | The directory that document URIs are relative to; by default a directory source is its own base. The source must be this directory or below it | | `` | generate | A model envelope, or plain Morphir IR JSON | | `--package NAME` | compile | The package name; required | | `--model-format ion\|json` | compile | The one model tuple that the CLI offers on the model line. Without it, the CLI offers Ion, then JSON, and uses the first tuple that the extension agrees on. MEP `0.1` compiles return Morphir JSON and refuse `ion` | diff --git a/openspec/changes/run-core-wasm-guests-from-any-language/design.html b/openspec/changes/run-core-wasm-guests-from-any-language/design.html index e118e47fa..1ecc8df21 100644 --- a/openspec/changes/run-core-wasm-guests-from-any-language/design.html +++ b/openspec/changes/run-core-wasm-guests-from-any-language/design.html @@ -23,7 +23,7 @@

run-core-wasm-guests-from-any-language: design

-

OpenSpec change · Bead morphir-ukvn.5.64.37 · 2026-10-30, updated 2026-10-31 · scope cut to the work that main does not do yet; FIT-1 to FIT-5, PIN-2, IGN-1 to IGN-5 and PIN-3 answered; Gleam relative nesting and the release resolved; named-directory rule confirmed

+

OpenSpec change · Bead morphir-ukvn.5.64.37 · 2026-10-30, updated 2026-11-01 · default source base changed to the input directory (SRC-1); scope cut to the work that main does not do yet; FIT-1 to FIT-5, PIN-2, IGN-1 to IGN-5 and PIN-3 answered; Gleam relative nesting and the release resolved; named-directory rule confirmed