diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md deleted file mode 100644 index ba880532e92..00000000000 --- a/.github/copilot-instructions.md +++ /dev/null @@ -1,245 +0,0 @@ -# SynapseML Copilot Instructions - -SynapseML is an open-source library providing scalable machine learning pipelines -for Apache Spark. It wraps algorithms (LightGBM, VW, Azure AI Services, ONNX, OpenCV) -as SparkML-compatible `PipelineStage`s with auto-generated Python bindings. - -## Architecture - -### Module Map - -| Module | Directory | Purpose | -|--------|-----------|---------| -| **core** | `core/` | Foundational transformers, featurizers, IO, codegen, automl, causal inference | -| **cognitive** | `cognitive/` | Azure AI Services wrappers (OpenAI, Vision, Speech, Text, etc.) | -| **lightgbm** | `lightgbm/` | LightGBM classifier/regressor/ranker for Spark | -| **vw** | `vw/` | Vowpal Wabbit integration | -| **deep-learning** | `deep-learning/` | ONNX Runtime inference | -| **opencv** | `opencv/` | Image transformations via OpenCV | - -All modules depend on `core`. `deep-learning` also depends on `opencv`. - -### Directory Layout (same pattern in every module) - -``` -{module}/ -├── src/ -│ ├── main/ -│ │ ├── scala/com/microsoft/azure/synapse/ml/{package}/ -│ │ │ ├── MyTransformer.scala ← primary source code -│ │ │ └── MyTransformerParams.scala ← parameter traits (optional) -│ │ └── python/synapse/ml/{package}/ -│ │ └── MyTransformer.py ← hand-written Python (if needed) -│ └── test/ -│ ├── scala/com/microsoft/azure/synapse/ml/{package}/ -│ │ └── MyTransformerSuite.scala ← ScalaTest tests -│ └── python/synapsemltest/{package}/ -│ └── test_my_transformer.py ← Python tests -└── target/ - └── scala-2.13/generated/src/python/ ← AUTO-GENERATED (never edit) -``` - -## Critical: The Code Generation Pipeline - -**SynapseML auto-generates Python wrappers from Scala code.** This is the most -important thing to understand. - -### How It Works - -1. A Scala class mixes in the `Wrappable` trait -2. Running `sbt codegen` calls `makePyFile()` which generates a Python class -3. Generated files go to `target/scala-2.13/generated/src/python/synapse/ml/` -4. Generated files use underscore prefix: `_ClassName.py` -5. Hand-written Python in `src/main/python/` can extend the generated class - -### What This Means for You - -- **To add or change a feature**: Edit the **Scala** code. The Python wrapper - regenerates automatically. -- **Never edit files in `target/`**: They are overwritten on every build. -- **Hand-written Python** (`src/main/python/`) is only for cases where the - generated wrapper needs manual overrides or additional logic. - -### Example: Generated vs Hand-Written Python - -Generated (DO NOT EDIT): `target/.../synapse/ml/isolationforest/_IsolationForestModel.py` - -Hand-written override (OK to edit): `core/src/main/python/synapse/ml/isolationforest/IsolationForestModel.py` -```python -from synapse.ml.isolationforest._IsolationForestModel import _IsolationForestModel - -class IsolationForestModel(_IsolationForestModel): - def getInnerModel(self): - return self._java_obj.getInnerModel() -``` - -## Scala Patterns - -### Transformer/Estimator Pattern - -Every SynapseML stage follows this pattern: - -```scala -// Copyright (C) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. See LICENSE in project root for information. - -package com.microsoft.azure.synapse.ml.stages - -import com.microsoft.azure.synapse.ml.codegen.Wrappable -import com.microsoft.azure.synapse.ml.logging.{FeatureNames, SynapseMLLogging} -import org.apache.spark.ml.Transformer -import org.apache.spark.ml.param._ -import org.apache.spark.ml.util._ -import org.apache.spark.sql.types._ -import org.apache.spark.sql.{DataFrame, Dataset} - -object DropColumns extends DefaultParamsReadable[DropColumns] - -class DropColumns(val uid: String) - extends Transformer with Wrappable with DefaultParamsWritable with SynapseMLLogging { - logClass(FeatureNames.Core) - - def this() = this(Identifiable.randomUID("DropColumns")) - - val cols: StringArrayParam = - new StringArrayParam(this, "cols", "Comma separated list of column names") - - def getCols: Array[String] = $(cols) - def setCols(value: Array[String]): this.type = set(cols, value) - - override def transform(dataset: Dataset[_]): DataFrame = { - logTransform[DataFrame]({ - dataset.toDF().drop(getCols: _*) - }, dataset.columns.length) - } - - def transformSchema(schema: StructType): StructType = { - val droppedCols = getCols.toSet - StructType(schema.fields.filter(f => !droppedCols(f.name))) - } - - def copy(extra: ParamMap): DropColumns = defaultCopy(extra) -} -``` - -### Key Conventions - -- **Companion object**: Always add `extends DefaultParamsReadable[ClassName]` - for model serialization. -- **`Wrappable` trait**: Required for Python code generation. Without it, no - Python wrapper is created. -- **`SynapseMLLogging` trait**: Required on all transformers/estimators. Call - `logClass(FeatureNames.X)` in the constructor and wrap `transform`/`fit` - with `logTransform`/`logFit`. -- **Parameter traits**: For complex stages, define params in a separate trait - (e.g., `trait MyParams extends Wrappable with HasInputCol`) and mix it into - the class. This is the SynapseML composition pattern. -- **`uid` parameter**: Every stage must accept `uid: String` and provide a - no-arg constructor that generates a random UID. - -### Cognitive Module (Azure AI Services) - -The `cognitive` module follows a different pattern using service-oriented traits: -```scala -trait HasServiceParams extends Params // base for all service parameters -trait HasSubscriptionKey extends HasServiceParams -trait HasAADToken extends HasServiceParams -``` -Services extend `CognitiveServicesBase` instead of raw `Transformer`. - -### File Headers - -Every Scala file **must** start with this exact header (enforced by scalastyle): -```scala -// Copyright (C) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. See LICENSE in project root for information. - -package com.microsoft.azure.synapse.ml.{package} -``` - -Python files use the same copyright comment: -```python -# Copyright (C) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See LICENSE in project root for information. -``` - -## Build System - -SynapseML uses **sbt** (not Maven or Gradle). Spark 4.1.1, Scala 2.13.17. - -### Essential Commands - -```bash -sbt compile # compile all modules -sbt test:compile # compile all tests -sbt core/compile # compile just the core module -sbt scalastyle test:scalastyle # run Scala style checks -sbt codegen # regenerate Python/R wrappers from Scala -``` - -### Python Style - -- **Formatter**: `black` pinned to **22.3.0** (configured in `pyproject.toml`) -- **Environment**: conda env named `synapseml` (defined in `environment.yml`) -- Run locally: `black --check --extend-exclude 'docs/' .` - -### Scalastyle Rules - -- Max file length: 800 lines -- Max line length: 120 characters -- No tabs, no trailing whitespace -- License header required (see above) -- Token names max 40 characters - -## Testing - -### Scala Tests - -- **Framework**: ScalaTest (`AnyFunSuite` via `TestBase` trait) -- **SparkSession**: Provided automatically by `TestBase` (local mode) -- **Test location**: `{module}/src/test/scala/com/microsoft/azure/synapse/ml/{package}/` - -```scala -class MyTransformerSuite extends TestBase { - test("MyTransformer should transform data") { - val df = spark.createDataFrame(Seq(("a", 1), ("b", 2))).toDF("col1", "col2") - val result = new MyTransformer().setCols(Array("col1")).transform(df) - assert(result.columns.length == 1) - } -} -``` - -Tests that call Azure services or require external resources will be skipped -without credentials. Pure Spark tests run anywhere. - -### Python Tests - -- Located in `{module}/src/test/python/synapsemltest/` -- Require PySpark and the `synapseml` conda environment -- Run via: `sbt "testOnly *PythonTests*"` (runs through sbt, not pytest directly) - -## CI/CD - -- **Main build**: Azure DevOps pipeline (`pipeline.yaml`) — full test suite, 45+ min -- **GitHub Actions**: Lightweight checks only (style, compile, dead links, dependency review) -- **PR feedback**: GitHub Actions runs in ~5 min; ADO requires `/azp run` comment -- **PR titles**: Must follow conventional commits (`feat:`, `fix:`, `ci:`, `chore:`, `test:`, `docs:`) - -## Common Mistakes - -1. **Editing generated Python files** — They live in `target/` and are overwritten. - Edit the Scala source instead. -2. **Forgetting `Wrappable`** — If you add a new Scala transformer and forget - `with Wrappable`, it won't get a Python wrapper. -3. **Forgetting `SynapseMLLogging`** — All stages must mix in this trait and - call `logClass()` in the constructor. -4. **Missing companion object** — Without `object Foo extends DefaultParamsReadable[Foo]`, - model deserialization will fail. -5. **Wrong black version** — Using latest black instead of 22.3.0 will show - false formatting failures. -6. **Putting logic in Python** — SynapseML is Scala-first. Python wrappers - delegate to the JVM. Put business logic in Scala. -7. **Missing license header** — Scalastyle will reject files without the - Microsoft copyright header. -8. **Using RDD API** — SynapseML uses the DataFrame/Dataset API exclusively. - Never introduce RDD-based code. diff --git a/.github/skills/scala-code/SKILL.md b/.github/skills/scala-code/SKILL.md new file mode 100644 index 00000000000..1b90452de2b --- /dev/null +++ b/.github/skills/scala-code/SKILL.md @@ -0,0 +1,49 @@ +--- +name: scala-code +description: Write and modify Scala code in SynapseML. Use when adding or changing Scala transformers, estimators, or params to follow repo patterns, keep scalastyle and compilation green, and cover behavior changes with tests. +--- + +# Scala Code Skill + +Use this skill for any Scala code change in this repository. + +## Objectives +- Keep Scala changes correct, minimal, and production-safe. +- Prevent CI/CD breakage by validating style and compilation before completion. +- Ensure every behavior change is covered by tests. +- Scala code should be optimized to be efficient and maintainable, following existing patterns and practices in the codebase. + +## Scala best practices +- Follow existing SynapseML patterns (`DefaultParamsReadable`, `DefaultParamsWritable`, `Wrappable`, `SynapseMLLogging`). +- Keep business logic in Scala (not generated Python wrappers). +- Do not edit generated files under `target/`. +- Preserve license headers and existing package structure. +- Prefer small, focused changes and reuse existing helpers/traits. +- Avoid introducing flaky tests or network-dependent behavior unless already required by the suite. +- To get around scalastyle issues, don't just use `// scalastyle:off` or `//scalastyle:ignore`, but instead fix the underlying issue or refactor to avoid it. Unless, the refactor results in a more complex code structure, in which case, it may be acceptable to disable the specific scalastyle rule for that line or block of code, but this should be done sparingly and with justification. + +## Required validation steps (must run) +Run these commands before finishing Scala changes: + +1. Scala style checks: + - `sbt scalastyle "Test / scalastyle"` +2. Scala compile checks: + - `sbt compile` + - `sbt test:compile` +3. Relevant tests for touched modules/files: + - Example: `sbt "core/testOnly *SuiteName*"` or `sbt "cognitive/testOnly *SuiteName*"` + +If a command fails, fix the issue and rerun until passing. + +## Testing requirement for code changes +- Any Scala code change must be accompanied by tests (new tests or updates to existing tests). +- Bug fixes must include a regression test that fails before the fix and passes after. +- New logic/branches should include coverage for success and failure/edge cases where practical. +- Keep tests deterministic and aligned with current module test conventions. + +## Completion checklist +- [ ] Code follows existing Scala/SynapseML conventions. +- [ ] Style checks pass. +- [ ] Compile checks pass. +- [ ] Relevant tests pass. +- [ ] Scala code changes include corresponding tests. diff --git a/.github/skills/synapseml-branches/SKILL.md b/.github/skills/synapseml-branches/SKILL.md new file mode 100644 index 00000000000..c896fb6510d --- /dev/null +++ b/.github/skills/synapseml-branches/SKILL.md @@ -0,0 +1,57 @@ +--- +name: synapseml-branches +description: >- + Resolve SynapseML branch-specific rules, runtime baselines, sync policy, and + CI expectations. Use before editing, rebasing, merging, testing, or declaring + readiness for master, spark3.5, spark4.0, spark4.1, port/sync branches, or a PR + whose base branch determines behavior. +compatibility: SynapseML repository with git and GitHub CLI. +--- + +# SynapseML branch context + +Use the PR base branch as the context. A feature branch name does not determine +the runtime, sync policy, or CI that must pass. + +## Workflow + +1. Read root `AGENTS.md`. +2. Resolve the target: + - For a PR, read `baseRefName` from GitHub. + - For direct branch work, use the checked-out shared branch. +3. Load the mapped reference. Filenames use `p` for the version decimal: + - `master` -> + [branch-spark3p5.md](references/branch-spark3p5.md). `master` is currently + the canonical Spark 3.5 development baseline. + - `spark3.5` -> + [branch-spark3p5.md](references/branch-spark3p5.md). This is the shared + Spark 3.5 release branch. + - `spark4.0` -> + [branch-spark4p0.md](references/branch-spark4p0.md). + - `spark4.1` -> + [branch-spark4p1.md](references/branch-spark4p1.md). + - Any other target -> + [branch-fallback.md](references/branch-fallback.md). +4. Verify every version, dependency, trigger, skip, and test command against + that branch's live `build.sbt`, `environment.yml`, workflows, and + `pipeline.yaml`. References are decision guides, not stale-value authority. +5. Recheck branch context at three points: before implementation, before + validation, and immediately before push/readiness. Target movement or a + changed base invalidates earlier evidence. + +## Responsibilities + +- Ordinary PRs rebase onto their latest target with `--force-with-lease`. +- Shared port branches receive `master` by merge; never rebase or force-push + the shared branch. +- For conflict resolution, compare merge base, `master`, and port branch; + ancestry alone does not prove both sides survived. +- Inspect CI definitions on the target branch and confirm builds actually + queued. Never infer port-branch coverage from `master`. +- Confirm the relevant suites ran by test result/class, not only job status. +- Treat `.github/skills/` as authoritative. `.agents/` is compatibility-only + and may contain stale copies. +- If no exact reference exists, follow the fallback, state uncertainty, and + add a concise reference from the + [branch template](references/branch-template.md) when the branch is an active + supported target. diff --git a/.github/skills/synapseml-branches/references/branch-fallback.md b/.github/skills/synapseml-branches/references/branch-fallback.md new file mode 100644 index 00000000000..d7cb24b9ca3 --- /dev/null +++ b/.github/skills/synapseml-branches/references/branch-fallback.md @@ -0,0 +1,14 @@ +# Unlisted target branch + +- Determine whether the target is an ordinary feature branch, a shared port + branch, a release branch, or an automation branch. +- Read its live build files, workflows, pipeline triggers, recent sync PRs, and + open/closed PRs targeting it. +- Do not guess versions, supported environments, merge policy, or CI coverage + from the branch name. +- Rebase ordinary PRs; merge into shared branches unless repository history + proves a different maintained policy. +- Confirm actual queued checks and per-test execution. +- State unresolved branch-specific uncertainty as a blocker. If the target is + actively supported, add a focused reference using + [branch-template.md](branch-template.md) before claiming readiness. diff --git a/.github/skills/synapseml-branches/references/branch-spark3p5.md b/.github/skills/synapseml-branches/references/branch-spark3p5.md new file mode 100644 index 00000000000..274195dffb8 --- /dev/null +++ b/.github/skills/synapseml-branches/references/branch-spark3p5.md @@ -0,0 +1,31 @@ +# Spark 3.5 branch context + +This reference intentionally serves both `master` and `spark3.5`. + +## Branch mapping + +- `master` is the canonical development branch and currently uses the Spark 3.5 + runtime family. Ordinary features and fixes target `master`. +- `spark3.5` is the shared Spark 3.5 release branch. It can differ from + `master` despite using the same Spark generation. +- Always verify exact Spark/Scala versions and dependency pins in the target + branch's live `build.sbt` and `environment.yml`. + +## Sync policy + +- PRs targeting `master` rebase onto latest `master` and push with lease. +- Synchronize the shared `spark3.5` branch by merging `master`; never rebase or + force-push that shared ref. +- Cross-version work lands on `master` first unless it exists only to preserve + the release branch. + +## Validation + +- Master validation covers the primary Spark 3.5 runtime, so compatibility + replay can intentionally omit a duplicate Spark 3.5 leg. This does not waive + direct CI for PRs targeting the `spark3.5` branch. +- Inspect workflows and Azure triggers on the actual target branch; historical + GitHub coverage for `spark3.5` differed from `master`. +- Confirm the affected suites were selected and executed. Green matrices can + omit an unclaimed package or explicit test class. +- Recheck target movement immediately before readiness. diff --git a/.github/skills/synapseml-branches/references/branch-spark4-common.md b/.github/skills/synapseml-branches/references/branch-spark4-common.md new file mode 100644 index 00000000000..d3063cf01ca --- /dev/null +++ b/.github/skills/synapseml-branches/references/branch-spark4-common.md @@ -0,0 +1,347 @@ +# Shared Spark 4 branch context + +Condensed from the branch guides developed in +[#2645](https://github.com/microsoft/SynapseML/pull/2645) and +[#2646](https://github.com/microsoft/SynapseML/pull/2646). + +**Read this as describing the branches as of those two sync PRs, not as a +snapshot of the live branches.** Both guides were written on the sync branches, +so they describe the merged result. Until those PRs land, a live branch can lack +things described here — at the time of writing, live `spark4.0` has no +`OpenAIPromptPythonOverrides.scala`, no `test_http_package.py` / +`test_package_exports.py`, no `new_ml_pipeline_stage` in generated R, and its +own GPU pool (`DatabricksUtilities.scala` sets +`GpuPoolName = "synapseml-build-17.3-gpu"` there, not master's +`synapseml-build-14.3-gpu`). Verify every item against the live target branch +with `git show :` or `git grep `; do not read it +out of a local sync worktree, which is a PR result rather than branch state. +That specific mistake produced several wrong claims in earlier revisions of this +file, and an automated reviewer then made it against this very paragraph — +reporting the GPU pool sentence as stale after reading the sync branch. Quote +the file and value you checked, so the next reader can repeat the check instead +of re-deriving it from whatever tree they happen to have open. + +## Purpose and sync + +- Spark 4 branches are maintained ports, not feature branches. Land ordinary + work on `master`, then merge it into the port branch. +- Resolve conflicts per hunk and compare content with the merge base and + `master`; blanket `ours`/`theirs` and reachability are insufficient. +- Diff `spark4.0` and `spark4.1` before debugging or merging + (`git diff spark4.0 spark4.1 -- `). `spark4.1` descends from `spark4.0`'s + upgrade commit and is maintained more actively, so it has usually already hit + and solved the same problem — it has directly supplied the fixes for R parsing, + the R Spark connection, nested-stage loading in R, the generated-wrapper + `super()` bug, the stale `__init__.py` shims, the local setup skill, the + notebook runtime, and a failing training test. This is the single + highest-value habit on these branches. Two cautions when porting: substitute + the target branch's version strings, and confirm the fix is not specific to + the source branch's Spark/Python version. + +## Common deliberate differences from master + +- Spark 4 uses Scala 2.13 and Java 17-era tooling, so generated Python lands in + `target/scala-2.13/generated/src/python/` rather than master's `scala-2.12` + path. The `tools/docker/*/Dockerfile` files set `JAVA_HOME` to Java 17 on + both Spark 4 branches, where master sets Java 11 — check the branch, not + master, if you are verifying this. `pipeline.yaml` + drops master's `-XX:+UseConcMarkSweepGC -XX:+CMSClassUnloadingEnabled` from + `SBT_OPTS`: CMS was removed in Java 17 and the JVM refuses to start with those + flags, so a sync that restores them fails before any test runs. +- `environment.yml` moves pins forward for the branch's Python, and each pin + carries a comment saying why. Those comments are the mechanism that stops a + later sync from "restoring" master's value, so preserve them through conflict + resolution. The recurring reasons: master's `pip` is too old to install for + these interpreters, `torch`/`torchvision` need their first releases supporting + the version, and `pandas`/`horovod` come from interpreter-specific wheel URLs. + + Measured, since these are easy to get backwards: + + | | master | spark4.0 | spark4.1 | + | --- | --- | --- | --- | + | `python` | 3.11.8 | 3.12.11 | 3.13 | + | `pyarrow` | 10.0.1 | 22.0.0 | 18.0.0 | + | `mlflow` | 2.21.3 | 1.26.1 | 2.21.3 | + + Each branch reached its `pyarrow` by a different route, so do not carry the + reasoning across: `spark4.0` needs a release with cp312 wheels (older ones + such as 11.0.0 have none and would build from source), `spark4.1` needs cp313 + wheels under an `mlflow` 2.x `pyarrow<19` bound, and master is held *down* at + 10.0.1 because Petastorm uses legacy Parquet and fsspec APIs removed after + PyArrow 10. Note also that `spark4.0` carries `mlflow==1.26.1`, a downgrade + from master's 2.21.3, which is not explained by the Python version and has not + been validated. +- The `pyarrow` and `mlflow` pins are coupled, and the pinned versions are not + the same on every branch — read both live values on the branch you are editing + before changing either. The bound comes from MLflow: `mlflow==2.21.3` declares + `pyarrow<20,>=4.0.0`, so on a branch pinning that MLflow, `pyarrow` must stay + under 20 or move together with `mlflow`; bumps inside the bound are fine. Older + MLflow pins carry different bounds, so check the pinned version's own metadata + rather than assuming this one. Do not trust the inline comments: they disagree + with each other and with the pins they sit next to. +- Scala 2.13 collection boundaries must produce immutable `Seq` values. The + failure mode is why this matters: code that yields a `mutable.ArraySeq` where + an `immutable.Seq` is expected throws `ClassCastException` **at runtime, not + at compile time**, so a green compile proves nothing and the break surfaces + one or two layers away from its cause. Prefer `toIndexedSeq` over `toList` + when converting, because it preserves O(1) indexing. Be careful what you + believe about *where* this is handled: the branch-local notes claimed + `CognitiveServiceBase.getValueOpt` converted centrally through a helper called + `asImmutableCollection`, and neither is true — `getValueOpt` returns the row + or default value with no conversion, and `asImmutableCollection` appears in no + branch, only in two abandoned commits (`745b342b48`, `6cab133efd`) that are + contained in no tip. Verify with + `git grep asImmutableCollection ms/master ms/spark4.0 ms/spark4.1`. If the + `ClassCastException` resurfaces, one conversion in `CognitiveServiceBase` is + the right shape of fix, but treat it as a change to make rather than one + already in place. +- Preserve the Spark 4 adaptations. In `SAR.scala`/`SARModel.scala` the affinity + pairs use a named `case class` with explicit struct fields because Spark 4 + rejects the old `Seq[Row]` UDF shape with `UnboundRowEncoder`, and the join + column is qualified (`col("sarUserFactors.flatList")`) because a self-join now + trips `DetectAmbiguousSelfJoin`. `Wrappable.safeGetDefault` guards + `getDefault`, which throws on Spark 4 where Spark 3 returned a default. + `VerifyTrainClassifier`'s vector fixture no longer feeds `Double.NaN` to the + trainer, because Spark 4 does not tolerate a NaN feature reaching logistic + regression the way 3.5 did. That test is about training on a vector column, + not about NaN, so the value was replaced rather than the assertion weakened. + Master still has the `Double.NaN` at `VerifyTrainClassifier.scala:121`, so a + sync will try to restore it; do not let it, and do not weaken the assertion + instead. +- `OpenAIPrompt` sets `pyInternalWrapper = true`, so codegen emits + `class _OpenAIPrompt` and a hand-written `OpenAIPrompt.py` supplies the public + name. Python emitted into that class must use zero-argument `super()`; a + hardcoded `super(OpenAIPrompt, self)` raises `NameError` because that name does + not exist inside the generated module. See `OpenAIPromptPythonOverrides.scala`, + which is on `spark4.1` and reaches `spark4.0` with #2646. +- `PythonInitMerger` makes hand-written `__init__.py` files live package code by + splicing them *after* the generated imports; before it, codegen overwrote them + and their contents were inert, so a stale one is now a real bug. Keep the HTTP + initializer empty — it listed `HTTPFunctions` and `ServingFunctions`, which are + modules of free functions with no same-named class, and the failed import broke + `PythonTests core` plus seven website samples. Remove initializers that only + duplicate generated exports, and do not narrow `import *` by redefining + `__all__` as a hand-maintained list. Keep the ones that add exports codegen + does not emit. `test_http_package.py` and `test_package_exports.py` guard this + where they exist; they are not on live `spark4.0` yet and arrive there with + #2646, so on that branch the policy is currently unenforced. +- `cyber/utils/spark_utils.py` differs between the branches without either form + being version-specific: `spark4.0` builds its indexed frame with + `rdd.toDF(schema)` and `spark4.1` uses `spark.createDataFrame(rdd, schema)`. + `toDF` was measured working on both 4.0.1 and 4.1.1, so this is a portable + choice rather than a hazard. Adopting 4.1's form only reduces reliance on the + monkey-patched RDD API, which does not exist under Spark Connect, and buys + little on its own while the surrounding `df.rdd.zipWithIndex()` remains an RDD + call. +- R generation requires ANSI double-quoted identifiers — `RTestGen.scala` sets + `spark.sql.ansi.enabled=true` and `spark.sql.ansi.doubleQuotedIdentifiers=true`, + because sparklyr emits `SELECT 0L AS "class", ...` and without the second flag + Spark 4 reads `"class"` as a string literal and fails with + `PARSE_SYNTAX_ERROR`. It also requires the validated sparklyr + 1.9.5 pin from the PR snapshots, `SPARK_HOME` connection behavior, and JVM + loading of nested stages. That pin is not yet everywhere — check + `environment.yml` on your branch, since a branch still on sparklyr 1.9.3 has + the failure below as a live concern rather than as history. Where 1.9.5 is + applied, keep it paired with `r-base=4.4`: 69/69 `RTests` was measured for the + combination, not for the sparklyr pin alone. Interleaved failures with + successful tests between them point to selection/proxy behavior, not a dead + Spark session; read the backtrace. Under sparklyr 1.9.3 with dbplyr 2.6 the + tell is a frame chain through `dbplyr:::select.tbl_lazy`, + `sparklyr:::tidyselect_data_proxy.tbl_spark` + and `simulate_vars_spark`, which surfaces as `invoke_static`/`hive_context` + being called on `NULL` and reads misleadingly like a dead session. +- `RCodegenSuite` asserts cheap R generation invariants without a full pipeline + run, but it is not present on every branch — `spark4.1` has it and `spark4.0` + does not yet. Check for it before relying on it, run it before spending a + pipeline run on an R failure, and keep its assertions in step when changing + generated R. +- Nested stages load off the JVM on branches that have adopted it — `spark4.1` + has, live `spark4.0` has not yet. `PipelineStageWrappable.rLoadLine` emits + `sparklyr:::new_ml_pipeline_stage(invoke(spark_jobj(x), "getStages")[[1]])` + rather than `ml_stages(x)[[1]]`. `new_ml_pipeline_stage` is sparklyr-internal + but has an identical signature in every release from v1.8.0 to v1.9.5. + `EstimatorParam`, `ModelParam`, `PipelineStageParam` and `TransformerParam` all + inherit this single implementation — do not reintroduce per-class overrides, + and keep the three `rLoadLine` assertions in + `VerifyModelParam`/`VerifyPipelineStageParams` in step with it. Be accurate + about its status: on a branch whose R tests died earlier on `ml_load`, this + line was never reached, so it is alignment with the working branch rather than + a proven fix. + +## Runtime and CI + +- Spark 4 Databricks builds contend for scarce GPU capacity, and the pool names + differ by branch — read them from `pipeline.yaml`/the Databricks test config on + your branch rather than assuming. Instance pools are runtime-agnostic, so a + GPU pool is often deliberately shared across branches to avoid duplicating + scarce quota; where it is shared it holds three workers + (`GpuWorkersPerRun` 1 x `GpuConcurrentRuns` 3), so two concurrent builds can + exhaust it. Queue Spark 4 builds sequentially. Where the pool *is* shared, the + sibling branch is a free control: an outcome that tracks the branch rather + than the timing is a code difference, not contention. +- `areLibrariesInstalled == false` is a timeout, not a capacity verdict, and the + logic inverts the way people expect. The check *throws* `Library Installation + Failure` with the offending statuses if any library reports `FAILED`, so + returning `false` means the opposite: nothing failed, the libraries simply had + not all reached `INSTALLED` before the retry budget ran out (`60 * 10` attempts + at 1s, about 10 minutes). A slow install reads exactly like a starved pool. + Read statuses and notebook duration before classifying it. +- `DatabricksCPUStreamingTests` exists only on the Spark 4 branches, and whether + it is scheduled varies by branch — read `pipeline.yaml` on the branch you are + working on rather than assuming. As measured on 2026-08-17, only live + `spark4.0` gives it a leg, and it got one in the original port commit + `b76c391be4`; `master` and `spark4.1` leave the class defined but unscheduled + pending pool capacity and a notebook fix. A sync from master therefore drops + that leg on `spark4.0`, which converges the three branches rather than + regressing one — but record it as a decision, because nothing reports it. It + is a separate class because the streaming notebook's `server.stop()` cancels + concurrent SparkContext jobs, so it needs its own cluster instead of a slot on + an existing leg, which is why scheduling it costs pool capacity. The in-repo + comment attributes that behaviour to Spark 4.0 and it has not been + re-confirmed on 4.1. If a sync drops the leg while leaving the class defined, + nothing fails to compile and nothing reports the gap, so check deliberately. +- The Databricks GPU suite was split and then deliberately re-merged, and the + history matters because `spark4.0` still carries the abandoned shape. #2538 + split it into `DatabricksGPUTests1/2/3`, each building its own cluster with two + workers and running exactly one notebook via `gpuNotebook(0)`, `(1)`, `(2)`. + #2573 (`fix: restore SynapseML Azure pipeline`) reverted that to a single + `DatabricksGPUTests` because the split could not fit: three clusters times two + workers needs six GPU nodes, against a pool holding + `GpuWorkersPerRun` 1 x `GpuConcurrentRuns` 3 = three. Master's current form + runs the whole `GPUNotebooks` set on one cluster sized at `GpuWorkersPerRun` + (one worker, so concurrent builds can share the pool), pins the driver to the + **CPU** pool (`driverInstancePoolId = Some(PoolId)`) so it does not consume a + GPU node, and rather than failing on a starved pool waits for one through + `createActiveCluster` with `maxAttempts = Int.MaxValue` and + `maxRetryDurationMs` of three hours. `SYNAPSEML_GPU_SMOKE_TESTS` passes + `synapseml_ci_smoke` through to the notebooks, and the job takes a 300-minute + timeout to absorb the sequential run. Read the file rather than this paragraph + for the mechanism: it changed between #2573 and now, and an earlier draft of + this bullet described the #2573 snapshot as if it were current. +- Prefer the consolidated form on every branch, and never restore the split + during a sync. Its indices are hardcoded, so it tests exactly three notebooks + no matter how many exist: at the time of writing `master` and `spark4.1` have + four GPU notebooks (`Fine-tune`/`Phi Model` matches) while live `spark4.0` has + three, so the split covers `spark4.0` today and would silently skip index 3 — + `Quickstart - End-to-end Local RAG with Phi Model` — the moment a sync brings + master's fourth notebook in. `DatabricksGPUTests` reads `GPUNotebooks` whole + and cannot drift that way. #2646 already lands exactly this: the merged + `DatabricksGPUTests.scala` is byte-identical to master's and the branch picks + up the fourth notebook, so `spark4.0` needs no separate change. +- Petastorm calls pyarrow APIs the pinned pyarrow no longer ships, so Horovod's + Spark backend needs a compatibility layer. Only `spark4.1` has one. This is a + library-version problem, not a Python-version one, so a branch on the same + pyarrow is not exempt. Deep-learning unit tests will not reveal the gap: + without a usable Horovod the estimators are stubbed and the Petastorm path + never runs. +- `/azp run` queues these targets. The ADO pull-request trigger filter allowed + only `master` until 2026-08-17; it now covers `master`, `spark3.5`, + `spark4.0` and `spark4.1`, verified by builds recording `reason=pullRequest` + and `requestedFor=GitHub` rather than `reason=manual`. Those two fields are + the reliable way to tell a trigger-driven run from one you queued by hand. If + a comment produces no build, re-read the + definition's trigger filter before assuming flakiness, and fall back to + queueing the PR merge ref (`refs/pull//merge`), never + `refs/heads/`, which fails service-connection authorization. +- **The trigger filter lives on the ADO definition, not in `pipeline.yaml`.** + The `pr:` block in `pipeline.yaml` is a red herring: a UI-defined trigger + overrides it silently, so editing the YAML changes nothing. The proof is on + the branch itself — `spark4.0`'s own `pipeline.yaml` `pr:` block lists + `master`, `spark3.3` and `spark3.5` and does **not** list `spark4.0`, yet PRs + targeting `spark4.0` build. Read the real value from the definition instead: + + ``` + GET .../_apis/build/definitions/17563?api-version=7.0 + ``` + + `triggers[].branchFilters` is currently `+master, +spark3.5, +spark4.0, + +spark4.1`, and the `continuousIntegration` trigger reports + `settingsSourceType: 2`, which means UI-defined rather than YAML-defined. + Consequence for a future release branch: adding it to `pipeline.yaml` does not + give it PR builds. Someone has to add it to the definition's filter. +- GitHub checks compile/lint but do not replace full Azure, Databricks, native, + R, or service validation. +- Intermittent ONNX OOM (`OutOfMemoryError` in `ImageFeaturizerSuite`, under the + `UnitTests onnx` leg) and R package HTTP failures + (a conda `HTTP 403` in `RTests vw`) require log evidence and a controlled + rerun; they are not automatic product regressions or exemptions. + +## Where the Java version is declared + +There is no single source of truth for the JDK. Each branch declares it in +several files, and a sync can silently disagree with itself if only some are +updated. Measured values: + +| File | master | spark4.0 | spark4.1 | +| --- | --- | --- | --- | +| `.github/workflows/pr-validation.yml` | 11 | **11** | 17 | +| `environment.yml` (`openjdk`) | absent | 17 | 17 | +| `environment.dev.yml` (`openjdk`) | no file | 17 | 17 | +| `templates/java_setup.yml` (`versionSpec`) | no file | 17 | 17 | +| `pipeline.yaml` (`JAVA_VERSION`, ReleaseBranchCompat) | 17 | absent | 17 | +| `tools/docker/*/Dockerfile` (`JAVA_HOME`) | 11 | 17 | 17 | + +Two things in that table are not typos. `spark4.0`'s GitHub workflow pins JDK +11 while the rest of the branch is on 17, so do not assume the workflow proves +the branch's Java version; check `environment.yml` or `java_setup.yml` instead. +And `spark4.0` has no `JAVA_VERSION` because it is not yet in the +ReleaseBranchCompat matrix, which is a separate follow-up. + +`templates/java_setup.yml` is the pin that CI jobs consume. On `spark4.0` it is +included by the `Style` job; on `spark4.1` the file exists but nothing includes +it yet. Master does **not** have the file at the time of writing: it arrives +with [#2652](https://github.com/microsoft/SynapseML/pull/2652), which sets it to +11 — master's already-effective JDK, measured from a build that echoes +`java -version` — and includes it from the InternalCompat job so that job stops +compiling Spark 4 code on master's JDK. If that PR has not merged yet, expect +the file to be absent on master and the table row above to read "no file"; +confirm with `git show ms/master:templates/java_setup.yml`. + +**Conflict rule: on the first sync after #2652 merges, `templates/java_setup.yml` +conflicts add/add and git leaves markers. Always keep the branch's own 17.** +Taking master's side is the intuitive resolution and the wrong one: it silently +drops the branch to Java 11 and reintroduces `Class java.lang.Record not found`. +The conflict is one-time — once resolved, the file histories are connected and +later syncs merge it cleanly. Verify with: + +``` +git show :templates/java_setup.yml | grep versionSpec +``` + +## Hand-written `__init__.py` files + +`PythonInitMerger` came from master and **preserves** hand-written `__init__.py` +content by splicing it after the generated imports. Codegen previously +overwrote these files, so their contents were inert; they are now live code in +the shipped package, which makes a stale one a real bug rather than dead text. +This is why the Spark 4 branches had to audit them. + +| Path | State | Why | +| --- | --- | --- | +| `core/.../io/http/__init__.py` | must stay empty | Listed free-function modules; see below | +| `vw/`, `services/openai/` | removed | Duplicated codegen output | +| `recommendation/`, `dl/`, `hf/`, `cognitive/`, `mmlspark/` | kept | Add exports codegen omits | + +`core/.../io/http/__init__.py` listed `HTTPFunctions` and `ServingFunctions`, +which are modules of free functions with no same-named class, so the import +failed and broke `PythonTests core` plus seven website-sample docs. The `vw/` +and `services/openai/` files also redefined `__all__`, which narrowed +`import *` to a hand-maintained list. + +Do not add new `__init__.py` files that re-list generated classes. On the Spark 4 +branches this is guarded by two tests, +`core/src/test/python/synapsemltest/io/http/test_http_package.py` and +`core/src/test/python/synapsemltest/recommendation/test_package_exports.py`. Note +where they are and are not: both are on `spark4.1`, both reach `spark4.0` through +[#2646](https://github.com/microsoft/SynapseML/pull/2646), and **neither is on +`master`**, which carries `PythonInitMerger` without them. So a change to these +files on `master` is unguarded, and the guards cannot be assumed from the merger's +presence. Verify with +`git ls-tree -r --name-only ms/ | grep -E 'test_http_package|test_package_exports'`. + +## Before merging a sync + +1. Recheck the target's live versions, pins, triggers, and skips. +2. Prove master content survived conflict resolution. +3. Run full Azure validation without a concurrent Spark 4 build. +4. Diff the sibling Spark 4 branch and explain every remaining difference. diff --git a/.github/skills/synapseml-branches/references/branch-spark4p0.md b/.github/skills/synapseml-branches/references/branch-spark4p0.md new file mode 100644 index 00000000000..d7db1879265 --- /dev/null +++ b/.github/skills/synapseml-branches/references/branch-spark4p0.md @@ -0,0 +1,125 @@ +# `spark4.0` + +Read [branch-spark4-common.md](branch-spark4-common.md) first. This is a condensed, +templatized version of the branch context from +[#2646](https://github.com/microsoft/SynapseML/pull/2646). + +## Purpose and baseline + +- Shared Spark 4.0 port. At the #2646 snapshot it used Spark 4.0.1, + Scala 2.13.16, Java 17, Python 3.12, and Databricks 17.3; verify live files. + The Databricks runtime strings are `17.3.x-scala2.13` and GPU + `17.3.x-gpu-ml-scala2.13`. DBR 17.3 LTS ML ships Spark 4.0 and 18.0 ML ships + Spark 4.1, which is why the runtime version is not a free knob here. +- Check `spark4.1` before debugging from scratch because it is the more actively + maintained descendant, then prove any candidate fix is not 4.1-specific. +- Live state lags the #2646 description. In `DatabricksUtilities.scala` the live + branch pairs its own GPU pool with a matching runtime -- + `GpuPoolName = "synapseml-build-17.3-gpu"` with + `AdbGpuRuntime = "17.3.x-gpu-ml-scala2.13"`, resolved by `getPoolIdByName` -- + and does not yet carry `OpenAIPromptPythonOverrides.scala`, the `__init__.py` + guard tests, or the `new_ml_pipeline_stage` R loading. #2646 brings those, and + also moves the GPU pool to master's shared `synapseml-build-14.3-gpu` + (resolved by `getPoolIdByNameAndNodeType`, which takes a node type and minimum + capacity) while keeping the 17.3 CPU pool and the 17.3.x GPU runtime. That + mixed pairing is not an oversight: `spark4.1` already runs master's + `14.3-gpu` pool against an `18.0.x-gpu-ml-scala2.13` runtime, so the pool name + identifies a warm node pool rather than a DBR version. Check the live branch + before assuming any of it. + +## Core differences + +- NumPy is currently unpinned here, as it is on 4.1. If you reintroduce a pin, + pin for a reason you can state: Python 3.12 has NumPy 1.26.4 wheels, and + pandas 2.0.3 is not compatible with the NumPy 2 ABI, so a pin is warranted + only while something in the resolved set actually requires it. +- This branch has `_horovod.py` but not `_petastorm_compat.py`, so it uses the + plain Horovod `SparkBackend` rather than 4.1's Petastorm-compatible subclass. + That gap is real rather than version-driven: the shim restores pyarrow APIs + Petastorm still calls, and this branch currently resolves to a *newer* pyarrow + than 4.1 does, so those APIs are at least as absent here. Confirm both pins + from `environment.yml` on each branch before reasoning about it. See + [branch-spark4-common.md](branch-spark4-common.md). +- `LongOffset` remains under `...execution.streaming`, not `.runtime`. +- Spark 4.0 returns `bytearray` for Python `BinaryType`; it does not require the + 4.1 `np.frombuffer` workaround. +- Preserve the Spark 4 R fixes shared with 4.1. The branch-local `JAVA_HOME` + fallback is extra; nested-stage loading was alignment, not proven root cause. +- **sparklyr must be 1.9.5, not 1.9.3** (`r-sparklyr=1.9.5` in `environment.yml`), + and on the live branch it is still + 1.9.3 — so this is a current failure, not history. Under dbplyr 2.6, sparklyr + 1.9.3's `tidyselect_data_proxy.tbl_spark` returns a proxy carrying no Spark + connection, so anything routed through `dplyr::select` on a `tbl_spark` loses + `sc`. It broke `RTests core` (21 of 69) and `RTests deep-learning` (3 of 3), + and it surfaces one or two layers away as `invoke_static` or `hive_context` + applied to `NULL`, which reads like a dead Spark session. Interleaving is the + tell: a dead session fails everything after a point, whereas this failed 21 + tests scattered among 48 passes, with `sar` passing while `sar_model` failed. + Read the backtrace, not the surface error. `spark4.1` pairs the same + `r-base=4.4` with 1.9.5 and passes 69/69. +- R connects through `SPARK_HOME`: `RTestGen.scala` generates + `spark_connect(master = "local", spark_home = Sys.getenv("SPARK_HOME"), ...)`, + byte-identical to `spark4.1`. The pipeline exports `SPARK_HOME`, so + `run_r_tests.R` only unsets it and installs the tarball when it is absent, + which is the local-developer path. Be accurate about what that bought: the + previous `version = "4.0"` form also worked, because `run_r_tests.R` had + already installed the tarball and sparklyr resolves an install it made itself. + Measured R results were identical before and after. It is kept for + byte-identical alignment with 4.1, not because it fixed anything. + +## Runtime and CI + +- Fabric E2E remains disabled because Fabric has no managed Spark 4.0 runtime — + Fabric Runtime 2.0 went GA on Spark 4.1. This is real lost coverage rather + than a cosmetic skip, and it should stay disabled here until a Spark + 4.0-capable Fabric runtime exists, which may never happen; the more likely + resolution is that this branch is superseded by `spark4.1`. +- At #2646, two GPU fine-tune notebooks failed because no Horovod wheel matched + DBR 17.3's PyTorch. The wheel this branch needs is one built against DBR 17.3 + ML's PyTorch, and no such wheel is published — the `synapse-extension` wheel is + built for 18.0 ML. Producing it is a build-artifact task rather than a code + change, which is why this is recorded rather than patched around. `spark4.1` + additionally calls `ensure_petastorm_compatibility()` *before* `import horovod` + in `_horovod.py`; that is a plausible second contributor but it is unproven, so + do not quote it as the cause without the notebook's stderr from the Databricks + run API. Do not switch `AdbGpuRuntime` to DBR 18 merely to turn them green; + that would test Spark 4.1 instead of this branch and make the suite green by no + longer testing what it exists to test. Revalidate this known gap. +- Two of four GPU notebooks failing is that gap's expected shape. Check the + failing count and which notebooks, not the job's red/green, before calling it + a regression. Note the denominator moves: live `spark4.0` has three GPU + notebooks and #2646 brings master's fourth + (`Quickstart - End-to-end Local RAG with Phi Model`), because the sync also + adopts master's consolidated `DatabricksGPUTests`, which runs the whole + `GPUNotebooks` set instead of three hardcoded indices. The Horovod wheel is + the first blocker and it masks the missing Petastorm layer noted above, so + fixing the wheel alone should not be expected to turn these notebooks green. + Confirm each step from the notebook's stderr output rather than inferring it. +- Avoid pinning runtime-provided torch/torchvision without a demonstrated need; + incompatible pins can trigger multi-gigabyte CUDA downgrades and timeouts. The + recorded instance was `torchvision==0.17.0` in `GPULibraries`, which + hard-requires `torch==2.2.0`: pip had to *downgrade* the runtime's much newer + torch and pull large CUDA wheels, slow enough to exhaust the install budget but + never reporting `FAILED`. The GPU ML runtime already ships both, so the pin + bought nothing. +- When the fine-tune notebooks were investigated, the notebooks and + `GPULibraries` were byte-identical to `spark4.1` and still failed here while + passing there, and swapping in `spark4.1`'s sha256-pinned wheel changed nothing + measurable (71.6s to 60.6s, 56.2s to 47.0s — the same failure in the same + window). Treat that swap as alignment with the working branch, not a fix. +- A sub-minute GPU notebook failure occurs during dependency setup, before + training. Use run timing and stderr rather than attributing it to the model. +- Confirm target-branch automation actually queued rather than assuming the + comment was enough; see + [branch-spark4-common.md](branch-spark4-common.md) for how to tell a + trigger-driven build from a hand-queued one. + +## Do not port from `spark4.1` + +- 4.1 `LongOffset` import, BinaryType `np.frombuffer` workaround, Python 3.13 + wheels, or version strings. +- Fabric Runtime 2.0 enablement. +- Any runtime/dependency change whose only evidence is a green 4.1 build. +- The Petastorm compatibility layer is not on this list. It is a back-port + candidate, not a 4.1-only change, but it needs validation on real 4.0 rather + than adoption on suspicion. diff --git a/.github/skills/synapseml-branches/references/branch-spark4p1.md b/.github/skills/synapseml-branches/references/branch-spark4p1.md new file mode 100644 index 00000000000..1b17c63a9ac --- /dev/null +++ b/.github/skills/synapseml-branches/references/branch-spark4p1.md @@ -0,0 +1,69 @@ +# `spark4.1` + +Read [branch-spark4-common.md](branch-spark4-common.md) first. This is a condensed, +templatized version of the branch context from +[#2645](https://github.com/microsoft/SynapseML/pull/2645). + +## Purpose and baseline + +- Shared Spark 4.1 port and the more actively maintained Spark 4 reference. At + the #2645 snapshot it used Spark 4.1.1, Scala 2.13.17, Java 17, Python 3.13, + and Databricks 18.0; verify live files. The Databricks runtime strings are + `18.0.x-scala2.13` and GPU `18.0.x-gpu-ml-scala2.13`. +- Ask whether every non-4.1-specific fix should be back-ported to `spark4.0`. + +## Core differences + +- Python 3.13 requires newer wheels and an intentionally unpinned NumPy. + Preserve explanatory pin comments through syncs. +- The Petastorm/Horovod compatibility layer is separate from that: it restores + pyarrow APIs Petastorm still calls, which the pinned pyarrow no longer + provides. Do not describe it as a Python 3.13 workaround — that framing makes + `spark4.0` look exempt when it is not. `spark4.0` pins a different, newer + pyarrow, so those APIs are missing there too. The layer has two halves: + `_petastorm_compat.py` and the `_serialize_petastorm_compatibility()` path in + `_horovod.py`. A back-port needs both. +- `LongOffset` moved to `...execution.streaming.runtime`; the 4.0 import does + not compile here. `HTTPSource.scala` and `DistributedHTTPSource.scala` are the + files that import it, and they are the whole surface of this difference. +- Spark 4.1 returns Python `bytes` for `BinaryType` where 4.0 returns + `bytearray`; `np.asarray(value, dtype=np.uint8)` raises `ValueError` on `bytes` + because it treats them as a scalar string, so `ImageTransformer.toNDArray` + uses `np.frombuffer`, which accepts both. +- `RCodegenSuite` directly guards generated R behavior, including nested-stage + loading and the Spark 4 ANSI settings. + +## Runtime and CI + +- Fabric Runtime 2.0 supports Spark 4.1, so the old "unsupported runtime" + reason for disabling Fabric E2E is stale. On this branch's `pipeline.yaml` the + job is switched off with a bare `condition: false`, so it is skipped rather + than reported. Do not check `master` to confirm that, because `master` has + the normal `and(succeeded(), eq('${{ parameters.testFabricE2E }}', true))` and + `spark4.0` a third form, `eq('${{ parameters.testFabricE2E }}', true)` with no + `succeeded()`. Re-enable only in a dedicated PR, where the pipeline run *is* + the test, rather than folding it into a sync: a Fabric provisioning failure + would otherwise block an unrelated merge. That PR should restore `master`'s + form, drop the stale comment, request Spark 4.1 in workspace creation, and + validate against real Fabric capacity and service connection. The + workspace-creation payload lives in the Fabric test package's + `FabricOperations.scala`, which hardcodes `'SparkVersion': '3.5'` and must + request `'4.1'`. That value is + hardcoded identically on `master`, `spark4.0` and `spark4.1`, so changing it + here does not alter master's behaviour. It also needs a Fabric capacity in the + `sempy-integration-region` that can provision Runtime 2.0 workspaces, and the + `SynapseML Build` service connection, which are the prerequisites not + discoverable from the code. +- Databricks CPU/GPU validation uses 18.x-era runtimes: CPU pool + `synapseml-build-18.0`, GPU pool `synapseml-build-14.3-gpu`. Run Spark 4 builds + sequentially because the GPU pool is shared with `master` and `spark4.0`. +- `DatabricksCPUStreamingTests` was unscheduled pending capacity and notebook + work; verify rather than silently accepting the omission. +- Master compatibility replay commonly applies release-relevant patches here + and runs `test:compile`; it is not full branch validation. + +## Do not port to `spark4.0` + +- 4.1 `LongOffset` import, BinaryType workaround, Python 3.13 shims, unpinned + NumPy, Fabric 4.1 enablement, or version/runtime strings. +- Treat other changes as back-port candidates and validate them on real 4.0. diff --git a/.github/skills/synapseml-branches/references/branch-template.md b/.github/skills/synapseml-branches/references/branch-template.md new file mode 100644 index 00000000000..691ddc7c5ac --- /dev/null +++ b/.github/skills/synapseml-branches/references/branch-template.md @@ -0,0 +1,18 @@ +# Branch reference template + +Keep each branch file concise and use these headings: + +1. **Purpose and baseline sources** — what targets the branch and which live + files define versions. +2. **Sync policy** — merge/rebase direction and conflict rules. +3. **Differences from master** — only deliberate runtime or tooling deltas. +4. **Sibling-port rules** — what should and must not move between branches. +5. **Runtime and CI** — supported environments, intentional skips, triggering, + capacity, and required real-environment validation. +6. **Known failures** — evidence-backed current exceptions, with a reminder to + revalidate rather than normalize them forever. +7. **Before merge** — content comparison, target refresh, tests, and sibling + branch diff. + +Link shared material instead of copying it. Treat versions and known failures +as snapshots; verify them against the live target branch. diff --git a/.github/skills/synapseml-pr-loop/SKILL.md b/.github/skills/synapseml-pr-loop/SKILL.md new file mode 100644 index 00000000000..ca803d5e71a --- /dev/null +++ b/.github/skills/synapseml-pr-loop/SKILL.md @@ -0,0 +1,166 @@ +--- +name: synapseml-pr-loop +description: >- + Make one or more SynapseML issues or pull requests evidence-based merge-ready. + Use for "5/5 confidence", "200% ready", stale/outdated PR remediation, + rebase-and-test requests, resolving all review comments, or proving a feature + ships without correctness, compatibility, performance, or Spark regressions. +compatibility: >- + SynapseML repository with git, GitHub CLI, PowerShell, WSL/Linux, sbt, Python, + and network access to GitHub/Azure Pipelines. +--- + +# SynapseML PR loop + +Treat "5/5" or "200%" as an evidence standard, never a literal guarantee. +The exit condition is: the requested value is proven through the public API, +the current target is integrated, review is exhausted, and every required check +is complete and green. + +## Workflow + +### 1. Establish scope and isolation + +- Load the [branch context skill](../synapseml-branches/SKILL.md) using the PR + base branch. Recheck it before validation and immediately before final push. +- Read the issue, PR body, linked work items, commit history, changed files, + and every review thread/body, including resolved, outdated, minimized, and + suppressed comments. Verify prior resolutions rather than trusting status. +- Inspect formal review decisions, requested-change votes, ownership gates, and + coverage thresholds; resolved threads do not clear those blockers. +- Check recently merged/closed related PRs and issues. Identify follow-up PRs + needing rebase/remediation, superseded work to close, and remaining issue + action items; do not assume closure completed the feature lifecycle. +- Give each PR a dedicated worktree and branch. Parallelize independent PRs, + but identify overlapping files and required merge order first. +- Run + [scripts/Get-PrReadiness.ps1](scripts/Get-PrReadiness.ps1) + with `-PullRequest ` and retain its JSON locally as the initial + snapshot. It can contain review text; redact it before public sharing. + +### 2. Integrate the current target + +- Fetch the PR's target branch and rebase an ordinary PR before validation. +- Use `--force-with-lease`, never an unguarded force push. +- Merge, rather than rebase, shared `spark` port branches. +- Record target SHA, head SHA, merge base, ahead/behind counts, and conflicts. +- Compare the intended patch before and after rebase/conflict resolution. +- Fetch again immediately before the final push. If the target advanced, + integrate it and rerun affected validation. + +### 3. Define the value and regression contract + +- Keep the PR title and description aligned with the current scope. Lead with a + short human-readable change/value summary; put detailed design and validation + evidence afterward. Refresh both after material changes. +- State the user-visible bug or feature, supported/unsupported cases, default + behavior, compatibility contract, and measurable acceptance criteria. +- Trace the real public path: Scala stage, generated/hand-written Python, + schema, serialization, persistence, service/native boundary, and packaging. +- Confirm the published package actually contains the capability; local jars, + custom natives, or provider discovery do not prove that users receive it. +- Establish a baseline when failures, performance, or external systems are + involved. A passing new test is insufficient if the old behavior was never + shown to fail. + +### 4. Review and implement + +- Apply the [code-review skill](../code-review/SKILL.md). +- Resolve root causes, not only the reported line. Recheck sibling APIs and + language surfaces that share the same serializer, schema, parameter, or + native/service path. +- Preserve public JVM and serialized compatibility unless explicitly approved. +- Update user-facing documentation/examples for changed public behavior. Edit + Scala sources rather than generated files under `target/`. +- Follow the Spark and performance gates in + [references/spark-performance.md](references/spark-performance.md). +- Reply in the existing thread with the fix and evidence, then resolve it. +- Re-audit after every push. Automated review is asynchronous and re-runs per + commit, so auditing immediately after pushing reads the *previous* review and + reports a false all-clear. Wait until the newest automated review's commit + equals the pushed head, then audit; poll rather than checking once. +- Suppressed comments are not review threads. They appear only inside a + collapsed section of the review body, so a `reviewThreads` query returns zero + while they exist, and they have no thread to reply to or resolve. Read every + automated review body for the current head, and address them in the follow-up + commit message or a PR comment. Treat them as ordinary findings: they are + suppressed for confidence, not for correctness. + +### 5. Add proof-oriented tests + +- Add a regression that fails before the fix and passes after it. +- Cover positive, negative, null/empty, boundary, schema, copy, save/load, and + Python/codegen behavior as applicable. +- Exercise the public transformer/estimator or request path end to end; helper + tests alone do not prove the feature ships. +- Use real hardware, native libraries, clusters, network families, or services + when the claim depends on them. Do not infer capability from configuration or + provider discovery alone. +- Before external service tests, audit resource creation/deletion and use only + authorized test resources. + +### 6. Validate locally and across branches + +- Use the [local setup skill](../synapseml-local-setup/SKILL.md) and its JDK + wrapper. +- Run the smallest targeted suites, compile, test compile, Scala style, pinned + Black, codegen, generated-wrapper checks, and relevant Python tests. +- Run release compatibility for every port branch affected by the change. +- Benchmark representative scale before/after when a hot path, network path, + accelerator, allocation pattern, or algorithmic complexity changes. + +### 7. Run and triage full CI + +- Push the exact validated head, comment `/azp run`, then confirm a build + actually queued -- a comment is not evidence that CI ran, so cite the build + ID. A trigger-driven build records `reason=pullRequest`; one you queued + yourself records `reason=manual`, which is the quickest way to tell whether + the trigger really fired or you merely re-ran it by hand. +- Do this after **every** push, not once per pull request. The build does not + re-queue itself when the head moves, so the previous run's result belongs to + code that no longer exists. The GitHub Actions checks do re-run on each push + and go green within a couple of minutes, which makes a head with no Azure + Pipelines build on it look fully checked; an absent check is neither failed + nor pending, so nothing reports it. Verify the build against the head SHA by + name, or run `Get-PrReadiness.ps1 -RunPipeline` to post the comment + automatically when it is missing. +- If no build appears, check the pipeline definition's own pull-request trigger + rather than assuming a transient failure. That trigger can be defined in the + pipeline UI, in which case it overrides the `pr:` block in `pipeline.yaml` + entirely and silently ignores targets the YAML lists. Read its branch filters + through the definitions API. Until the filter is corrected, queue explicitly + against `refs/pull//merge` -- never `refs/heads/`, which + validates the branch instead of the merge result. +- Inspect every failed, canceled, skipped, and pending job. Use + [references/ci-triage.md](references/ci-triage.md) to separate product + defects, test defects, baseline failures, and infrastructure failures. +- Fix product/test defects and rerun. Infrastructure classification requires + logs proving tests did not exercise the change; "looks flaky" is not evidence. +- If path filters or a CI-only diff bypass the behavior being repaired, validate + it with a representative product change or controlled integration PR. +- Do not declare readiness while any required check is pending. + +### 8. Final readiness loop + +Run `Get-PrReadiness.ps1 -PullRequest -WaitForReview -RunPipeline` +after the final push and confirm every gate in +[references/readiness-gates.md](references/readiness-gates.md). Those two +switches cover the asynchronous gaps that a bare snapshot reports as clean: the +automated review has not arrived yet, and the Azure Pipelines build has not been +asked to start. Both leave the same signature -- nothing failed, nothing +pending, nothing there. + +For multiple PRs, after each merge: + +1. fetch the new target; +2. rebase overlapping downstream PRs; +3. rerun targeted, compatibility, and full CI; +4. re-audit review threads and suppressed comments. + +After any merge or closure, reconcile linked work: update or close fulfilled +issues, close superseded PRs with an explanation, and rebase/remediate still +valuable follow-ups. Preserve separate unresolved scope rather than closing it +for convenience. + +Report the exact remaining blocker. "Only human approval remains" is valid only +when all engineering gates are complete. diff --git a/.github/skills/synapseml-pr-loop/references/ci-triage.md b/.github/skills/synapseml-pr-loop/references/ci-triage.md new file mode 100644 index 00000000000..d290e8f61f2 --- /dev/null +++ b/.github/skills/synapseml-pr-loop/references/ci-triage.md @@ -0,0 +1,75 @@ +# CI triage + +Do not rerun a failed pipeline blindly. Preserve the job URL and first determine +which category the failure belongs to. + +## Product defect + +The changed code compiled or ran and produced an incorrect result, crash, +resource leak, performance regression, or incompatible API/schema. + +Action: reproduce locally or in the closest environment, add/strengthen the +regression test, fix, and rerun targeted plus full CI. + +## Test defect + +The product behavior is correct but the test has a race, wrong assumption, +unsafe cleanup, overly strict tolerance, environment-order dependence, or does +not test the public path. + +Action: fix the test without weakening the requirement. Demonstrate the product +behavior separately. + +## Baseline/pre-existing failure + +The same failure occurs on the target SHA or is unrelated to every changed path. + +Action: collect comparable target/head evidence. Do not silently ignore it; link +the tracking issue or repair it when tightly coupled. + +## Infrastructure failure + +Repository setup, capacity allocation, authentication, TLS, artifact download, +agent loss, or publishing failed before relevant tests ran. + +Action: cite the log line proving where execution stopped, verify no test result +was produced, and rerun. Repeated infrastructure failures still block readiness +when they prevent required evidence. + +## Reading job results correctly + +Azure Pipelines job results are not binary. A job can end as `succeeded`, +`succeededWithIssues`, `failed`, `canceled`, or `skipped`, and a triage filter +that accepts only `succeeded` will report phantom failures. + +`succeededWithIssues` most often comes from a non-gating task -- dependency +cache upload/download, TLS errors, artifact publishing -- while every test in +the job passed. Confirm by opening the job and finding which task raised the +warning, then read the published test results rather than trusting the job +badge in either direction: + +- If the warning is from a non-gating task and the test run is complete and + green, the job passed. Do not rerun it. +- If the warning is from a task that runs or publishes tests, treat it as a + real failure until the test counts prove otherwise. + +Job-level status also cannot tell you whether the tests you care about ran. +For any claim about a specific suite, read the per-test results from the test +run the job published, and compare them against a prior build. Comparing +per-test outcomes across builds is the only reliable way to tell a real fix +from a coincidence: a fix that changes nothing will leave the same tests +failing in the same way, which a green/red job summary will not reveal. + +## False-green patterns to reject + +- A job succeeded because the affected tests were skipped. +- The relevant suite was never selected by the matrix and therefore was not + reported as skipped. +- A helper test passed while the transformer/request path remained broken. +- Provider/device discovery succeeded without executing real kernels. +- A custom native or local jar worked although the published artifact lacks it. +- Aggregate CI is green while a required branch replay never ran. +- A CI/path-filter fix passed because its own diff bypassed the path it changed; + no representative product patch exercised the workflow. +- A test count increased but the requested edge case has no assertion. +- Commit ancestry is correct but a merge conflict discarded target content. diff --git a/.github/skills/synapseml-pr-loop/references/readiness-gates.md b/.github/skills/synapseml-pr-loop/references/readiness-gates.md new file mode 100644 index 00000000000..c5ecd26b1a5 --- /dev/null +++ b/.github/skills/synapseml-pr-loop/references/readiness-gates.md @@ -0,0 +1,102 @@ +# Merge-readiness gates + +A SynapseML PR is engineering-ready only when every applicable gate is backed +by current-head evidence. + +## Integration + +- Matching branch context was checked at start, before validation, and before + final push against the target branch's live build and CI files. +- Head is based on the latest target SHA and is not behind. +- Conflicts are resolved by combining independent changes, not choosing a side + wholesale. +- The intended patch remains equivalent after rebase/conflict resolution. +- For overlapping PRs, merge order is explicit and downstream PRs are + revalidated after predecessors merge. +- Local, remote, and GitHub head SHAs match. +- Recently closed/merged related work was checked; valuable follow-ups are + rebased and revalidated, while superseded PRs are closed with an explanation. + +## User value and scope + +- The title and opening description accurately explain the current change and + user value to a human reader; deeper technical evidence follows afterward. +- The original issue and every material discussion point are addressed. +- The behavior is reachable through the published artifact and public API. +- Defaults remain backward compatible, or the intentional change is documented. +- Unsupported cases fail early with actionable errors; no success-shaped + fallback hides missing capability. +- Documentation describes what is shipped, not a custom validation artifact or + an unbundled native/provider. +- Fulfilled linked issues are updated/closed, while distinct unresolved scope + remains open and explicit. + +## Correctness and compatibility + +- Runtime output matches `transformSchema`. +- `copy`, save/load, Params, generated Python, hand-written Python, and other + language bindings preserve behavior. +- Public JVM signatures and serialized shapes pass compatibility review. +- Shared serializers, request models, evaluators, and sibling API variants are + checked for the same defect. +- Native/service behavior is verified past the wrapper boundary. + +## Test quality + +- A regression test demonstrates the old failure. +- Public end-to-end behavior is tested, not only helpers. +- Positive, negative, null/empty, malformed, boundary, and cleanup paths are + covered where relevant. +- Tests assert values, schemas, ordering, resource cleanup, and errors rather + than only "no exception". +- Generated wrappers compile/import, and Python tests exercise the supported + surface when user-facing APIs change. +- Real environment tests exist when emulation cannot prove the claim. + +## Performance and Spark + +- No driver collection, accidental cross join, unbounded materialization, + per-row client/model construction, or unnecessary repartition/shuffle is + introduced. +- DataFrame/Dataset and Spark SQL built-ins are preferred over RDDs and UDFs. +- Cache/persist/broadcast/native resources have bounded lifetimes and cleanup. +- Network/native concurrency has bounds, backpressure, timeout, retry, and + terminal failure behavior. +- Representative before/after measurements show no material regression for a + changed hot path. + +## Review and validation + +- Active review threads: zero. +- No blocking review decision, requested-change vote, ownership gate, or + required coverage failure remains. +- Suppressed/minimized Copilot feedback was read and either fixed or rebutted + with evidence. Read it from the review body for the current head; it never + appears as a review thread, so a zero-thread query does not clear this gate. +- Latest automated review covers the final head, compared by commit rather than + by recency. A review produced before the last push does not clear the two + gates above, because it never saw that code. +- Targeted tests, compile, test compile, style, Black, codegen, Python, and + port-branch compatibility pass as applicable. +- Full Azure Pipelines and required GitHub checks are complete with zero + unexplained failures or pending jobs. +- The Azure Pipelines build is present on the current head at all. It does not + queue itself on a push here, so every push needs its own `/azp run`; a head + that never got one carries only the GitHub Actions checks, and those going + green is not CI passing. An absent check is neither failed nor pending, so it + is invisible to both of those gates -- confirm the build by name against the + head SHA, not by the absence of red. +- Skips are expected and documented; a skipped required scenario is a blocker. +- `Get-PrReadiness.ps1` reports these as `completeness.complete`, which is true + only when comment pagination was not truncated, an automated review covers the + head, and unresolved threads, suppressed-for-head items, missing required + checks, failed checks and pending checks are all zero. Treat a pending check as + unknown rather than passing. Trust the individual fields over the summary when + they disagree: that flag has been wrong before, in both directions. + +## Honest confidence language + +Use "5/5 engineering confidence" only with the evidence above. State residual +risk explicitly: external service variability, unowned infrastructure, hardware +not available for validation, or required human approval. Never claim absolute +certainty. diff --git a/.github/skills/synapseml-pr-loop/references/spark-performance.md b/.github/skills/synapseml-pr-loop/references/spark-performance.md new file mode 100644 index 00000000000..e041e018a5b --- /dev/null +++ b/.github/skills/synapseml-pr-loop/references/spark-performance.md @@ -0,0 +1,46 @@ +# Spark and performance review + +Apply these checks to every changed execution path. + +## Data movement + +- Prefer DataFrame/Dataset and Spark SQL expressions. +- Avoid `collect`, `toLocalIterator`, driver-side aggregation, RDD conversion, + accidental cartesian products, and unbounded arrays/maps. +- Check partition count, repartition/coalesce choices, shuffles, sorts, joins, + and broadcast size. +- Preserve streaming/lazy execution; do not eagerly scan data merely to + validate a parameter when schema/metadata can answer it. + +## Per-row and per-partition work + +- Construct clients, models, sessions, parsers, and native handles once per + partition or task where safe, not once per row. +- Bound concurrency and queues. Add backpressure, timeouts, cancellation, and + terminal failure propagation. +- Batch remote/native work when the API supports it. +- Avoid repeated serialization, parsing, schema inference, or metric + recomputation. + +## Memory and resources + +- Close streams, sessions, responses, sockets, native handles, and thread pools + on success and failure. +- Unpersist cached RDD/DataFrame or metric intermediates. +- Use bounded buffers and avoid retaining complete partitions or responses. +- Test cleanup, half-close/cancellation, and retry exhaustion. + +## Measurement + +Benchmark the old and new head with the same data, cluster/runtime, warm-up, and +iteration count. Report: + +- dataset dimensions and partitioning; +- hardware/runtime and dependency/native artifact; +- cold and warm timing; +- throughput or latency distribution; +- memory/spill/shuffle when relevant; +- correctness parity. + +Small synthetic tests prove logic, not performance. Use representative scale +and real accelerator/network/service paths for performance claims. diff --git a/.github/skills/synapseml-pr-loop/scripts/Get-PrReadiness.ps1 b/.github/skills/synapseml-pr-loop/scripts/Get-PrReadiness.ps1 new file mode 100644 index 00000000000..2f9d02462fd --- /dev/null +++ b/.github/skills/synapseml-pr-loop/scripts/Get-PrReadiness.ps1 @@ -0,0 +1,448 @@ +<# +.SYNOPSIS + Captures GitHub merge-readiness evidence for one or more pull requests. +.DESCRIPTION + Reports head/base state, target divergence, checks, active review threads, + and review bodies containing suppressed comments. Review threads and reviews + are fully paginated, and the emitted `completeness` object reports page + counts plus any thread whose comments were truncated, so an incomplete + snapshot is visible rather than silent. + + Automated review is asynchronous, so a snapshot taken right after a push + describes the previous head and can report zero findings for code nobody has + reviewed yet. `automatedReviewCoversHead` compares the newest automated + review's commit with the current head and gates `complete`, and + `suppressedReviewBodiesForHead` narrows suppressed feedback to that head. + Poll until it is true rather than trusting a single clean snapshot. + + A pull request that was never reviewed at all reports the same zero findings, + so `-RequestReview` asks for one as a fallback. Review is normally triggered + automatically, so the usual need is not to ask but to wait: `-WaitForReview` + blocks until the review of the current head arrives, instead of returning a + premature all-clear and leaving a human to notice the comments later. + Suppressed comments are reported too: they live inside the review body rather + than as review threads, so a thread query alone never surfaces them. + + Checks are reported by absence as well as outcome. `failedChecks` and + `pendingChecks` can only describe checks that exist, so a head whose build + never started scores zero in both and reads as finished; + `missingRequiredChecks` catches that and gates `complete`. The Azure DevOps + build is the usual casualty because it does not queue itself on a push -- it + waits for an `/azp run` comment -- so `-RunPipeline` posts one. + + It does not make the readiness decision; use the skill's evidence gates for + that judgment. Output can contain review content; keep it local or redact it + before sharing. +#> +param( + [Parameter(Mandatory)] + [int[]]$PullRequest, + + [string]$Repo = "microsoft/SynapseML", + + # Logins whose reviews count as automated coverage of the head commit. Matched + # exactly, with an optional "[bot]" suffix, so a human account that merely + # contains one of these words is never mistaken for the reviewer. Defaults to + # this repo's reviewer bot only: shorter generic logins such as "copilot" are + # registerable by humans, and an exact match on one of those would count a + # human review as automated coverage. Pass -AutomatedReviewer to extend. + [string[]]$AutomatedReviewer = @("copilot-pull-request-reviewer"), + + # Block until the automated review of the current head arrives. Review is + # triggered automatically on push but lands afterwards, so a snapshot taken + # immediately reports zero findings for code nobody has reviewed yet. Waiting + # here keeps that from being mistaken for a clean result. + [switch]$WaitForReview, + + [ValidateRange(1, 240)] + [int]$TimeoutMinutes = 20, + + [ValidateRange(5, 600)] + [int]$PollSeconds = 60, + + # Fallback only. Review is normally triggered automatically; use this when a + # pull request has somehow never been reviewed at all, which reports the same + # zero findings as a clean review. + [switch]$RequestReview, + + # Handle used when requesting that review. This is the requestable handle, not + # the login the submitted review is attributed to. + [string]$ReviewerHandle = "Copilot", + + # Checks that must be PRESENT on the head commit, matched as a name prefix so + # one entry covers a pipeline's several legs. Absence is the point: the + # failed/pending sets can only describe checks that exist, so a head whose CI + # never started scores zero failures and zero pending and looks finished. The + # Azure DevOps build does not queue itself on every push here -- it needs an + # `/azp run` comment -- which is exactly the check most likely to be missing. + [string[]]$RequiredCheck = @("microsoft.SynapseML"), + + # Post `/azp run` when a required check is missing from the head, instead of + # leaving a human to notice that full CI never started. + [switch]$RunPipeline +) + +$ErrorActionPreference = "Stop" + +if (-not (Get-Command gh -ErrorAction SilentlyContinue)) { + throw "GitHub CLI 'gh' is required." +} + +$repoParts = $Repo.Split("/") +if ($repoParts.Count -ne 2 -or -not $repoParts[0] -or -not $repoParts[1]) { + throw "Repo must use owner/name format; got '$Repo'." +} +$owner = $repoParts[0] +$name = $repoParts[1] + +$automatedLogins = @($AutomatedReviewer | + Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | + ForEach-Object { ($_.Trim() -replace '\[bot\]$', '').ToLowerInvariant() } | + Where-Object { $_ }) +if (-not $automatedLogins) { + throw "AutomatedReviewer must contain at least one non-blank login." +} + +# Review requests are attempted at most once per PR per invocation; see the +# -RequestReview block in Get-PrSnapshot. +$script:reviewRequestOutcome = @{} + +# Likewise for `/azp run` comments; see the -RunPipeline block in Get-PrSnapshot. +$script:pipelineRunOutcome = @{} + +$threadQuery = @' +query($owner: String!, $name: String!, $number: Int!, $cursor: String) { + repository(owner: $owner, name: $name) { + pullRequest(number: $number) { + reviewThreads(first: 100, after: $cursor) { + pageInfo { hasNextPage endCursor } + nodes { + id + isResolved + isOutdated + path + line + comments(first: 100) { + pageInfo { hasNextPage } + nodes { + author { login } + body + url + } + } + } + } + } + } +} +'@ + +$reviewQuery = @' +query($owner: String!, $name: String!, $number: Int!, $cursor: String) { + repository(owner: $owner, name: $name) { + pullRequest(number: $number) { + reviews(first: 100, after: $cursor) { + pageInfo { hasNextPage endCursor } + nodes { + submittedAt + body + commit { oid } + author { login } + } + } + } + } +} +'@ + +function Invoke-PagedQuery { + param( + [Parameter(Mandatory)][string]$Query, + [Parameter(Mandatory)][string]$Owner, + [Parameter(Mandatory)][string]$Name, + [Parameter(Mandatory)][int]$Number, + [Parameter(Mandatory)][scriptblock]$Select, + [Parameter(Mandatory)][string]$Description + ) + + # GraphQL responses are sparse: absent fields are simply missing rather than + # null, and this function reads them positionally. Under StrictMode that is a + # terminating error, so a caller with StrictMode enabled would get no snapshot + # at all instead of a snapshot reporting what is missing. Scoped to this + # function so dot-sourcing the script cannot disable StrictMode session-wide. + Set-StrictMode -Off + + $nodes = @() + $cursor = $null + $pages = 0 + + do { + $requestCursor = $cursor + $ghArgs = @("api", "graphql", "-f", "query=$Query", + "-F", "owner=$Owner", "-F", "name=$Name", "-F", "number=$Number") + if ($cursor) { $ghArgs += @("-F", "cursor=$cursor") } + + $text = & gh @ghArgs + if ($LASTEXITCODE -ne 0) { + throw "$Description failed for PR #$Number" + } + + $response = $text | ConvertFrom-Json + if ($response.PSObject.Properties['errors'] -and $response.errors) { + $messages = @($response.errors | ForEach-Object { $_.message }) -join "; " + throw "$Description returned GraphQL errors for PR #${Number}: $messages" + } + $pullRequest = $response.data.repository.pullRequest + if (-not $pullRequest) { + throw "$Description returned no pull request data for PR #$Number" + } + $connection = & $Select $pullRequest + if (-not $connection -or -not $connection.pageInfo) { + throw "$Description returned an incomplete connection for PR #$Number" + } + $nodes += @($connection.nodes) + $cursor = $connection.pageInfo.endCursor + $hasNext = $connection.pageInfo.hasNextPage + $pages++ + + # Guard against a cursor that never advances rather than looping forever. + if ($hasNext -and (-not $cursor -or $cursor -eq $requestCursor)) { + throw "$Description reported more pages without advancing the cursor for PR #$Number" + } + } while ($hasNext) + + [pscustomobject]@{ nodes = $nodes; pages = $pages } +} + +function Get-PrSnapshot { + param([int]$number) + + # Same reason as Invoke-PagedQuery: sparse GraphQL fields are read + # positionally here, and StrictMode is scoped to this function so + # dot-sourcing cannot change the caller's session. + Set-StrictMode -Off + + $jsonFields = "number,title,state,isDraft,mergeable,mergeStateStatus,reviewDecision," + + "headRefOid,baseRefName,statusCheckRollup,url" + $viewText = & gh pr view $number --repo $Repo --json $jsonFields + if ($LASTEXITCODE -ne 0) { + throw "gh pr view failed for PR #$number" + } + $view = $viewText | ConvertFrom-Json + + $threadPage = Invoke-PagedQuery -Query $threadQuery -Owner $owner -Name $name ` + -Number $number -Description "GraphQL review-thread query" ` + -Select { param($pr) $pr.reviewThreads } + + $reviewPage = Invoke-PagedQuery -Query $reviewQuery -Owner $owner -Name $name ` + -Number $number -Description "GraphQL review query" ` + -Select { param($pr) $pr.reviews } + + # Escape the base ref: branch names legitimately contain '/', which would + # otherwise produce an invalid REST path. + $baseRef = [uri]::EscapeDataString($view.baseRefName) + $compareText = & gh api "repos/$Repo/compare/$baseRef...$($view.headRefOid)" + if ($LASTEXITCODE -ne 0) { + throw "Target comparison failed for PR #$number" + } + $compare = $compareText | ConvertFrom-Json + + $failedChecks = @($view.statusCheckRollup | Where-Object { + $_.conclusion -in @("FAILURE", "CANCELLED", "TIMED_OUT", "ACTION_REQUIRED", "STALE") -or + $_.state -in @("ERROR", "FAILURE") + } | ForEach-Object { if ($_.name) { $_.name } else { $_.context } }) + + $pendingChecks = @($view.statusCheckRollup | Where-Object { + ($_.status -and $_.status -ne "COMPLETED") -or + $_.state -in @("EXPECTED", "PENDING") + } | ForEach-Object { if ($_.name) { $_.name } else { $_.context } }) + + # A check that never started cannot fail and cannot be pending, so it leaves + # no trace in either set above. Test presence separately, by prefix, so one + # required name covers every leg the pipeline reports. + $checkNames = @($view.statusCheckRollup | + ForEach-Object { if ($_.name) { $_.name } else { $_.context } } | + Where-Object { $_ }) + $missingRequiredChecks = @($RequiredCheck | Where-Object { + $required = $_ + -not ($checkNames | Where-Object { + $_.StartsWith($required, [StringComparison]::OrdinalIgnoreCase) + }) + }) + + $threads = @($threadPage.nodes) + $unresolved = @($threads | Where-Object { -not $_.isResolved }) + $truncatedThreadComments = @($threads | + Where-Object { $_.comments.pageInfo.hasNextPage } | + ForEach-Object { "$($_.path):$($_.line)" }) + $suppressed = @($reviewPage.nodes | Where-Object { + $_.body -and $_.body -imatch 'suppressed' + } | ForEach-Object { + [pscustomobject]@{ + author = $_.author.login + submittedAt = $_.submittedAt + commit = $_.commit.oid + body = $_.body + } + }) + + $automatedReviews = @($reviewPage.nodes | Where-Object { + $_.author.login -and + ($automatedLogins -contains ($_.author.login -replace '\[bot\]$', '').ToLowerInvariant()) + }) + # Coverage must be decided by commit, not recency. A force-push back to an + # already-reviewed commit leaves the newest review pointing at a commit that + # is no longer head, which would report a reviewed head as uncovered and + # send -WaitForReview into a pointless wait. + # Only submitted reviews count. A pending review already carries a commit oid + # but has a null submittedAt, so matching on the oid alone would report the + # head as covered while the review is still being written - the exact + # premature all-clear this gate exists to prevent. + $reviewsForHead = @($automatedReviews | + Where-Object { $_.submittedAt -and $_.commit.oid -eq $view.headRefOid }) + $latestAutomated = if ($reviewsForHead) { + $reviewsForHead | + Sort-Object { [datetime]$_.submittedAt } | + Select-Object -Last 1 + } else { + $automatedReviews | + Where-Object { $_.submittedAt } | + Sort-Object { [datetime]$_.submittedAt } | + Select-Object -Last 1 + } + $automatedReviewCoversHead = [bool]$reviewsForHead + $suppressedForHead = @($suppressed | + Where-Object { $_.commit -eq $view.headRefOid }) + + $reviewRequested = $false + if ($script:reviewRequestOutcome.ContainsKey($number)) { + $reviewRequested = $script:reviewRequestOutcome[$number] + } + # Under -WaitForReview this function runs once per poll, so requesting + # unconditionally would re-POST the request every cycle until coverage + # arrived - notification spam and a rate-limit risk. Attempt it at most once + # per PR per invocation and reuse the recorded outcome afterwards. + if ($RequestReview -and -not $automatedReviewCoversHead -and + -not $script:reviewRequestOutcome.ContainsKey($number)) { + # Requesting by handle through the REST endpoint is the only path that works. + # The GraphQL requestReviews mutation rejects the reviewer's Bot node id + # ("Could not resolve to User node"), and `gh pr edit --add-reviewer` cannot + # resolve the bot login at all. The reviewer also never shows up in + # requested_reviewers afterwards, so a successful request cannot be confirmed + # by reading that list back; confirm it by polling until + # automatedReviewCoversHead turns true. + gh api "repos/$owner/$name/pulls/$number/requested_reviewers" ` + -X POST -f "reviewers[]=$ReviewerHandle" *> $null + $reviewRequested = ($LASTEXITCODE -eq 0) + $script:reviewRequestOutcome[$number] = $reviewRequested + if (-not $reviewRequested) { + Write-Warning "PR #${number}: could not request a review from '$ReviewerHandle'." + } + } + + $pipelineRunRequested = $false + if ($script:pipelineRunOutcome.ContainsKey($number)) { + $pipelineRunRequested = $script:pipelineRunOutcome[$number] + } + # Same once-per-invocation guard as the review request above: under + # -WaitForReview this runs every poll, and each `/azp run` comment queues + # another build and notifies every subscriber. + if ($RunPipeline -and @($missingRequiredChecks).Count -gt 0 -and + -not $script:pipelineRunOutcome.ContainsKey($number)) { + # A comment is the only trigger the pipeline honours from here; queueing + # through the ADO API needs credentials this script does not assume. + gh pr comment $number --repo $Repo --body "/azp run" *> $null + $pipelineRunRequested = ($LASTEXITCODE -eq 0) + $script:pipelineRunOutcome[$number] = $pipelineRunRequested + if (-not $pipelineRunRequested) { + Write-Warning "PR #${number}: could not comment '/azp run'." + } + } + + [pscustomobject]@{ + number = $view.number + title = $view.title + url = $view.url + state = $view.state + draft = $view.isDraft + mergeable = $view.mergeable + mergeState = $view.mergeStateStatus + reviewDecision = $view.reviewDecision + base = $view.baseRefName + headSha = $view.headRefOid + targetStatus = $compare.status + aheadBy = $compare.ahead_by + behindBy = $compare.behind_by + failedChecks = $failedChecks + pendingChecks = $pendingChecks + missingRequiredChecks = $missingRequiredChecks + unresolvedThreads = $unresolved + suppressedReviewBodies = $suppressed + suppressedReviewBodiesForHead = $suppressedForHead + completeness = [pscustomobject]@{ + reviewThreadPages = $threadPage.pages + reviewThreadCount = $threads.Count + reviewPages = $reviewPage.pages + reviewCount = @($reviewPage.nodes).Count + threadsWithUnreadComments = $truncatedThreadComments + latestAutomatedReviewCommit = if ($latestAutomated) { $latestAutomated.commit.oid } else { $null } + latestAutomatedReviewAt = if ($latestAutomated) { $latestAutomated.submittedAt } else { $null } + latestAutomatedReviewAuthor = if ($latestAutomated) { $latestAutomated.author.login } else { $null } + automatedReviewCoversHead = $automatedReviewCoversHead + automatedReviewRequested = $reviewRequested + pipelineRunRequested = $pipelineRunRequested + # Everything verifiable must be clear. This previously tested only + # $truncatedThreadComments, which counts comment-pagination truncation + # rather than unresolved review threads, so it reported complete=true + # with review feedback still outstanding. It also read a head whose + # required build had never queued as clean, because an absent check + # is neither failed nor pending. + complete = ( + @($truncatedThreadComments).Count -eq 0 -and + $automatedReviewCoversHead -and + @($unresolved).Count -eq 0 -and + @($suppressedForHead).Count -eq 0 -and + @($missingRequiredChecks).Count -eq 0 -and + @($failedChecks).Count -eq 0 -and + @($pendingChecks).Count -eq 0 + ) + } + } +} + +$results = @(foreach ($number in $PullRequest) { + $snapshot = Get-PrSnapshot -number $number + + # Automated review lands some time after a push, so a single snapshot taken + # straight after one reports zero findings for code that has not been looked + # at yet. The required build is worse: it does not start at all until someone + # comments, so waiting only on the review would still return a head with no + # CI on it. Wait for both, for this exact head. + if ($WaitForReview) { + $deadline = (Get-Date).AddMinutes($TimeoutMinutes) + while (-not ($snapshot.completeness.automatedReviewCoversHead -and + @($snapshot.missingRequiredChecks).Count -eq 0) -and + (Get-Date) -lt $deadline) { + Write-Verbose ("PR #{0}: waiting for automated review and required checks of {1}" -f + $number, $snapshot.headSha) + Start-Sleep -Seconds $PollSeconds + $snapshot = Get-PrSnapshot -number $number + } + if (-not $snapshot.completeness.automatedReviewCoversHead) { + Write-Warning ("PR #{0}: no automated review covered {1} within {2} minute(s). " + + "Findings for this head may still be pending; do not read this as clean." -f + $number, $snapshot.headSha, $TimeoutMinutes) + } + if (@($snapshot.missingRequiredChecks).Count -gt 0) { + Write-Warning ("PR #{0}: required check(s) '{1}' never appeared on {2} within " + + "{3} minute(s). Full CI has not run on this head; comment '/azp run' " + + "(or pass -RunPipeline)." -f + $number, ($snapshot.missingRequiredChecks -join ", "), + $snapshot.headSha, $TimeoutMinutes) + } + } + + $snapshot +}) + +ConvertTo-Json -InputObject $results -Depth 12 diff --git a/.github/workflows/ado-integration.yml b/.github/workflows/ado-integration.yml index 11a485d71fe..f4fbf658884 100644 --- a/.github/workflows/ado-integration.yml +++ b/.github/workflows/ado-integration.yml @@ -9,7 +9,7 @@ jobs: alert: runs-on: ubuntu-latest steps: - - uses: mhamilton723/github-actions-issue-to-work-item@master + - uses: danhellem/github-actions-issue-to-work-item@8d0ead9b49a65aa66dac6949b1ff149d7ef8b4de # v2.5 env: ado_token: "${{ secrets.ADO_PERSONAL_ACCESS_TOKEN }}" github_token: "${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}" diff --git a/.github/workflows/ado-pr-to-workitem.yml b/.github/workflows/ado-pr-to-workitem.yml index 8a238371423..9e64474bb05 100644 --- a/.github/workflows/ado-pr-to-workitem.yml +++ b/.github/workflows/ado-pr-to-workitem.yml @@ -10,7 +10,7 @@ jobs: alert: runs-on: ubuntu-latest steps: - - uses: danhellem/github-actions-pr-to-work-item@master + - uses: danhellem/github-actions-pr-to-work-item@496254e48adbe7f1ed14a8afb71dc520b2c052ac # master env: ado_token: '${{ secrets.ADO_PERSONAL_ACCESS_TOKEN }}' github_token: '${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}' diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 4dc5a73f578..b1ea47005c7 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -13,11 +13,11 @@ name: "CodeQL" on: push: - branches: [ "master" ] + branches: [ "master", "spark3.5", "spark4.0", "spark4.1" ] paths-ignore: [ "**.md" ] pull_request: # The branches below must be a subset of the branches above - branches: [ "master" ] + branches: [ "master", "spark3.5", "spark4.0", "spark4.1" ] paths-ignore: [ "**.md" ] schedule: - cron: '17 7 * * 3' @@ -46,7 +46,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@d1ba80a13dd99fba24a470575428917156a28b43 # v4.37.5 + uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 with: languages: ${{ matrix.language }} # Explicitly set source-root to handle runner directory naming @@ -56,7 +56,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@d1ba80a13dd99fba24a470575428917156a28b43 # v4.37.5 + uses: github/codeql-action/autobuild@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 # ℹ️ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun @@ -69,6 +69,6 @@ jobs: # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@d1ba80a13dd99fba24a470575428917156a28b43 # v4.37.5 + uses: github/codeql-action/analyze@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 889c3046535..814a5adcc6d 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -2,7 +2,7 @@ name: Dependency Review on: pull_request: - branches: [ "master" ] + branches: [ "master", "spark3.5", "spark4.0", "spark4.1" ] permissions: contents: read diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index b81c4d011eb..f6d5f0c11ee 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -2,7 +2,7 @@ name: PR Validation on: pull_request: - branches: [ "master", "spark4.1" ] + branches: [ "master", "spark3.5", "spark4.0", "spark4.1" ] paths-ignore: [ "**.md", "docs/**", "website/**" ] env: @@ -35,6 +35,7 @@ jobs: - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # Spark 4.1 requires Java 17; do not take master's JDK 11 when syncing. - name: Set up JDK 17 uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0 with: @@ -58,7 +59,56 @@ jobs: sbt sbtVersion - name: Scalastyle check - run: sbt scalastyle test:scalastyle + run: | + set -o pipefail + sbt scalastyle test:scalastyle 2>&1 | tee "$RUNNER_TEMP/scalastyle-output.log" + + - name: Annotate scalastyle violations + if: failure() + run: | + set -uo pipefail + log="$RUNNER_TEMP/scalastyle-output.log" + [ -f "$log" ] || exit 0 + workspace="${GITHUB_WORKSPACE%/}/" + # sbt colorizes its output, so strip ANSI escapes before matching + violations="$(sed -e 's/\x1B\[[0-9;]*[a-zA-Z]//g' "$log" \ + | grep -aE '^\[(error|warn)\] .+\.scala:[0-9]+(:[0-9]+)?: ' \ + | awk '!seen[$0]++' || true)" + if [ -z "$violations" ]; then + echo "No scalastyle violations found in the log; the failure is reported in the step above." + exit 0 + fi + # Workflow commands treat %, CR and LF specially; property values + # additionally need : and , encoded + encode() { + local value="${1//%/%25}" + value="${value//$'\r'/%0D}" + value="${value//$'\n'/%0A}" + if [ "${2:-}" = "property" ]; then + value="${value//:/%3A}" + value="${value//,/%2C}" + fi + printf '%s' "$value" + } + while IFS= read -r violation; do + case "$violation" in + '[warn]'*) severity="warning" ;; + *) severity="error" ;; + esac + body="${violation#*] }" + location="${body%%: *}" + message="${body#*: }" + file="${location%%:*}" + position="${location#*:}" + lineno="${position%%:*}" + # File-level checks (line length, tabs, EOF newline) report no column + if [ "$position" = "$lineno" ]; then + column="" + else + column=",col=${position#*:}" + fi + echo "::${severity} file=$(encode "${file#"$workspace"}" property),line=${lineno}${column}::$(encode "$message")" + done <<< "$violations" - name: Compile - run: sbt compile test:compile + run: sbt compile "Test / compile" diff --git a/.github/workflows/release-tag-spark.yml b/.github/workflows/release-tag-spark.yml new file mode 100644 index 00000000000..9c036af30f7 --- /dev/null +++ b/.github/workflows/release-tag-spark.yml @@ -0,0 +1,217 @@ +name: Release Tag — Spark Branch Tags + +# Triggers when a release rebase PR is merged into spark4.0 or spark4.1. +# Creates the spark and python derivative tags automatically. +# +# Part of the SynapseML Fabric Release Guide (Steps 1.4–1.5). +# See also: release-tag.yml (triggers the rebase PR). + +on: + pull_request: + types: [closed] + branches: + - spark4.0 + - spark4.1 + +permissions: + contents: write + +jobs: + create-spark-tags: + name: Create spark & python tags + runs-on: ubuntu-latest + + if: >- + github.event.pull_request.merged == true && + github.event.pull_request.head.repo.full_name == github.repository && + startsWith(github.event.pull_request.head.ref, 'release/') + + steps: + - name: Extract version and target + id: info + env: + BRANCH: ${{ github.event.pull_request.head.ref }} + TARGET: ${{ github.event.pull_request.base.ref }} + run: | + set -euo pipefail + + # Require the exact release branch convention for the target branch. + if [[ ! "$BRANCH" =~ ^release/v([0-9]+\.[0-9]+\.[0-9]+)-(spark4\.[01])$ ]]; then + echo "❌ Release branch '$BRANCH' must match release/vX.Y.Z-spark4.0 or release/vX.Y.Z-spark4.1" + exit 1 + fi + + VERSION="${BASH_REMATCH[1]}" + BRANCH_TARGET="${BASH_REMATCH[2]}" + if [ "$BRANCH_TARGET" != "$TARGET" ]; then + echo "❌ Release branch target '$BRANCH_TARGET' does not match PR base '$TARGET'" + exit 1 + fi + + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "target=$TARGET" >> "$GITHUB_OUTPUT" + + # Map branch to python version + case "$TARGET" in + spark4.0) echo "python_ver=3.12" >> "$GITHUB_OUTPUT" ;; + spark4.1) echo "python_ver=3.13" >> "$GITHUB_OUTPUT" ;; + *) echo "❌ Unknown target: $TARGET"; exit 1 ;; + esac + + echo "📦 Version: $VERSION, Target: $TARGET" + + # For a merged PR, GitHub defines merge_commit_sha as the commit written + # to the base branch, including the final rebased commit for "Rebase and + # merge". Guard its nullable schema because an empty checkout ref falls + # back to the default branch instead of identifying the released commit. + - name: Verify the merged commit is known + env: + MERGED_SHA: ${{ github.event.pull_request.merge_commit_sha }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + run: | + set -euo pipefail + if [ -z "$MERGED_SHA" ]; then + echo "::error::pull_request.merge_commit_sha is empty, so the commit that landed on \ + $BASE_REF cannot be identified. Refusing to guess. This workflow normally creates both \ + the spark tag and the matching python tag, so create both manually to avoid a \ + partially tagged release: \ + git tag v- && git tag v-python && \ + git push origin v- v-python" + exit 1 + fi + echo "✅ Merged commit: $MERGED_SHA" + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # Pin to the commit recorded by the merge event, not the base branch + # ref. Any later commit on spark4.x must not receive this release tag. + ref: ${{ github.event.pull_request.merge_commit_sha }} + # Full history and tags: the "tag already exists" guard below is + # meaningless against a shallow clone with no tags fetched. + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Configure git + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + # Nothing so far has established that this release exists. The version is + # read out of a branch name, so a branch called release/v9.9.9-spark4.0 + # merged into spark4.0 -- which carries no branch protection at all -- + # would mint public v9.9.9-spark4.0 and v9.9.9-python3.12 tags for a + # release that was never cut. A derivative tag cannot be valid without + # its primary, and release-tag.yml always creates the primary first, so + # requiring it costs a legitimate release nothing. + - name: Verify the release exists + env: + VERSION: ${{ steps.info.outputs.version }} + run: | + set -euo pipefail + + if ! git rev-parse -q --verify "refs/tags/v${VERSION}" >/dev/null; then + echo "::error::No v${VERSION} tag exists, so there is no such release to derive \ + tags from. This workflow only runs for releases cut by release-tag.yml, which creates \ + v${VERSION} on master first. If this PR is not part of a release, its branch should \ + not be named release/v${VERSION}-*." + exit 1 + fi + echo "✅ Release v${VERSION} exists at $(git rev-parse --short "refs/tags/v${VERSION}^{commit}")" + + # Ancestry is reported rather than enforced. "Rebase and merge" -- one of + # only two methods this repo allows -- replays the release commit onto the + # target branch under a new SHA, so the original v${VERSION} object is + # legitimately not an ancestor of what landed. Failing on that would reject + # every release that goes through this workflow. It is still worth printing: + # when it does hold, the tags satisfy the same ancestry the manual release + # guide produces, and when it does not, that is the known limitation to + # check before the tags are consumed downstream. + if git merge-base --is-ancestor "refs/tags/v${VERSION}" HEAD; then + echo "✅ v${VERSION} is an ancestor of the merged commit" + else + echo "::warning::v${VERSION} is not an ancestor of the merged commit. This is \ + expected when the PR was landed with \"Rebase and merge\", which rewrites SHAs. Verify \ + the release branches carry the intended content before these tags are consumed." + fi + + - name: Create and push tags + env: + VERSION: ${{ steps.info.outputs.version }} + TARGET: ${{ steps.info.outputs.target }} + PYTHON_VER: ${{ steps.info.outputs.python_ver }} + run: | + set -euo pipefail + + SPARK_TAG="v${VERSION}-${TARGET}" + PYTHON_TAG="v${VERSION}-python${PYTHON_VER}" + + TO_PUSH=() + for TAG in "$SPARK_TAG" "$PYTHON_TAG"; do + if git rev-parse -q --verify "refs/tags/$TAG" >/dev/null; then + # An existing tag is only benign if it already points where this run + # would put it. If it points elsewhere, the release is mis-tagged and + # silently skipping would report success for a bad release. + EXISTING=$(git rev-parse "refs/tags/$TAG^{commit}") + if [ "$EXISTING" != "$(git rev-parse HEAD)" ]; then + echo "::error::$TAG already exists at $EXISTING but this release is \ + $(git rev-parse HEAD). Refusing to move a published release tag — resolve manually." + exit 1 + fi + echo "⚠️ $TAG already exists at the same commit — skipping" + else + git tag "$TAG" + TO_PUSH+=("$TAG") + echo "🏷️ Created $TAG" + fi + done + + if [ ${#TO_PUSH[@]} -eq 0 ]; then + echo "✅ Nothing to push — both tags already exist" + else + git push origin "${TO_PUSH[@]}" + echo "✅ Pushed: ${TO_PUSH[*]}" + fi + + - name: Cleanup release branch + env: + HEAD_REF: ${{ github.event.pull_request.head.ref }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + set -euo pipefail + REF="refs/heads/$HEAD_REF" + REMOTE_HEAD=$(git ls-remote --heads origin "$REF") + if [ -z "$REMOTE_HEAD" ]; then + echo "🧹 $HEAD_REF is already absent from origin" + else + REMOTE_SHA=${REMOTE_HEAD%%$'\t'*} + if [ "$REMOTE_SHA" != "$HEAD_SHA" ]; then + echo "::error::$HEAD_REF moved from $HEAD_SHA to $REMOTE_SHA after merge; refusing to delete it" + exit 1 + fi + git push --force-with-lease="$REF:$HEAD_SHA" origin --delete "$REF" + echo "🧹 Deleted $HEAD_REF" + fi + + - name: Summary + env: + VERSION: ${{ steps.info.outputs.version }} + TARGET: ${{ steps.info.outputs.target }} + PYTHON_VER: ${{ steps.info.outputs.python_ver }} + run: | + echo "" + echo "═══════════════════════════════════════════════════" + echo " Spark Branch Tags — Summary" + echo "═══════════════════════════════════════════════════" + echo "" + echo " ✅ v${VERSION}-${TARGET} tag verified at the merged commit" + echo " ✅ v${VERSION}-python${PYTHON_VER} tag verified at the merged commit" + echo " 🧹 Release branch absent from origin" + echo "" + echo " Next steps:" + echo " 1. Queue ADO pipeline 17563 for refs/tags/v${VERSION}-${TARGET}" + echo " 2. Approve ESRP signing on SAW machine" + echo " 3. Proceed to Step 2 (SynapseML-Internal)" + echo "" + echo " Pipeline: https://msdata.visualstudio.com/A365/_build?definitionId=17563" + echo "═══════════════════════════════════════════════════" diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml new file mode 100644 index 00000000000..1d433594a6b --- /dev/null +++ b/.github/workflows/release-tag.yml @@ -0,0 +1,363 @@ +name: Release Tag Orchestrator + +# Triggers when a primary version tag (v1.1.2) is pushed to master. +# Creates the python3.11 and spark3.5 tags at the tagged commit, then opens a +# rebase PR for each spark branch as a chain: spark4.0 onto the release commit, +# then spark4.1 onto spark4.0. +# +# spark4.0 can be opted out of. See SKIP_SPARK40 below. +# +# Part of the SynapseML Fabric Release Guide (Steps 1.4–1.5). +# See also: release-tag-spark.yml (creates spark/python tags after PR merge). + +on: + push: + tags: + # Deliberately permissive: GitHub filter patterns are globs, and support for + # `+` as a quantifier is inconsistently documented. `[0-9]*` matches under + # every reading. Strict X.Y.Z enforcement is done by the "Extract version" + # step below, and derivative tags are excluded by the job-level `if:`. + - "v[0-9]*.[0-9]*.[0-9]*" + workflow_dispatch: + inputs: + skip_spark40: + description: 'Skip the spark4.0 rebase PR for this run' + type: boolean + default: false + +permissions: + contents: write + pull-requests: write + +env: + # Opt out of spark4.0 in two ways, checked in this order: + # 1. the workflow_dispatch checkbox above, for a single manual run; + # 2. repository variable SKIP_SPARK40='true', for a persistent opt-out — + # a tag push carries no inputs, so this is the only way to skip it on + # the automatic trigger. + # Default is false: spark4.0 is processed exactly as before. + # Note this only controls whether the rebase PR is opened here. Merging a + # spark4.0 release branch by hand still tags normally via release-tag-spark.yml. + SKIP_SPARK40: ${{ inputs.skip_spark40 == true || vars.SKIP_SPARK40 == 'true' }} + +jobs: + release-tags: + name: Create release tags & rebase PR + runs-on: ubuntu-latest + + # Skip derivative tags (v1.1.2-spark4.0, v1.1.2-python3.11, etc.) + if: >- + !contains(github.ref_name, '-spark') && + !contains(github.ref_name, '-python') + + steps: + - name: Extract version + id: version + run: | + set -euo pipefail + # The version comes from the ref this ran on, so a manual run must be + # dispatched with a release tag selected in the ref picker. Dispatching + # from master (the default the UI offers) fails here by design rather + # than guessing which release was meant. + VERSION="${GITHUB_REF_NAME#v}" + if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$'; then + echo "❌ Version '$VERSION' is not X.Y.Z format. Re-run this workflow with a \ + vX.Y.Z tag selected as the ref." + exit 1 + fi + # Published only after validation, so no later step can consume a version + # this job already rejected. + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "📦 Version: $VERSION" + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Configure git + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + - name: Capture released commit + run: | + set -euo pipefail + RELEASE_COMMIT=$(git rev-parse HEAD) + # The release tag must identify a commit from master. Refuse to mint + # derivative tags or PRs from an accidental tag on another branch. + git fetch --no-tags origin '+refs/heads/master:refs/remotes/origin/master' + if ! git merge-base --is-ancestor "$RELEASE_COMMIT" origin/master; then + echo "::error::${GITHUB_REF_NAME} points to $RELEASE_COMMIT, which is not contained in origin/master" + exit 1 + fi + echo "RELEASE_COMMIT=$RELEASE_COMMIT" >> "$GITHUB_ENV" + echo "✅ ${GITHUB_REF_NAME} points at master commit $RELEASE_COMMIT" + + # ── Tags at the released master commit ──────────────── + - name: Create master release tags + env: + VERSION: ${{ steps.version.outputs.version }} + run: | + set -euo pipefail + # Both derivative tags every release carries at the master commit. master + # targets Spark 3.5 and Python 3.11, and v1.1.1 and v1.1.3 each carry both + # tags here, so creating only the python one leaves every release short a tag. + for TAG in "v${VERSION}-python3.11" "v${VERSION}-spark3.5"; do + # Use the immutable tagged commit captured above, not a moving branch ref. + if git rev-parse -q --verify "refs/tags/$TAG" >/dev/null; then + # Same reasoning as release-tag-spark.yml: an existing tag is only + # benign if it already points at this release. Otherwise the release + # is mis-tagged and skipping would report success for a bad release. + EXISTING=$(git rev-parse "refs/tags/$TAG^{commit}") + if [ "$EXISTING" != "$RELEASE_COMMIT" ]; then + echo "::error::$TAG already exists at $EXISTING but v${VERSION} is \ + $RELEASE_COMMIT. Refusing to move a published release tag — resolve manually." + exit 1 + fi + echo "⚠️ $TAG already exists at the same commit — skipping" + else + git tag "$TAG" "$RELEASE_COMMIT" + git push origin "$TAG" + echo "✅ Created $TAG" + fi + done + + # ── Rebase PRs for the spark release branches ────────── + - name: Create spark rebase PRs + env: + VERSION: ${{ steps.version.outputs.version }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + + # spark4.0 first, and spark4.1 rebased onto it rather than onto master. + # spark4.1 is maintained as spark4.0 plus its own upgrade commit — every + # release so far satisfies `git merge-base --is-ancestor v-spark4.0 + # v-spark4.1`, and origin/spark4.0 is an ancestor of origin/spark4.1 + # today. Rebasing both onto master independently replays the Spark 4.0 + # upgrade patch twice, producing two commits with the same tree and + # different SHAs, which breaks that ancestry for good: a later spark4.0-only + # fix would no longer reach spark4.1. Chaining them lets `git rebase` drop + # the patch-equivalent commit and keeps the branches a single line, which is + # also what the release guide's `git checkout spark4.1; git rebase + # upstream/spark4.0` produces. + TARGETS=(spark4.0 spark4.1) + if [ "${SKIP_SPARK40}" = "true" ]; then + echo "⏭️ SKIP_SPARK40 is set — not opening a spark4.0 rebase PR" + TARGETS=(spark4.1) + fi + + # Each target is rebased onto the previous one's release branch, starting + # from the master release commit. + REBASE_ONTO="$RELEASE_COMMIT" + + # Skipping spark4.0 must not silently un-chain spark4.1. Leaving REBASE_ONTO + # at the master release commit is exactly the "both onto master + # independently" case described above: it replays the Spark 4.0 upgrade + # patch a second time and breaks spark4.0 -> spark4.1 ancestry for good. + # Chain onto spark4.0 whenever that branch already carries this release. + if [ "${SKIP_SPARK40}" = "true" ]; then + # The checkout can predate both refs this decision reads. + git fetch --no-tags origin '+refs/heads/spark4.0:refs/remotes/origin/spark4.0' 2>/dev/null || true + git fetch origin "+refs/tags/v${VERSION}-spark4.0:refs/tags/v${VERSION}-spark4.0" 2>/dev/null || true + + SPARK40_HAS_RELEASE=false + if git rev-parse -q --verify refs/remotes/origin/spark4.0 >/dev/null; then + # "Rebase and merge" rewrites SHAs, so once the spark4.0 release PR has + # landed $RELEASE_COMMIT is no longer an ancestor of origin/spark4.0 even + # though the release is sitting there under a different SHA. The derivative + # tag release-tag-spark.yml mints on that merge is the dependable signal; + # the ancestry probe stays as a fallback for a fast-forwarded spark4.0. + if git rev-parse -q --verify "refs/tags/v${VERSION}-spark4.0" >/dev/null || + git merge-base --is-ancestor "$RELEASE_COMMIT" origin/spark4.0; then + SPARK40_HAS_RELEASE=true + fi + fi + + if [ "$SPARK40_HAS_RELEASE" = true ]; then + REBASE_ONTO="origin/spark4.0" + echo " spark4.1 will be rebased onto origin/spark4.0, which already carries v${VERSION}." + else + echo " spark4.1 will be rebased onto v${VERSION} directly this run." + echo "::warning::spark4.0 has no v${VERSION} release yet, so rebasing spark4.1 onto master breaks spark4.0 -> spark4.1 ancestry until a release runs without SKIP_SPARK40." + fi + fi + + FAILED=() + for TARGET in "${TARGETS[@]}"; do + if ! git rev-parse -q --verify "refs/remotes/origin/$TARGET" >/dev/null; then + echo "⚠️ $TARGET branch doesn't exist — skipping" + continue + fi + + BRANCH="release/v${VERSION}-${TARGET}" + echo "═══ $TARGET (onto $REBASE_ONTO) ═══" + + # Nothing to do if the branch already has the released commit, and + # rebasing anyway is not harmless. `git rebase` only no-ops when the + # branch is linear on top of the release; if the branch picked up a + # merge commit (e.g. the previous release PR was merged with a merge + # commit rather than "Rebase and merge"), the rebase flattens that + # merge and rewrites SHAs. If the tip *is* the release commit there + # is nothing ahead at all and `gh pr create` fails with "No commits + # between". Both only show up on a re-run, which is exactly when the + # workflow is supposed to be safe to repeat. + if git merge-base --is-ancestor "$REBASE_ONTO" "origin/$TARGET"; then + echo "✅ $TARGET already contains $REBASE_ONTO — nothing to rebase" + REBASE_ONTO="origin/$TARGET" + continue + fi + + # Check before rebuilding the branch: an existing PR may contain + # manual conflict resolution that must not be replaced or made to + # conflict again locally. Ignore same-named PRs from forks. + EXISTING=$(gh pr list --head "$BRANCH" --base "$TARGET" \ + --state open --json number,isCrossRepository \ + --jq 'map(select(.isCrossRepository == false))[0].number // empty') + if [ -n "$EXISTING" ]; then + echo "♻️ PR #$EXISTING already open for $BRANCH — leaving it untouched." + echo " If that PR needs refreshing, update it manually." + # Chain the next target onto what that PR will land, not onto master. + git fetch --no-tags origin "+refs/heads/$BRANCH:refs/remotes/origin/$BRANCH" + REBASE_ONTO="origin/$BRANCH" + continue + fi + + # "Rebase and merge" rewrites the primary release commit, so the + # ancestor check above cannot recognize a completed release on its + # own. Reuse a merged PR only while its recorded result remains on + # the target branch; otherwise fail instead of opening a duplicate. + MERGED=$(gh pr list --head "$BRANCH" --base "$TARGET" \ + --state merged --json number,mergeCommit,isCrossRepository \ + --jq '(map(select(.isCrossRepository == false))[0] // empty) | + [.number, .mergeCommit.oid] | @tsv') + if [ -n "$MERGED" ]; then + IFS=$'\t' read -r MERGED_PR MERGED_SHA <<< "$MERGED" + if [ -z "$MERGED_SHA" ]; then + echo "::error::PR #$MERGED_PR was merged for $BRANCH but has no recorded merge commit" + FAILED+=("$TARGET") + break + fi + if ! git merge-base --is-ancestor "$MERGED_SHA" "origin/$TARGET"; then + echo "::error::PR #$MERGED_PR merged at $MERGED_SHA, which is no longer on $TARGET" + FAILED+=("$TARGET") + break + fi + echo "✅ PR #$MERGED_PR already merged at $MERGED_SHA — $TARGET is complete" + REBASE_ONTO="origin/$TARGET" + continue + fi + + git checkout -B "$BRANCH" "origin/$TARGET" + + if ! git rebase "$REBASE_ONTO"; then + git rebase --abort + echo "::error::Rebase of $TARGET onto $REBASE_ONTO conflicted — resolve manually" + FAILED+=("$TARGET") + break + fi + + git push --force-with-lease -u origin "$BRANCH" + BASED_ON="$REBASE_ONTO" + REBASE_ONTO="$BRANCH" + + # Only tell the release engineer to merge something else first when there + # really is something else. With SKIP_SPARK40, or on a re-run where + # spark4.0 has already landed, this PR is based on a commit that is + # already on master or on the target branch and can merge on its own. + case "$BASED_ON" in + release/v*|origin/release/v*) + MERGE_ORDER="The spark branches form a chain: \`spark4.1\` is maintained as + \`spark4.0\` plus its own upgrade commit. This PR is stacked on + \`${BASED_ON#origin/}\`, so **merge that PR first**, then re-check this one — + \"Rebase and merge\" rewrites SHAs on the base branch, so merging this PR + first would leave it without the spark4.0 release commit." + ;; + *) + MERGE_ORDER="This PR is based on \`${BASED_ON#origin/}\`, which has already + landed, so there is no other release PR to merge ahead of it." + ;; + esac + + # An empty check list means different things on the two branches, and + # reading "no checks" as "checks passed" is the failure mode worth + # spending a paragraph on. + if [ "$TARGET" = "spark4.0" ]; then + CHECKS="\`spark4.0\` is outside the branch filters of \`pr-validation.yml\` and the + ADO pipeline, so this PR legitimately runs no checks at all. A PR opened by a + workflow may also need \"Approve workflows to run\" clicked first. Either way an + empty check list is not the same as a green one." + else + CHECKS="A PR opened by a workflow may need \"Approve workflows to run\" clicked + before its checks start, so an empty check list is not the same as a green one." + fi + + gh pr create \ + --base "$TARGET" \ + --head "$BRANCH" \ + --title "chore: Rebase ${TARGET} for v${VERSION} release" \ + --body "## Release v${VERSION} — ${TARGET} rebase + + Auto-generated by the Release Tag Orchestrator. + + ### What this does + Rebases \`${TARGET}\` onto \`${BASED_ON}\`. + + ### Merge order + ${MERGE_ORDER} + + ### Checks + ${CHECKS} + + ### After merging + Tags \`v${VERSION}-${TARGET}\` and the matching python tag will + be created automatically by the \`release-tag-spark\` workflow. + + ### Next steps + 1. ✅ Merge this PR (use **Rebase and merge**) + 2. Tags are auto-created on merge + 3. Queue ADO pipeline 17563 for \`refs/tags/v${VERSION}\` + 4. Queue ADO pipeline 17563 for \`refs/tags/v${VERSION}-${TARGET}\` + 5. Approve ESRP signing on SAW machine" || { + echo "::error::Failed to open PR for $TARGET" + FAILED+=("$TARGET") + # Stop the chain, matching the rebase-conflict path above. The next + # target would be stacked on a pushed branch that has no PR, so it + # could never be merged in order. + break + } + done + + if [ ${#FAILED[@]} -gt 0 ]; then + echo "::error::Failed for: ${FAILED[*]}" + exit 1 + fi + + # ── Summary ──────────────────────────────────────────── + - name: Summary + env: + VERSION: ${{ steps.version.outputs.version }} + run: | + set -euo pipefail + echo "" + echo "═══════════════════════════════════════════════════" + echo " Release Tag Orchestrator — Summary" + echo "═══════════════════════════════════════════════════" + echo "" + echo " Version: v${VERSION}" + echo "" + echo " ✅ v${VERSION}-python3.11 tag verified at the tagged commit" + echo " ✅ v${VERSION}-spark3.5 tag verified at the tagged commit" + echo " 📋 Spark release branches processed" + echo "" + if [ "${SKIP_SPARK40}" = "true" ]; then + echo " ⏭️ spark4.0 skipped (SKIP_SPARK40)" + echo " Next: merge the spark4.1 rebase PR." + else + echo " Next: merge the spark4.0 rebase PR first, then spark4.1 —" + echo " spark4.1 is maintained on top of spark4.0." + fi + echo "═══════════════════════════════════════════════════" diff --git a/.github/workflows/remove-awaiting-response-label.yml b/.github/workflows/remove-awaiting-response-label.yml index 1ff1e4b94d1..9db3ffb4f44 100644 --- a/.github/workflows/remove-awaiting-response-label.yml +++ b/.github/workflows/remove-awaiting-response-label.yml @@ -13,7 +13,7 @@ jobs: github.event.comment.author_association != 'COLLABORATOR' steps: - name: Remove needs-reply label - uses: octokit/request-action@v2.x + uses: octokit/request-action@02f5e7c637a73a3b12ed81015fa7fb5f11cc5d7d # v2.x continue-on-error: true with: route: DELETE /repos/:repository/issues/:issue/labels/:label @@ -21,4 +21,4 @@ jobs: issue: ${{ github.event.issue.number }} label: "awaiting response" env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index ae99565a717..fdc14e5c43d 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -67,6 +67,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@d1ba80a13dd99fba24a470575428917156a28b43 # v4 + uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 with: sarif_file: results.sarif diff --git a/.gitignore b/.gitignore index 0ece5fc4d30..88d91adf650 100644 --- a/.gitignore +++ b/.gitignore @@ -103,7 +103,5 @@ mlflow-save-model-0/ # Agent .agent/ -.agents/ -AGENTS.md .anvil/ .hypothesis/ diff --git a/.pipelines/release-compat-prerequisites.txt b/.pipelines/release-compat-prerequisites.txt new file mode 100644 index 00000000000..930f62be2d7 --- /dev/null +++ b/.pipelines/release-compat-prerequisites.txt @@ -0,0 +1,12 @@ +# PR #2591 adds Azure AI Search AAD auth required by this change. +# Remove this prerequisite once every validated release branch contains that backport. +04897bae9baa08f0d67855566f7bad235791d508 +# PR #2612 introduces the WorkerMessage protocol used by the IPv6 endpoint parsing fix. +# Remove this prerequisite once every validated release branch contains that backport. +4a52d9ae4184d3ce00394cd9cb4209c35990fc47 +# PR #2507 introduced tests modified by PR #2635 but absent from Spark 4.1. +# Remove this scoped prerequisite once every validated release branch contains these tests. +6938c472175ed1592ec24e7d8c65d9174f17c4e5 core/src/test/scala/com/microsoft/azure/synapse/ml/automl/VerifyEvaluationUtils.scala core/src/test/scala/com/microsoft/azure/synapse/ml/core/metrics/VerifyMetricConstants.scala +# PR #2635 changed ComputeModelStatistics and its tests before this metadata fix. +# Remove this prerequisite once every validated release branch contains that backport. +f7a1dc50d09d400d279d08bf69a1fac322896748 diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000000..4fef1e0fe56 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,158 @@ +# SynapseML agent guide + +Use this file for repository-wide rules. Human contributors should start with +[CONTRIBUTING.md](CONTRIBUTING.md). + +## Start here + +1. Resolve the target branch (the PR base, not merely the checked-out feature + branch), then use the + [branch context skill](.github/skills/synapseml-branches/SKILL.md). +2. Read versions from [build.sbt](build.sbt) and + [environment.yml](environment.yml). Do not copy version numbers into this + shared guide. +3. Use the narrowest relevant repository skill: + [Scala changes](.github/skills/scala-code/SKILL.md), + [local setup](.github/skills/synapseml-local-setup/SKILL.md), and + [code review](.github/skills/code-review/SKILL.md). +4. For requests to make one or more issues or PRs "5/5", "200% ready", or + merge-ready, use the + [SynapseML PR loop](.github/skills/synapseml-pr-loop/SKILL.md). + +## Non-negotiable rules + +- Never edit `target/`; generated files are overwritten. +- Never commit or print credentials, keys, connection strings, or `.env` files. +- Do not add RDD-based implementations. Use DataFrame/Dataset APIs so code works + with Spark Connect and managed Spark modes. +- Do not rebase or force-push shared port branches. +- Do not add a hand-written `__init__.py` merely to re-export generated classes; + a stale `__all__` can hide public APIs. +- Keep existing public JVM signatures and serialized parameter shapes unless a + breaking change is explicitly approved. + +Ask before changing [pipeline.yaml](pipeline.yaml), workflows under +[`.github/workflows/`](.github/workflows/), release tooling, or dependency +pins. These changes affect every branch and require CI evidence. + +## Branch model + +| Branch | Use it for | +| --- | --- | +| `master` | Ordinary features, fixes, and repository-wide changes | +| `spark` port branches | Differences required specifically by that Spark port | + +Run `git branch -r` to see which port branches currently exist; this guide does +not name them, so that adding one does not require editing a file that must stay +identical on every branch. + +Land cross-version changes on `master`; port branches receive them by merging +`master`. When resolving a port-branch merge: + +- keep the branch side only for version-driven differences; +- take the `master` side for ordinary fixes; +- combine both when each changed the same file for a different reason. + +Reachability is not proof that a sync preserved content. Compare the merge base, +`master`, and the port branch for every conflicted file, and verify that +`master` additions remain present. + +`AGENTS.md` and [CONTRIBUTING.md](CONTRIBUTING.md) must stay identical across +branches, so neither may name a Spark, Scala, Java, or Python version, or a path +containing one. Put branch-only facts in the +[branch context skill](.github/skills/synapseml-branches/SKILL.md). Every other +document — [README.md](README.md), the website, and module docs — is free to be +branch- and version-specific, because nothing requires those to match across +branches. + +## Repository map + +| Module | Path | Purpose | +| --- | --- | --- | +| Core | [`core/`](core/) | SparkML foundations, IO, codegen, AutoML, causal and exploratory tools | +| Cognitive | [`cognitive/`](cognitive/) | Azure AI and OpenAI service stages | +| LightGBM | [`lightgbm/`](lightgbm/) | Distributed classifier, regressor, and ranker | +| Vowpal Wabbit | [`vw/`](vw/) | VW integration | +| Deep learning | [`deep-learning/`](deep-learning/) | ONNX Runtime inference | +| OpenCV | [`opencv/`](opencv/) | Image transformations | + +All modules depend on `core`; `deep-learning` also depends on `opencv`. + +Each module follows `src/main/scala`, optional hand-written +`src/main/python`, and `src/test/{scala,python}`. Follow nearby code before +introducing a new pattern. + +## Scala-first API and code generation + +Public SparkML behavior belongs in Scala. Classes mixing in +[`Wrappable`](core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/Wrappable.scala) +generate Python wrappers under +`target/scala-*/generated/src/python/synapse/ml/`. + +For a new or changed stage, verify: + +- companion object extends `DefaultParamsReadable[Stage]`; +- class accepts `uid: String` and has a random-UID no-arg constructor; +- `Wrappable` is present when a Python wrapper is required; +- [`SynapseMLLogging`](core/src/main/scala/com/microsoft/azure/synapse/ml/logging/SynapseMLLogging.scala) + is mixed in, `logClass(...)` is called, and `fit`/`transform` is logged; +- `copy(extra)` preserves parameters, normally through `defaultCopy(extra)`; +- `transformSchema` matches runtime output; +- save/load and generated wrapper behavior are tested. + +Hand-written Python may extend a generated `_ClassName` wrapper when JVM +delegation needs a Python convenience method. Do not move core behavior into +Python. + +## Validation + +Use the smallest command that proves the change, then expand when the affected +surface requires it. Run SBT with the JDK selected by the +[local setup skill](.github/skills/synapseml-local-setup/SKILL.md), not the +machine default. + +```bash +sbt /compile +sbt /Test/compile +sbt "/testOnly fully.qualified.Suite" +sbt /scalastyle /Test/scalastyle +sbt codegen +black --check --extend-exclude 'docs/' . +``` + +Black is pinned in [pyproject.toml](pyproject.toml); use that version. + +Scala tests extend +[`TestBase`](core/src/test/scala/com/microsoft/azure/synapse/ml/core/test/base/TestBase.scala). +Add positive, negative, schema, persistence, and end-to-end coverage where the +behavior warrants it. A helper-only test is not proof that the public +transformer path works. + +Tests that require Azure credentials should skip when credentials are absent. +Do not turn a skip into a pass by embedding a secret. Inspect service tests +before running them because some create or delete cloud resources. + +## Pull requests and CI + +- Use a conventional title: `feat:`, `fix:`, `test:`, `docs:`, `ci:`, or + `chore:`. +- Target `master` unless the change exists only for a port branch. +- Resolve active and suppressed review findings; document why any finding is + invalid. +- Trigger Azure validation with `/azp run` where supported. Branch-specific + exceptions are documented in the + [branch context skill](.github/skills/synapseml-branches/SKILL.md). +- Treat [GitHub Actions](.github/workflows/) as fast checks and + [pipeline.yaml](pipeline.yaml) as the full build. +- Do not equate green checks with correctness: compare before/after failures, + inspect skipped tests, and verify the requested behavior directly. +- Rebase feature PRs onto the latest target before final validation. Merge + `master` into shared port branches instead of rebasing them. + +## Keep this file useful + +Add only durable, repository-wide guidance that changes an agent's decision. +Prefer a link to the source of truth over copied commands, versions, or long +examples. Put implementation detail beside the code, contributor process in +[CONTRIBUTING.md](CONTRIBUTING.md), and branch-specific facts in +the [branch context skill](.github/skills/synapseml-branches/SKILL.md). diff --git a/CODEOWNERS b/CODEOWNERS index 6153356e53d..c5a20102ec1 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,35 +1,10 @@ # For general updates to the library -* @mhamilton723 @drdarshan - -# Ilya's Areas -lightgbm/ @imatiach-msft -automl/ @imatiach-msft -featurize/ @imatiach-msft -featurize/text/ @mhamilton723 -train/ @imatiach-msft - -# Dan's Areas -recommendation/ @dciborow -recommendation/ @miguelgfierro -recommendation/ @gramhagen - -# Dalitso's Areas -tools/helm/ @dbanda +* @ranadeepsingh @BrendanWalsh # Markus' Areas vw/ @eisber isolationforest/ @eisber recommendation/ @eisber -#Roy's Areas -cyber/ @rolevin - -#Jason's Areas -explainers/ @memoryz - -#Serena's Areas -website/ @serena-ruan -core/src/main/scala/com/microsoft/azure/synapse/ml/codegen @serena-ruan - # Scott's Areas lightgbm/ @svotaw diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2556638be7f..a764e46ee60 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,6 +17,28 @@ an issue. Also, you can up-vote or comment on existing issues. If you want to add code, examples or documentation to the repository, follow this process: +### Which branch should I target? + +Most contributions target `master`. + +This repository also maintains ports of the library to newer Spark versions on +long-lived branches named `spark`. Run `git branch -r` to see which +ones currently exist. Target one of those only when the change exists *because +of* that Spark version — for example, replacing an API that behaves differently +there. Ordinary bug fixes and new features belong on `master` and reach the port +branches when `master` is merged into them. + +If a fix applies everywhere, land it on `master` first so the port branches +inherit it on the next sync. Fixing the same thing separately on each branch +creates a conflict that someone then has to resolve by hand. + +Each port branch carries an `AGENTS_.md` describing what diverges there +and why. Read it before changing anything on that branch — several of the +differences look like mistakes until you know the reason for them, and a +"cleanup" that reverts one tends to break the build in a way that is not obvious +from the diff. Repository-wide guidance for automated coding agents is in +[AGENTS.md](AGENTS.md). + #### Propose a contribution - Preferably, get started by tackling existing issues to get yourself acquainted diff --git a/build.sbt b/build.sbt index 662ebf2debe..cf5d4e7f94d 100644 --- a/build.sbt +++ b/build.sbt @@ -395,7 +395,32 @@ val settings = Seq( assembly / assemblyOption := (assembly / assemblyOption).value.copy(includeScala = false), autoAPIMappings := true, pomPostProcess := pomPostFunc, - sbtPlugin := false + sbtPlugin := false, + // Scoverage: exclude only code that genuinely cannot be exercised by unit tests -- + // the build-time generator entry points and the classes that require a live + // Microsoft Fabric environment (token brokering / workload endpoints). + // Deliberately NOT excluded: CodegenConfig, DefaultParamInfo, GenerationUtils and + // Wrappable are pure logic with existing unit tests, and fabric's FabricTokenParser, + // OpenAIFabricSetting and RESTUtils have no live-environment dependency. + // Kept inline (rather than a top-level val) so this diff stays a single hunk anchored + // on a stable line, which lets it cherry-pick cleanly onto the release branches. + coverageExcludedPackages := Seq( + "com\\.microsoft\\.azure\\.synapse\\.ml\\.codegen\\.CodeGen.*", + "com\\.microsoft\\.azure\\.synapse\\.ml\\.codegen\\.PyCodegen.*", + "com\\.microsoft\\.azure\\.synapse\\.ml\\.codegen\\.RCodegen.*", + "com\\.microsoft\\.azure\\.synapse\\.ml\\.codegen\\.PythonInitMerger.*", + // sbt-buildinfo generates BuildInfo into src_managed. It is emitted code with no + // branches, it is instrumented at 28 lines / 0% by scoverage, and there is nothing + // meaningful to unit test, so measuring it only depresses the reported number. + "com\\.microsoft\\.azure\\.synapse\\.ml\\.build\\..*", + "com\\.microsoft\\.azure\\.synapse\\.ml\\.fabric\\.FabricClient.*", + "com\\.microsoft\\.azure\\.synapse\\.ml\\.fabric\\.TokenLibrary.*", + "com\\.microsoft\\.azure\\.synapse\\.ml\\.logging\\.fabric\\..*" + ).mkString(";"), + coverageFailOnMinimum := false, + coverageHighlighting := true, + // Cobertura XML is the format the Azure DevOps coverage publisher consumes. + coverageOutputCobertura := true ) ThisBuild / publishMavenStyle := true @@ -419,12 +444,18 @@ lazy val deepLearning = (project in file("deep-learning")) .settings(settings ++ Seq( libraryDependencies ++= Seq( "com.microsoft.azure" % "onnx-protobuf_2.13" % "0.9.24", - "com.microsoft.onnxruntime" % "onnxruntime_gpu" % "1.8.1", + // Default to the CPU-only, cross-platform ONNX Runtime artifact (ships native libraries for + // Windows x64, Linux x64/aarch64, and macOS x64/aarch64) so ONNXModel works out of the box on + // every supported platform. CUDA support is an explicit opt-in: the onnxruntime_gpu artifact + // adds NVIDIA CUDA/TensorRT execution providers (Linux/Windows only, ~300+MB) and is never + // bundled by default. See docs/Explore Algorithms/Deep Learning/ONNX.md for GPU setup. + "com.microsoft.onnxruntime" % "onnxruntime" % "1.8.1", "org.apache.hadoop" % "hadoop-common" % "3.3.4" % "test", "org.apache.hadoop" % "hadoop-azure" % "3.3.4" % "test", ), name := "synapseml-deep-learning" ): _*) + .settings(OnnxRuntimeDependency.settings: _*) lazy val lightgbm = (project in file("lightgbm")) .dependsOn(core % "test->test;compile->compile") diff --git a/codecov.yaml b/codecov.yaml index a9b25613766..1b7bcf19df9 100644 --- a/codecov.yaml +++ b/codecov.yaml @@ -1,6 +1,20 @@ codecov: notify: require_ci_to_pass: no + # Wait for expected number of coverage uploads before sending notifications + # This prevents premature comments with incomplete data. + # 54 = UnitTests 40 + PythonTests 7 + RTests 6 + WebsiteSamplesTests 1 + # (every leg of those jobs runs templates/codecov.yml on succeededOrFailed). + after_n_builds: 54 + +comment: + layout: "reach,diff,flags,files" + behavior: new + require_changes: false + require_base: false + require_head: true + # Wait for all expected uploads before commenting (see note above) + after_n_builds: 54 coverage: precision: 2 @@ -25,6 +39,9 @@ flags: scala: paths: - src/main/scala + # Carry forward coverage from previous builds for unchanged files + carryforward: true python: paths: - src/main/python + carryforward: true diff --git a/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/CognitiveServiceBase.scala b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/CognitiveServiceBase.scala index 2b8f90549b0..536a5626b78 100644 --- a/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/CognitiveServiceBase.scala +++ b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/CognitiveServiceBase.scala @@ -99,9 +99,18 @@ trait HasServiceParams extends Params { emptyParamData(row, p) } + // Spark returns mutable collections (e.g. mutable.ArraySeq) for array and map columns. Under + // Scala 2.13 the unqualified Seq and Map aliases denote the immutable variants, so letting a + // mutable value reach a call site typed as Seq or Map fails the checkcast at runtime. + private def asImmutableCollection(value: Any): Any = value match { + case s: scala.collection.Seq[_] if !s.isInstanceOf[scala.collection.immutable.Seq[_]] => s.toIndexedSeq + case m: scala.collection.Map[_, _] if !m.isInstanceOf[scala.collection.immutable.Map[_, _]] => m.toMap + case other => other + } + protected def getValueOpt[T](row: Row, p: ServiceParam[T]): Option[T] = { get(p).orElse(getDefault(p)).flatMap { - case Right(colName) => Option(row.getAs[T](colName)) + case Right(colName) => Option(row.getAs[Any](colName)).map(asImmutableCollection(_).asInstanceOf[T]) case Left(value) => Some(value) } } diff --git a/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/search/AzureSearch.scala b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/search/AzureSearch.scala index 5a4817528a1..15dfaf98d9f 100644 --- a/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/search/AzureSearch.scala +++ b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/search/AzureSearch.scala @@ -20,7 +20,8 @@ import org.apache.spark.ml.util._ import org.apache.spark.ml.{ComplexParamsReadable, NamespaceInjections, PipelineModel} import org.apache.spark.ml.linalg.SQLDataTypes.VectorType import org.apache.spark.ml.functions.vector_to_array -import org.apache.spark.sql.functions.{col, expr, struct, to_json, to_utc_timestamp, date_format, when} +import org.apache.spark.sql.functions.{col, concat, expr, forall, from_json, lit, raise_error, size, + struct, to_json, to_utc_timestamp, date_format, when} import org.apache.spark.sql.streaming.DataStreamWriter import org.apache.spark.sql.types._ import org.apache.spark.sql.{DataFrame, Dataset, Row} @@ -207,7 +208,7 @@ object AzureSearchWriter extends IndexParser with IndexJsonGetter with SLogging } /** - * Converts date and timestamp columns to ISO8601 format strings as required by Azure Search. + * Converts date and timestamp columns to ISO8601 format strings as required by Azure AI Search. * * @param df DataFrame with potential date/time columns * @param indexJson JSON string containing the index schema @@ -249,6 +250,74 @@ object AzureSearchWriter extends IndexParser with IndexJsonGetter with SLogging } } + /** + * Converts string columns containing GeoJSON to the proper struct shape required for + * Azure Search `Edm.GeographyPoint` fields. + * + * Azure AI Search expects spatial values to be sent as a GeoJSON object + * (e.g. `{"type":"Point","coordinates":[lon, lat]}`), not as a JSON-encoded string. + * Users frequently have their GeoJSON readily available as a string column, and + * passing it as a `StringType` previously caused a `400 Bad Request` + * (see [[https://github.com/microsoft/SynapseML/issues/2420]]) because the writer + * JSON-escaped the entire string. + * + * For each '''top-level''' field declared as `Edm.GeographyPoint` in the index, if the + * corresponding DataFrame column is a `StringType`, parse it into the canonical + * `StructType(type: StringType, coordinates: ArrayType(DoubleType))` so that downstream + * `to_json` emits a proper GeoJSON object. Columns that are already structured are + * left as-is. GeographyPoint fields nested inside complex types are not auto-converted + * (mirrors the existing top-level-only handling in `convertDateTimeToISO8601`). + * + * Parsing uses Spark's `FAILFAST` mode so malformed GeoJSON surfaces an explicit + * exception instead of being silently coerced to `null` and shipped to Azure Search. + * `FAILFAST` alone only rejects syntactically invalid JSON, so the parsed value is + * additionally validated to be a genuine GeoJSON Point (`type == "Point"` with exactly + * two non-null coordinates). Anything else raises an error naming the column and the + * offending value rather than indexing a silently-null location. NULL inputs are + * preserved as NULL. + * + * @param df DataFrame with potential GeographyPoint columns + * @param indexJson JSON string containing the index schema + * @return DataFrame with string GeographyPoint columns converted to GeoJSON structs + */ + private[ml] def convertGeographyPointToStruct(df: DataFrame, indexJson: String): DataFrame = { + // Derived from edmTypeToSparkType so the parsed shape can never drift from the type + // checkSchemaParity expects for Edm.GeographyPoint + val geoStructType = edmTypeToSparkType(GeographyPointEdmType, None) + val parseOptions = Map("mode" -> "FAILFAST") + val geoFields = parseIndexJson(indexJson).fields + .filter(_.`type` == GeographyPointEdmType) + .map(_.name) + geoFields.foldLeft(df) { (currentDF, fieldName) => + if (currentDF.columns.contains(fieldName)) { + currentDF.schema(fieldName).dataType match { + case StringType => + val parsed = from_json(col(fieldName), geoStructType, parseOptions) + val coordinates = parsed.getField("coordinates") + val isValidPoint = parsed.getField("type") === lit("Point") && + coordinates.isNotNull && + size(coordinates) === lit(GeographyPointCoordinateCount) && + forall(coordinates, c => c.isNotNull) + val invalidValueError = raise_error(concat( + lit(s"AzureSearchWriter: column '$fieldName' is mapped to an " + + s"$GeographyPointEdmType field but the value is not a valid GeoJSON Point " + + """(expected {"type":"Point","coordinates":[longitude,latitude]}). """ + + "Offending value: "), + col(fieldName))) + currentDF.withColumn(fieldName, + when(col(fieldName).isNull || isValidPoint, parsed) + .otherwise(invalidValueError.cast(geoStructType)) + ) + case _ => + // Already a struct (or otherwise compatible); checkSchemaParity will validate. + currentDF + } + } else { + currentDF + } + } + } + private def dfToIndexJson(schema: StructType, indexName: String, keyCol: String, @@ -278,6 +347,34 @@ object AzureSearchWriter extends IndexParser with IndexJsonGetter with SLogging documents } + private def resolveIndexDefinition(existingIndexJsonOpt: Option[String], + indexJsonOpt: Option[String], + vectorColsInfo: Option[String], + df: DataFrame, + indexName: String, + keyCol: Option[String], + actionCol: String): (String, DataFrame) = { + existingIndexJsonOpt match { + case Some(existingIndexJson) => + val vectorColNameTypeTuple = getVectorColConf(existingIndexJson) + (existingIndexJson, makeColsCompatible(vectorColNameTypeTuple, df)) + case None => + indexJsonOpt match { + case Some(indexJson) => + val vectorColNameTypeTuple = getVectorColConf(indexJson) + (indexJson, makeColsCompatible(vectorColNameTypeTuple, df)) + case None => + val vectorCols = vectorColsInfo.map(parseVectorColsJson) + val vectorColNameTypeTuple = vectorCols + .map(_.map(vc => (vc.name, "Collection(Edm.Single)"))).getOrElse(Seq.empty) + val newDF = makeColsCompatible(vectorColNameTypeTuple, df) + val inferredIndexJson = dfToIndexJson( + newDF.schema, indexName, keyCol.getOrElse(""), actionCol, vectorCols) + (inferredIndexJson, newDF) + } + } + } + private def prepareDF(df: DataFrame, //scalastyle:ignore method.length options: Map[String, String] = Map()): DataFrame = { val applicableOptions = Set( @@ -316,45 +413,44 @@ object AzureSearchWriter extends IndexParser with IndexJsonGetter with SLogging } } - val (indexJson, preppedDF) = if (getExisting(auth, serviceName, apiVersion).contains(indexName)) { + val existingIndexJsonOpt = if (getExisting(auth, serviceName, apiVersion).contains(indexName)) { if (indexJsonOpt.isDefined) { println(f"indexJsonOpt is specified, however an index for $indexName already exists," + f"we will use the index definition obtained from the existing index instead") } - val existingIndexJson = getIndexJsonFromExistingIndex(auth, serviceName, indexName, apiVersion) - val vectorColNameTypeTuple = getVectorColConf(existingIndexJson) - (existingIndexJson, makeColsCompatible(vectorColNameTypeTuple, df)) - } else if (indexJsonOpt.isDefined) { - val vectorColNameTypeTuple = getVectorColConf(indexJsonOpt.get) - (indexJsonOpt.get, makeColsCompatible(vectorColNameTypeTuple, df)) + val existingIndexJson = IndexJsonReader.get(auth, serviceName, indexName, apiVersion) + VectorSchema.requireCompatibleExistingIndex(existingIndexJson.parseJson, apiVersion) + Some(existingIndexJson) } else { - val vectorCols = vectorColsInfo.map(parseVectorColsJson) - val vectorColNameTypeTuple = vectorCols.map(_.map(vc => (vc.name, "Collection(Edm.Single)"))).getOrElse(Seq.empty) - val newDF = makeColsCompatible(vectorColNameTypeTuple, df) - val inferredIndexJson = dfToIndexJson(newDF.schema, indexName, keyCol.getOrElse(""), actionCol, vectorCols) - (inferredIndexJson, newDF) + None } + val (indexJson, preppedDF) = resolveIndexDefinition( + existingIndexJsonOpt, indexJsonOpt, vectorColsInfo, df, indexName, keyCol, actionCol) + // TODO: Support vector search in nested fields // Throws an exception if any nested field is a vector in the schema parseIndexJson(indexJson).fields.foreach(_.fields.foreach(assertNoNestedVectors)) - SearchIndex.createIfNoneExists(auth, serviceName, indexJson, apiVersion) + if (existingIndexJsonOpt.isEmpty) { + SearchIndex.createIfNoneExists(auth, serviceName, indexJson, apiVersion) + } val dateConvertedDF = convertDateTimeToISO8601(preppedDF, indexJson) + val geoConvertedDF = convertGeographyPointToStruct(dateConvertedDF, indexJson) logInfo("checking schema parity") - checkSchemaParity(dateConvertedDF.schema, indexJson, actionCol) + checkSchemaParity(geoConvertedDF.schema, indexJson, actionCol) val df1 = if (filterNulls) { val collectionColumns = parseIndexJson(indexJson).fields .filter(_.`type`.startsWith("Collection")) .map(_.name) - collectionColumns.foldLeft(dateConvertedDF) { (ndf, c) => filterOutNulls(ndf, c) } + collectionColumns.foldLeft(geoConvertedDF) { (ndf, c) => filterOutNulls(ndf, c) } } else { - dateConvertedDF + geoConvertedDF } - // Convert date/timestamp columns to ISO8601 strings for Azure Search + // Convert date/timestamp columns to ISO8601 strings for Azure AI Search val addDocuments = configureAuthentication( new AddDocuments() @@ -363,7 +459,11 @@ object AzureSearchWriter extends IndexParser with IndexJsonGetter with SLogging .setActionCol(actionCol) .setBatchSize(batchSize) .setOutputCol("out") - .setErrorCol("error"), + .setErrorCol("error") + // Pin the document endpoint to the same api-version used to create/read the index, otherwise + // an explicit apiVersion option would only apply to the index APIs. + .setUrl(s"https://$serviceName.search.windows.net" + + s"/indexes/$indexName/docs/index?api-version=$apiVersion"), auth) addDocuments.transform(df1) @@ -373,7 +473,7 @@ object AzureSearchWriter extends IndexParser with IndexJsonGetter with SLogging private def assertNoNestedVectors(fields: Seq[IndexField]): Unit = { def checkVectorField(field: IndexField): Unit = { - if (field.dimensions.nonEmpty && field.vectorSearchConfiguration.nonEmpty) { + if (field.isVectorField) { throw new IllegalArgumentException(s"Nested field ${field.name} is a vector field, vector fields in nested" + s" fields are not supported.") } @@ -384,7 +484,7 @@ object AzureSearchWriter extends IndexParser with IndexJsonGetter with SLogging private def getVectorColConf(indexJson: String): Seq[(String, String)] = { parseIndexJson(indexJson).fields - .filter(f => f.vectorSearchConfiguration.nonEmpty && f.dimensions.nonEmpty) + .filter(_.isVectorField) .map(f => (f.name, f.`type`)) } private def makeColsCompatible(vectorColNameTypeTuple: Seq[(String, String)], @@ -421,6 +521,11 @@ object AzureSearchWriter extends IndexParser with IndexJsonGetter with SLogging t.substring("Collection(".length).dropRight(1) } + private[ml] val GeographyPointEdmType = "Edm.GeographyPoint" + + // GeoJSON Points are always [longitude, latitude] + private[ml] val GeographyPointCoordinateCount = 2 + private[ml] def edmTypeToSparkType(dt: String, //scalastyle:ignore cyclomatic.complexity fields: Option[Seq[IndexField]]): DataType = dt match { case t if isEdmCollection(t) => @@ -432,7 +537,7 @@ object AzureSearchWriter extends IndexParser with IndexJsonGetter with SLogging case "Edm.Double" => DoubleType case "Edm.Single" => FloatType case "Edm.DateTimeOffset" => StringType // We convert date/time to ISO8601 strings - case "Edm.GeographyPoint" => + case GeographyPointEdmType => StructType(Seq( StructField("type", StringType), StructField("coordinates", ArrayType(DoubleType)) diff --git a/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/search/AzureSearchAPI.scala b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/search/AzureSearchAPI.scala index 5114b8bb990..885b3e7d3eb 100644 --- a/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/search/AzureSearchAPI.scala +++ b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/search/AzureSearchAPI.scala @@ -6,15 +6,53 @@ package com.microsoft.azure.synapse.ml.services.search import com.microsoft.azure.synapse.ml.services.search.AzureSearchProtocol._ import com.microsoft.azure.synapse.ml.io.http.RESTHelpers._ import org.apache.commons.io.IOUtils +import org.apache.http.client.methods.{CloseableHttpResponse, HttpRequestBase} import org.apache.log4j.{LogManager, Logger} import spray.json._ +import java.time.LocalDate +import java.time.format.DateTimeParseException import scala.util.{Failure, Success, Try} object AzureSearchAPIConstants { - val DefaultAPIVersion = "2023-07-01-Preview" + /** Latest generally available Azure AI Search (formerly Azure Cognitive Search) data plane API version. + * + * The previous default, `2023-07-01-Preview`, was deprecated on 2024-04-08 and has been out of + * support since 2024-07-08. + */ + val DefaultAPIVersion = "2026-04-01" val VectorConfigName = "vectorConfig" val VectorSearchAlgorithm = "hnsw" + + /** First API version that replaced `vectorSearch.algorithmConfigurations` with + * `vectorSearch.algorithms` + `vectorSearch.profiles`, and field-level + * `vectorSearchConfiguration` with `vectorSearchProfile`. + */ + val VectorProfileMinAPIVersion = "2023-10-01-Preview" + private val VectorProfileMinAPIDate = LocalDate.parse("2023-10-01") + private val APIVersion = """^(\d{4}-\d{2}-\d{2})(?:-[A-Za-z0-9.-]+)?$""".r + + /** True when the api-version expects the profile-based vector schema. + * + * API versions are parsed as `yyyy-MM-dd` optionally followed by a preview suffix. Invalid values + * fail before a request is sent rather than being assigned a schema generation by string ordering. + */ + def supportsVectorProfiles(apiVersion: String): Boolean = { + val version = Option(apiVersion).map(_.trim).filter(_.nonEmpty).getOrElse { + throw new IllegalArgumentException("Azure AI Search apiVersion must be non-empty") + } + val date = version match { + case APIVersion(dateText) => + try { + LocalDate.parse(dateText) + } catch { + case _: DateTimeParseException => + throw new IllegalArgumentException(s"Invalid Azure AI Search apiVersion: $apiVersion") + } + case _ => throw new IllegalArgumentException(s"Invalid Azure AI Search apiVersion: $apiVersion") + } + !date.isBefore(VectorProfileMinAPIDate) + } } import com.microsoft.azure.synapse.ml.services.search.AzureSearchAPIConstants._ @@ -49,6 +87,27 @@ trait IndexLister { } } +private[search] object IndexJsonReader { + def get(auth: AzureSearchAuth, + serviceName: String, + indexName: String, + apiVersion: String): String = { + read( + AzureSearchRequests.getIndex(auth, serviceName, indexName, apiVersion), + request => safeSend(request, close = false)) + } + + private[search] def read(request: HttpRequestBase, + send: HttpRequestBase => CloseableHttpResponse): String = { + val response = send(request) + try { + IOUtils.toString(response.getEntity.getContent, "utf-8") + } finally { + response.close() + } + } +} + trait IndexJsonGetter extends IndexLister { def getIndexJsonFromExistingIndex(key: String, serviceName: String, @@ -70,10 +129,44 @@ trait IndexJsonGetter extends IndexLister { val existingIndexNames = getExisting(auth, serviceName, apiVersion) assert(existingIndexNames.contains(indexName), s"Cannot find an existing index name with $indexName") - val response = safeSend( - AzureSearchRequests.getIndex(auth, serviceName, indexName, apiVersion), close = false) + IndexJsonReader.get(auth, serviceName, indexName, apiVersion) + } +} + +private[search] trait SearchIndexClient { + def getExisting(auth: AzureSearchAuth, serviceName: String, apiVersion: String): Seq[String] + + def getIndexJson(auth: AzureSearchAuth, + serviceName: String, + indexName: String, + apiVersion: String): String + + def createIndex(auth: AzureSearchAuth, + serviceName: String, + indexJson: String, + apiVersion: String): Int +} + +private object DefaultSearchIndexClient extends SearchIndexClient { + override def getExisting(auth: AzureSearchAuth, + serviceName: String, + apiVersion: String): Seq[String] = + SearchIndex.getExisting(auth, serviceName, apiVersion) + + override def getIndexJson(auth: AzureSearchAuth, + serviceName: String, + indexName: String, + apiVersion: String): String = + IndexJsonReader.get(auth, serviceName, indexName, apiVersion) + + override def createIndex(auth: AzureSearchAuth, + serviceName: String, + indexJson: String, + apiVersion: String): Int = { + val request = AzureSearchRequests.createIndex(auth, serviceName, indexJson, apiVersion) + val response = safeSend(request, close = false) try { - IOUtils.toString(response.getEntity.getContent, "utf-8") + response.getStatusLine.getStatusCode } finally { response.close() } @@ -103,56 +196,67 @@ object SearchIndex extends IndexParser with IndexLister { serviceName: String, indexJson: String, apiVersion: String): Unit = { + createIfNoneExists(auth, serviceName, indexJson, apiVersion, DefaultSearchIndexClient) + } + + private[search] def createIfNoneExists(auth: AzureSearchAuth, + serviceName: String, + indexJson: String, + apiVersion: String, + client: SearchIndexClient): Unit = { + AzureSearchAPIConstants.supportsVectorProfiles(apiVersion) val indexName = parseIndexJson(indexJson).name.get - val existingIndexNames = getExisting(auth, serviceName, apiVersion) + val existingIndexNames = client.getExisting(auth, serviceName, apiVersion) if (!existingIndexNames.contains(indexName)) { - val request = AzureSearchRequests.createIndex(auth, serviceName, prepareEntity(indexJson), apiVersion) - val response = safeSend(request, close = false) - try { - assert(response.getStatusLine.getStatusCode == 201) - } finally { - response.close() - } + val statusCode = client.createIndex(auth, serviceName, prepareEntity(indexJson, apiVersion), apiVersion) + assert(statusCode == 201) + } else { + val existingIndexJson = client.getIndexJson(auth, serviceName, indexName, apiVersion) + VectorSchema.requireCompatibleExistingIndex(existingIndexJson.parseJson, apiVersion) } } - private def prepareEntity(indexJson: String): String = { - validIndexJson(indexJson).get + private[search] def prepareEntity(indexJson: String, apiVersion: String): String = { + validIndexJson(indexJson, apiVersion).get } // validate schema - private def validIndexJson(indexJson: String): Try[String] = { - validateIndexInfo(indexJson).map(_.toJson.compactPrint) + private def validIndexJson(indexJson: String, apiVersion: String): Try[String] = { + Try(VectorSchema.align(indexJson.parseJson, apiVersion)).flatMap { aligned => + validateIndexInfo(aligned, apiVersion).map(_ => aligned.compactPrint) + } } - private def validateIndexInfo(indexJson: String): Try[IndexInfo] = { - val schema = parseIndexJson(indexJson) - for { - _ <- validName(schema.name.get) - _ <- validIndexFields(schema.fields) - } yield schema + private def validateIndexInfo(indexJson: JsValue, apiVersion: String): Try[IndexInfo] = { + Try(indexJson.convertTo[IndexInfo]).flatMap { schema => + for { + _ <- validName(schema.name.get) + _ <- validIndexFields(schema.fields, AzureSearchAPIConstants.supportsVectorProfiles(apiVersion)) + } yield schema + } } - private def validIndexField(field: IndexField): Try[IndexField] = { + private def validIndexField(field: IndexField, supportsVectorProfiles: Boolean): Try[IndexField] = { for { _ <- validName(field.name) _ <- validType(field.`type`, field.fields) - _ <- validSearchable(field.`type`, field.searchable) + _ <- validSearchable(field.`type`, field.searchable, field.dimensions, supportsVectorProfiles) _ <- validSortable(field.`type`, field.sortable) _ <- validFacetable(field.`type`, field.facetable) _ <- validKey(field.`type`, field.key) _ <- validAnalyzer(field.analyzer, field.searchAnalyzer, field.indexAnalyzer) _ <- validSearchAnalyzer(field.analyzer, field.searchAnalyzer, field.indexAnalyzer) _ <- validIndexAnalyzer(field.analyzer, field.searchAnalyzer, field.indexAnalyzer) - _ <- validVectorField(field.dimensions, field.vectorSearchConfiguration) + _ <- validVectorField(field.dimensions, field.vectorReference) // TODO: Fix and add back validSynonymMaps check. SynonymMaps needs to be Option[Seq[String]] type //_ <- validSynonymMaps(field.synonymMap) } yield field } - private def validIndexFields(fields: Seq[IndexField]): Try[Seq[IndexField]] = { - Try(fields.map(f => validIndexField(f).get)) + private def validIndexFields(fields: Seq[IndexField], + supportsVectorProfiles: Boolean): Try[Seq[IndexField]] = { + Try(fields.map(f => validIndexField(f, supportsVectorProfiles).get)) } private def validName(n: String): Try[String] = { @@ -166,8 +270,13 @@ object SearchIndex extends IndexParser with IndexLister { tdt.map(_ => t) } - private def validSearchable(t: String, s: Option[Boolean]): Try[Option[Boolean]] = { - if (Set("Edm.String", "Collection(Edm.String)")(t)) { + private def validSearchable(t: String, + s: Option[Boolean], + dimensions: Option[Int], + supportsVectorProfiles: Boolean): Try[Option[Boolean]] = { + if (dimensions.nonEmpty) { + validVectorSearchable(s, supportsVectorProfiles) + } else if (Set("Edm.String", "Collection(Edm.String)")(t)) { Success(s) } else if (s.contains(true)) { Failure(new IllegalArgumentException("Only Edm.String and Collection(Edm.String) fields can be searchable")) @@ -176,6 +285,23 @@ object SearchIndex extends IndexParser with IndexLister { } } + private def validVectorSearchable(s: Option[Boolean], + supportsVectorProfiles: Boolean): Try[Option[Boolean]] = { + if (supportsVectorProfiles) { + if (s.contains(true)) { + Success(s) + } else { + Failure(new IllegalArgumentException( + "Vector fields must set searchable=true for api-version 2023-10-01-Preview and later")) + } + } else if (s.contains(true)) { + Failure(new IllegalArgumentException( + "Legacy vector fields cannot set searchable=true; use api-version 2023-10-01-Preview or later")) + } else { + Success(s) + } + } + private def validSortable(t: String, s: Option[Boolean]): Try[Option[Boolean]] = { if (t == "Collection(Edm.String)" & s.contains(true)) { Failure(new IllegalArgumentException("Collection(Edm.String) fields cannot be sortable")) @@ -237,8 +363,8 @@ object SearchIndex extends IndexParser with IndexLister { private def validVectorField(d: Option[Int], v: Option[String]): Try[Option[String]] = { if ((d.isDefined && v.isEmpty) || (v.isDefined && d.isEmpty)) { - Failure(new IllegalArgumentException("Both dimensions and vectorSearchConfig fields need to be defined for " + - "vector search")) + Failure(new IllegalArgumentException("Both dimensions and vectorSearchProfile (or the legacy " + + "vectorSearchConfiguration) fields need to be defined for vector search")) } else { Success(v) } diff --git a/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/search/AzureSearchAuth.scala b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/search/AzureSearchAuth.scala index a1441cbd3c4..01e186bccb9 100644 --- a/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/search/AzureSearchAuth.scala +++ b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/search/AzureSearchAuth.scala @@ -40,7 +40,7 @@ final case class AzureSearchAuth(subscriptionKey: Option[String] = None, } require( auth.subscriptionKey.nonEmpty || auth.aadToken.nonEmpty || auth.customAuthHeader.nonEmpty || customCredential, - "Azure Search authentication requires subscriptionKey, AADToken, CustomAuthHeader, " + + "Azure AI Search authentication requires subscriptionKey, AADToken, CustomAuthHeader, " + "or an api-key/Authorization custom header") auth } @@ -60,7 +60,7 @@ final case class AzureSearchAuth(subscriptionKey: Option[String] = None, auth.aadToken, auth.customAuthHeader, Option(auth.customHeaders).filter(_.nonEmpty), - None, // Azure Search management requests have no automatic Fabric fallback + None, // Azure AI Search management requests have no automatic Fabric fallback None, if (addContentType) Some("application/json") else None) } @@ -80,7 +80,7 @@ object AzureSearchAuth { // everywhere else) never conflicts with a valid sibling. Values are compared verbatim -- never // trimmed -- and the failure names only the option keys, never their (credential) values. val values = names.flatMap(options.get).filter(ServiceAuthHeaders.nonBlank).distinct - require(values.size <= 1, s"Conflicting Azure Search options: ${names.mkString(" and ")}") + require(values.size <= 1, s"Conflicting Azure AI Search options: ${names.mkString(" and ")}") values.headOption } diff --git a/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/search/AzureSearchSchemas.scala b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/search/AzureSearchSchemas.scala index 82ca48c256f..a0267a28a3b 100644 --- a/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/search/AzureSearchSchemas.scala +++ b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/search/AzureSearchSchemas.scala @@ -5,7 +5,7 @@ package com.microsoft.azure.synapse.ml.services.search import com.microsoft.azure.synapse.ml.core.schema.SparkBindings import spray.json.DefaultJsonProtocol._ -import spray.json.{DefaultJsonProtocol, JsonFormat, RootJsonFormat} +import spray.json._ object ASResponses extends SparkBindings[ASResponses] @@ -13,17 +13,24 @@ case class ASResponses(value: Seq[ASResponse]) case class ASResponse(key: String, status: Boolean, errorMessage: Option[String], statusCode: Int) +// These six fields are pure pass-through: the writer round-trips them to the service and never +// inspects their contents. The Search REST API returns objects for all of them (a custom analyzer +// is {"@odata.type":"#Microsoft.Azure.Search.CustomAnalyzer",...}, corsOptions is a single object), +// so typing them as strings made any index using these features fail to deserialize. Keeping them +// as opaque JsValue also stops the library from breaking when the service adds a new analyzer kind. +// spray.json.JsValue is written out in full because the release branches this change is replayed +// onto import spray.json explicitly rather than by wildcard. case class IndexInfo( name: Option[String], fields: Seq[IndexField], - suggesters: Option[Seq[String]], + suggesters: Option[Seq[spray.json.JsValue]], scoringProfiles: Option[Seq[ScoringProfile]], - analyzers: Option[Seq[String]], - charFilters: Option[Seq[String]], - tokenizers: Option[Seq[String]], - tokenFilters: Option[Seq[String]], + analyzers: Option[Seq[spray.json.JsValue]], + charFilters: Option[Seq[spray.json.JsValue]], + tokenizers: Option[Seq[spray.json.JsValue]], + tokenFilters: Option[Seq[spray.json.JsValue]], defaultScoringProfile: Option[String], - corsOptions: Option[Seq[String]], + corsOptions: Option[spray.json.JsValue], vectorSearch: Option[VectorSearch] ) @@ -52,7 +59,16 @@ case class IndexField( fields: Option[Seq[IndexField]], dimensions: Option[Int], vectorSearchConfiguration: Option[String] - ) + ) { + def vectorReference: Option[String] = vectorSearchConfiguration + + // A field is a vector field if it declares dimensions. The algorithm/profile reference + // describes *how* the vectors are indexed, not *whether* the field holds vectors, and the + // service does not always surface it: reading a legacy index under a modern api-version + // returns `dimensions` with a null `vectorSearchProfile` and an empty `profiles` list. + // Requiring a reference here would silently stop treating those fields as vectors. + def isVectorField: Boolean = dimensions.nonEmpty +} case class VectorColParams( name: String, @@ -100,12 +116,28 @@ case class IndexList(`@odata.context`: String, value: Seq[IndexName]) case class IndexName(name: String) object AzureSearchProtocol extends DefaultJsonProtocol { - implicit val IfEnc: JsonFormat[IndexField] = lazyFormat(jsonFormat( - IndexField,"name","type","searchable","filterable","sortable", - "facetable","retrievable", "key","analyzer","searchAnalyzer", "indexAnalyzer", "synonymMaps", "fields", - "dimensions", "vectorSearchConfiguration")) implicit val AcEnc: RootJsonFormat[AlgorithmConfigs] = jsonFormat2(AlgorithmConfigs.apply) - implicit val VsEnc: RootJsonFormat[VectorSearch] = jsonFormat1(VectorSearch.apply) + implicit val VsEnc: RootJsonFormat[VectorSearch] = { + val legacyFormat = jsonFormat1(VectorSearch.apply) + new RootJsonFormat[VectorSearch] { + override def read(json: JsValue): VectorSearch = + legacyFormat.read(VectorSchema.legacyVectorSearchView(json)) + + override def write(vectorSearch: VectorSearch): JsValue = legacyFormat.write(vectorSearch) + } + } + implicit val IfEnc: JsonFormat[IndexField] = { + val legacyFormat = lazyFormat(jsonFormat( + IndexField, "name", "type", "searchable", "filterable", "sortable", + "facetable", "retrievable", "key", "analyzer", "searchAnalyzer", "indexAnalyzer", "synonymMaps", "fields", + "dimensions", "vectorSearchConfiguration")) + new JsonFormat[IndexField] { + override def read(json: JsValue): IndexField = + legacyFormat.read(VectorSchema.legacyIndexFieldView(json)) + + override def write(field: IndexField): JsValue = legacyFormat.write(field) + } + } implicit val FfEnc: RootJsonFormat[FreshnessFunction] = jsonFormat1(FreshnessFunction.apply) implicit val MfEnc: RootJsonFormat[MagnitudeFunction] = jsonFormat3(MagnitudeFunction.apply) implicit val DfEnc: RootJsonFormat[DistanceFunction] = jsonFormat2(DistanceFunction.apply) @@ -119,3 +151,205 @@ object AzureSearchProtocol extends DefaultJsonProtocol { implicit val IlEnc: RootJsonFormat[IndexList] = jsonFormat2(IndexList.apply) implicit val VcpEnc: RootJsonFormat[VectorColParams] = jsonFormat2(VectorColParams.apply) } + +/** Translates index JSON between the two generations of the Azure AI Search vector schema. + * + * `2023-10-01-Preview` renamed `vectorSearch.algorithmConfigurations` to `vectorSearch.algorithms`, added + * `vectorSearch.profiles`, and replaced the field-level `vectorSearchConfiguration` with + * `vectorSearchProfile`. The public Scala case classes retain their original constructor and extractor + * shapes; REST payload modernization happens on the JSON AST so parameters, vectorizers, compressions, + * and fields introduced by future service versions are not discarded. + */ +private[search] object VectorSchema { + + private val LegacyAlgorithmsKey = "algorithmConfigurations" + private val ModernAlgorithmsKey = "algorithms" + private val ProfilesKey = "profiles" + private val LegacyFieldReferenceKey = "vectorSearchConfiguration" + private val ModernFieldReferenceKey = "vectorSearchProfile" + + def align(index: JsValue, apiVersion: String): JsValue = { + if (AzureSearchAPIConstants.supportsVectorProfiles(apiVersion)) toProfileSchema(index) + else toLegacySchema(index) + } + + def requireCompatibleExistingIndex(index: JsValue, apiVersion: String): Unit = { + val modern = AzureSearchAPIConstants.supportsVectorProfiles(apiVersion) + val legacyMarkers = vectorSearchContainsKey(index, LegacyAlgorithmsKey) || + containsFieldKey(index, LegacyFieldReferenceKey) + val modernMarkers = vectorSearchContainsKey(index, ModernAlgorithmsKey) || + vectorSearchContainsKey(index, ProfilesKey) || + containsFieldKey(index, ModernFieldReferenceKey) + + if (modern && legacyMarkers) { + throw new IllegalArgumentException( + "The index already exists with the legacy vector schema. createIfNoneExists does not update or migrate " + + "existing indexes. Migrate it with Azure AI Search Create or Update Index after reviewing the schema " + + "changes, or set an apiVersion earlier than 2023-10-01-Preview to keep using it as-is.") + } else if (!modern && modernMarkers) { + throw new IllegalArgumentException( + s"The index already exists with the profile-based vector schema, which is not supported by api-version " + + s"$apiVersion. Use apiVersion=2023-10-01-Preview or later.") + } + } + + private[search] def legacyVectorSearchView(json: JsValue): JsValue = + aliasForPublicModel(json, LegacyAlgorithmsKey, ModernAlgorithmsKey) + + private[search] def legacyIndexFieldView(json: JsValue): JsValue = + aliasForPublicModel(json, LegacyFieldReferenceKey, ModernFieldReferenceKey) + + private def aliasForPublicModel(json: JsValue, legacyKey: String, modernKey: String): JsValue = json match { + case JsObject(fields) => + (fields.get(legacyKey), fields.get(modernKey)) match { + case (Some(legacy), Some(modern)) if legacy != modern => + deserializationError(s"Conflicting $legacyKey and $modernKey values") + case (None, Some(modern)) => JsObject(fields - modernKey + (legacyKey -> modern)) + case _ => json + } + case _ => json + } + + private def toProfileSchema(index: JsValue): JsValue = index match { + case JsObject(indexFields) => + val (rewrittenVectorSearch, references) = indexFields.get("vectorSearch") match { + case Some(vectorSearch: JsObject) => modernizeVectorSearch(vectorSearch) + case Some(JsNull) | None => (indexFields.get("vectorSearch"), Map.empty[String, String]) + case Some(_) => throw new IllegalArgumentException("vectorSearch must be a JSON object") + } + val rewrittenFields = indexFields.get("fields").map(rewriteFields(_, references)) + JsObject(indexFields ++ rewrittenVectorSearch.map("vectorSearch" -> _) ++ rewrittenFields.map("fields" -> _)) + case _ => throw new IllegalArgumentException("Azure AI Search index JSON must be an object") + } + + private def modernizeVectorSearch(vectorSearch: JsObject): (Option[JsValue], Map[String, String]) = { + val fields = vectorSearch.fields + val upgradingLegacy = !fields.contains(ModernAlgorithmsKey) && fields.contains(LegacyAlgorithmsKey) + val algorithms = selectAlias(fields, ModernAlgorithmsKey, LegacyAlgorithmsKey) + val existingProfiles = fields.get(ProfilesKey).map(asArray(_, ProfilesKey)).getOrElse(Vector.empty) + val profileNames = existingProfiles.flatMap(profileValue(_, "name")).toSet + val profileByAlgorithm = existingProfiles.flatMap { profile => + for { + name <- profileValue(profile, "name") + algorithm <- profileValue(profile, "algorithm") + } yield algorithm -> name + }.toMap + + val generatedProfiles = if (upgradingLegacy) { + algorithms.toSeq.flatMap(asArray(_, ModernAlgorithmsKey)).flatMap { algorithm => + val algorithmName = objectString(algorithm, "name", ModernAlgorithmsKey) + if (profileByAlgorithm.contains(algorithmName)) { + None + } else if (profileNames.contains(algorithmName)) { + throw new IllegalArgumentException( + s"Cannot generate a vector profile for algorithm '$algorithmName' because that profile name is in use") + } else { + Some(JsObject("name" -> JsString(algorithmName), "algorithm" -> JsString(algorithmName))) + } + } + } else { + Seq.empty + } + val profiles = existingProfiles ++ generatedProfiles + val allProfileByAlgorithm = profiles.flatMap { profile => + for { + name <- profileValue(profile, "name") + algorithm <- profileValue(profile, "algorithm") + } yield algorithm -> name + }.toMap + val allProfileNames = profiles.flatMap(profileValue(_, "name")).toSet + val references = allProfileByAlgorithm ++ allProfileNames.map(name => name -> name) + + val rewritten = fields - LegacyAlgorithmsKey ++ algorithms.map(ModernAlgorithmsKey -> _) ++ + (if (profiles.nonEmpty) Some(ProfilesKey -> JsArray(profiles)) else None) + (Some(JsObject(rewritten)), references) + } + + private def toLegacySchema(index: JsValue): JsValue = { + if (vectorSearchContainsKey(index, ModernAlgorithmsKey) || vectorSearchContainsKey(index, ProfilesKey) || + containsFieldKey(index, ModernFieldReferenceKey)) { + throw new IllegalArgumentException( + "Profile-based vector JSON cannot be losslessly sent to a legacy Azure AI Search api-version. " + + "Use apiVersion=2023-10-01-Preview or later, or provide an explicit legacy definition using " + + "algorithmConfigurations and vectorSearchConfiguration.") + } + index + } + + private def rewriteFields(value: JsValue, + references: Map[String, String]): JsValue = value match { + case JsArray(fields) => JsArray(fields.map(rewriteField(_, references))) + case _ => throw new IllegalArgumentException("fields must be a JSON array") + } + + private def hasVectorDimensions(fields: Map[String, JsValue]): Boolean = + fields.get("dimensions").exists(_ != JsNull) + + private def rewriteField(value: JsValue, + references: Map[String, String]): JsValue = value match { + case JsObject(fields) => + val nested = fields.get("fields").map(rewriteFields(_, references)) + val reference = selectAlias(fields, ModernFieldReferenceKey, LegacyFieldReferenceKey).map { + case JsString(name) => + JsString(references.getOrElse(name, throw new IllegalArgumentException( + s"Cannot find or create a vector profile for algorithm '$name'"))) + case _ => throw new IllegalArgumentException( + s"$LegacyFieldReferenceKey and $ModernFieldReferenceKey must be strings") + } + val vectorField = hasVectorDimensions(fields) && reference.nonEmpty + JsObject(fields - LegacyFieldReferenceKey ++ reference.map(ModernFieldReferenceKey -> _) ++ + nested.map("fields" -> _) ++ (if (vectorField) Some("searchable" -> JsBoolean(true)) else None)) + case _ => throw new IllegalArgumentException("Each field definition must be a JSON object") + } + + private def selectAlias(fields: Map[String, JsValue], + preferredKey: String, + alternateKey: String): Option[JsValue] = { + (fields.get(preferredKey), fields.get(alternateKey)) match { + case (Some(preferred), Some(alternate)) if preferred != alternate => + throw new IllegalArgumentException(s"Conflicting $preferredKey and $alternateKey values") + case (Some(preferred), _) => Some(preferred) + case (_, alternate) => alternate + } + } + + private def asArray(value: JsValue, key: String): Vector[JsValue] = value match { + case JsArray(elements) => elements + case _ => throw new IllegalArgumentException(s"$key must be a JSON array") + } + + private def objectString(value: JsValue, key: String, owner: String): String = value match { + case JsObject(fields) => fields.get(key) match { + case Some(JsString(text)) => text + case _ => throw new IllegalArgumentException(s"Each $owner entry must contain a string $key") + } + case _ => throw new IllegalArgumentException(s"Each $owner entry must be a JSON object") + } + + private def profileValue(value: JsValue, key: String): Option[String] = value match { + case JsObject(fields) => fields.get(key).collect { case JsString(text) => text } + case _ => None + } + + private def vectorSearchContainsKey(index: JsValue, key: String): Boolean = index match { + case JsObject(fields) => fields.get("vectorSearch").exists { + case JsObject(vectorSearchFields) => vectorSearchFields.contains(key) + case _ => false + } + case _ => false + } + + private def containsFieldKey(index: JsValue, key: String): Boolean = index match { + case JsObject(fields) => fields.get("fields").exists(fieldArrayContainsKey(_, key)) + case _ => false + } + + private def fieldArrayContainsKey(value: JsValue, key: String): Boolean = value match { + case JsArray(elements) => elements.exists { + case JsObject(fields) => + fields.contains(key) || fields.get("fields").exists(fieldArrayContainsKey(_, key)) + case _ => false + } + case _ => false + } +} diff --git a/cognitive/src/test/java/mssparkutils/cognitiveService.java b/cognitive/src/test/java/mssparkutils/cognitiveService.java new file mode 100644 index 00000000000..fbcfde09ea8 --- /dev/null +++ b/cognitive/src/test/java/mssparkutils/cognitiveService.java @@ -0,0 +1,27 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package mssparkutils; + +public final class cognitiveService { + + private cognitiveService() { } + + public static String getEndpoint(String linkedServiceName) { + return "https://" + linkedServiceName + ".endpoint"; + } + + public static String getKey(String linkedServiceName) { + return "key-" + linkedServiceName; + } + + public static String getLocation(String linkedServiceName) { + if ("gov".equals(linkedServiceName)) { + return "usgovvirginia"; + } + if ("cn".equals(linkedServiceName)) { + return "chinanorth"; + } + return "eastus"; + } +} diff --git a/cognitive/src/test/python/synapsemltest/services/openai/test_OpenAICompletionDeprecated.py b/cognitive/src/test/python/synapsemltest/services/openai/test_OpenAICompletionDeprecated.py index eaece7350fb..075359b1d79 100644 --- a/cognitive/src/test/python/synapsemltest/services/openai/test_OpenAICompletionDeprecated.py +++ b/cognitive/src/test/python/synapsemltest/services/openai/test_OpenAICompletionDeprecated.py @@ -1,7 +1,6 @@ # Copyright (C) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See LICENSE in project root for information. -import importlib import sys import unittest import warnings @@ -27,18 +26,6 @@ def _has_openai_completion_warning(caught): class TestOpenAICompletionDeprecated(unittest.TestCase): - def test_package_import_without_deprecated_access_does_not_warn(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", FutureWarning) - package = importlib.import_module(_PACKAGE_NAME) - _clear_openai_completion_imports() - - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - importlib.reload(package) - - self.assertFalse(_has_openai_completion_warning(caught)) - def test_package_import_warns(self): _clear_openai_completion_imports() diff --git a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/core/utils/utils/ModelEqualitySuite.scala b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/core/utils/utils/ModelEqualitySuite.scala index b2bc5ed750f..4b021ed2707 100644 --- a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/core/utils/utils/ModelEqualitySuite.scala +++ b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/core/utils/utils/ModelEqualitySuite.scala @@ -12,6 +12,33 @@ import com.microsoft.azure.synapse.ml.stages.DropColumns class ModelEqualitySuite extends TestBase { spark + test("jaccardSimilarity grades partial overlap") { + // Sets of whole strings would collapse this to a 1.0/0.0 equality check. + assert(ModelEquality.jaccardSimilarity("abcd", "abcd") === 1.0) + assert(ModelEquality.jaccardSimilarity("abcd", "wxyz") === 0.0) + val partial = ModelEquality.jaccardSimilarity("the quick brown fox", "the quick brown cat") + assert(partial > 0.0 && partial < 1.0) + assert(partial > ModelEquality.jaccardSimilarity("the quick brown fox", "entirely unlike")) + } + + test("jaccardSimilarity is case insensitive and symmetric") { + assert(ModelEquality.jaccardSimilarity("Hello World", "hello world") === 1.0) + assert(ModelEquality.jaccardSimilarity("kitten", "sitting") + === ModelEquality.jaccardSimilarity("sitting", "kitten")) + assert(ModelEquality.jaccardSimilarity("", "") === 1.0) + } + + test("jaccardSimilarity handles strings shorter than one bigram") { + // sliding(2) keeps the short final window, so a 1-char string yields Set(char) + // rather than the empty set. Distinct 1-char strings must therefore score 0.0, + // not fall into the both-empty shortcut. + assert(ModelEquality.jaccardSimilarity("a", "b") === 0.0) + assert(ModelEquality.jaccardSimilarity("a", "a") === 1.0) + assert(ModelEquality.jaccardSimilarity("", "a") === 0.0) + assert(ModelEquality.jaccardSimilarity("a", "") === 0.0) + assert(ModelEquality.jaccardSimilarity("", "") === 1.0) + } + test("Complex param equality") { val m1 = new TextSentiment().setLocation("eastus") val m2 = new TextSentiment().setLocation("eastus") diff --git a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/CognitiveServiceBaseSuite.scala b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/CognitiveServiceBaseSuite.scala new file mode 100644 index 00000000000..46b1ef30150 --- /dev/null +++ b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/CognitiveServiceBaseSuite.scala @@ -0,0 +1,189 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.services + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import com.microsoft.azure.synapse.ml.param.ServiceParam +import org.apache.http.entity.AbstractHttpEntity +import org.apache.spark.ml.param.{ParamMap, Params} +import org.apache.spark.sql.Row +import spray.json.DefaultJsonProtocol._ + +private class ServiceParamHarness(override val uid: String = "serviceParamHarness") + extends Params with HasServiceParams { + + val requiredText: ServiceParam[String] = + new ServiceParam[String](this, "requiredText", "required text", isRequired = true) + + val optionalText: ServiceParam[String] = + new ServiceParam[String](this, "optionalText", "optional text") + + val urlVersion: ServiceParam[String] = + new ServiceParam[String](this, "urlVersion", "url version", isURLParam = true) + + def vectorParamMap: Map[String, String] = getVectorParamMap + + def requiredParamNames: Set[String] = getRequiredParams.map(_.name).toSet + + def urlParamNames: Set[String] = getUrlParams.map(_.name).toSet + + def shouldSkipRow(row: Row): Boolean = shouldSkip(row) + + def valueMap(row: Row, excludes: Set[ServiceParam[_]] = Set()): Map[String, Any] = getValueMap(row, excludes) + + def valueAnyOpt(row: Row, p: ServiceParam[_]): Option[Any] = getValueAnyOpt(row, p) + + override def copy(extra: ParamMap): Params = this +} + +private class LocationHarness(override val uid: String = "locationHarness") + extends Params with HasSetLocation { + + override def urlPath: String = "/v1/resource" + + override def copy(extra: ParamMap): Params = this +} + +private class CustomDomainHarness(override val uid: String = "customDomainHarness") + extends Params with HasCustomCogServiceDomain { + + override def urlPath: String = "/deployments/chat/completions" + + override private[ml] def internalServiceType: String = "openai" + + override def copy(extra: ParamMap): Params = this +} + +private class LinkedServiceHarness(override val uid: String = "linkedServiceHarness") + extends Params with HasSetLinkedService { + + override def urlPath: String = "/analyze" + + override def copy(extra: ParamMap): Params = this +} + +private class LinkedServiceLocationHarness(override val uid: String = "linkedServiceLocationHarness") + extends Params with HasSetLinkedServiceUsingLocation { + + override def urlPath: String = "/analyze" + + override def copy(extra: ParamMap): Params = this +} + +private class CognitiveInputHarness(override val uid: String = "cognitiveInputHarness") + extends Params with HasCognitiveServiceInput { + + val apiVersion: ServiceParam[String] = + new ServiceParam[String](this, "apiVersion", "api version", isURLParam = true) + + val requiredText: ServiceParam[String] = + new ServiceParam[String](this, "requiredText", "required text", isRequired = true) + + override protected def prepareEntity: Row => Option[AbstractHttpEntity] = _ => None + + def buildUrl(row: Row): String = prepareUrl(row) + + def headers(row: Row, addContentType: Boolean = true): Map[String, String] = getHeaders(row, addContentType) + + def shouldSkipRow(row: Row): Boolean = shouldSkip(row) + + override def copy(extra: ParamMap): Params = this +} + +class CognitiveServiceBaseSuite extends TestBase { + + import spark.implicits._ + + test("setLocation maps cloud domains deterministically") { + val service = new LocationHarness() + .setLocation("eastus") + assert(service.getUrl == "https://eastus.api.cognitive.microsoft.com/v1/resource") + + service.setLocation("usgovarizona") + assert(service.getUrl == "https://usgovarizona.api.cognitive.microsoft.us/v1/resource") + + service.setLocation("chinanorth") + assert(service.getUrl == "https://chinanorth.api.cognitive.microsoft.cn/v1/resource") + } + + test("custom domain helpers build deterministic urls") { + val service = new CustomDomainHarness() + .setCustomServiceName("contoso") + assert(service.getUrl == "https://contoso.cognitiveservices.azure.com/deployments/chat/completions") + + service.setEndpoint("https://custom.endpoint/") + assert(service.getUrl == "https://custom.endpoint/deployments/chat/completions") + + val internal = new CustomDomainHarness() + .setDefaultInternalEndpoint("https://fabric") + assert(internal.getOrDefault(internal.url) == "https://fabric/cognitive/openai/deployments/chat/completions") + } + + test("linked service setter resolves endpoint and key locally") { + val service = new LinkedServiceHarness() + .setLinkedService("demo") + + assert(service.getUrl == "https://demo.endpoint/analyze") + assert(service.getSubscriptionKey == "key-demo") + } + + test("linked service location setter resolves domain and key locally") { + val service = new LinkedServiceLocationHarness() + .setLinkedService("gov") + + assert(service.getUrl == "https://usgovvirginia.api.cognitive.microsoft.us/analyze") + assert(service.getSubscriptionKey == "key-gov") + } + + test("service param helper methods are deterministic") { + val harness = new ServiceParamHarness() + harness.setVectorParam(harness.requiredText, "requiredCol") + harness.setVectorParam("urlVersion", "versionCol") + harness.setScalarParam("optionalText", "fallback") + + val row = Seq(("hello", "2024-10-01")).toDF("requiredCol", "versionCol").head() + assert(harness.vectorParamMap == Map("requiredText" -> "requiredCol", "urlVersion" -> "versionCol")) + assert(harness.requiredParamNames == Set("requiredText")) + assert(harness.urlParamNames == Set("urlVersion")) + assert(!harness.shouldSkipRow(row)) + assert(harness.valueAnyOpt(row, harness.urlVersion).contains("2024-10-01")) + assert( + harness.valueMap(row, Set(harness.urlVersion)) == + Map("requiredText" -> "hello", "optionalText" -> "fallback") + ) + + val missingRequired = Seq((Option.empty[String], "2024-10-01")).toDF("requiredCol", "versionCol").head() + assert(harness.shouldSkipRow(missingRequired)) + } + + test("cognitive input helper methods build urls and headers locally") { + val input = new CognitiveInputHarness() + input.setUrl("https://example.test/root") + input.setVectorParam(input.requiredText, "textCol") + input.setVectorParam(input.apiVersion, "versionCol") + + val row = Seq(("hello", "2024-10-01")).toDF("textCol", "versionCol").head() + assert(input.buildUrl(row) == "https://example.test/root?apiVersion=2024-10-01") + assert(!input.shouldSkipRow(row)) + + input.setCustomUrlRoot("https://override.test/root") + assert(input.buildUrl(row) == "https://override.test/root") + + val subscriptionHeaders = new CognitiveInputHarness().setSubscriptionKey("sub-key").headers(Row.empty) + assert(subscriptionHeaders("Ocp-Apim-Subscription-Key") == "sub-key") + assert(subscriptionHeaders("Content-Type") == "application/json") + assert(!subscriptionHeaders.contains("Authorization")) + + val aadHeaders = new CognitiveInputHarness().setAADToken("aad-token").headers(Row.empty) + assert(aadHeaders("Authorization") == "Bearer aad-token") + + val customHeaders = new CognitiveInputHarness() + .setCustomAuthHeader("Shared custom-auth") + .setCustomHeaders(Map("X-Test" -> "1")) + .headers(Row.empty) + assert(customHeaders("Authorization") == "Shared custom-auth") + assert(customHeaders("X-Test") == "1") + assert(customHeaders.contains("x-ai-telemetry-properties")) + } +} diff --git a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/form/FormCoreOfflineSuite.scala b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/form/FormCoreOfflineSuite.scala new file mode 100644 index 00000000000..65f6423a8a3 --- /dev/null +++ b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/form/FormCoreOfflineSuite.scala @@ -0,0 +1,235 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.services.form + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import com.microsoft.azure.synapse.ml.io.http.ErrorUtils +import org.apache.http.client.utils.URLEncodedUtils +import org.apache.http.entity.AbstractHttpEntity +import org.apache.http.util.EntityUtils +import org.apache.spark.sql.Row +import org.apache.spark.sql.types.{ArrayType, DoubleType, StringType, StructField, StructType} +import spray.json.DefaultJsonProtocol._ +import spray.json._ + +import java.net.URI +import java.nio.charset.StandardCharsets +import scala.collection.JavaConverters._ + +class FormCoreOfflineSuite extends TestBase { + + private class ExposedAnalyzeLayout(uid: String) extends AnalyzeLayout(uid) { + def requestEntity(row: Row): Option[AbstractHttpEntity] = prepareEntity(row) + def requestUrl(row: Row): String = prepareUrl(row) + } + + private class ExposedAnalyzeDocument(uid: String) extends AnalyzeDocument(uid) { + def requestEntity(row: Row): Option[AbstractHttpEntity] = prepareEntity(row) + def requestUrl(row: Row): String = prepareUrl(row) + } + + private class ExposedGetCustomModel(uid: String) extends GetCustomModel(uid) { + def requestUrl(row: Row): String = prepareUrl(row) + } + + private class ExposedAnalyzeCustomModel(uid: String) extends AnalyzeCustomModel(uid) { + def requestUrl(row: Row): String = prepareUrl(row) + } + + private def entityToBody(entity: AbstractHttpEntity): String = EntityUtils.toString(entity) + + private def queryParams(url: String): Map[String, String] = { + URLEncodedUtils + .parse(new URI(url), StandardCharsets.UTF_8) + .asScala + .map(p => p.getName -> p.getValue) + .toMap + } + + test("Form Recognizer params validate allowed values offline") { + val receipts = new AnalyzeReceipts("offline-receipts").setLocale("en-US") + assert(receipts.getLocale == "en-US") + + val document = new AnalyzeDocument("offline-document") + .setStringIndexType("utf16CodeUnit") + .setFeatures(Seq("barcodes", "languages")) + assert(document.getStringIndexType == "utf16CodeUnit") + assert(document.getFeatures == Seq("barcodes", "languages")) + + intercept[IllegalArgumentException] { + receipts.setLocale("fr-FR") + } + intercept[IllegalArgumentException] { + document.setStringIndexType("bad-value") + } + intercept[IllegalArgumentException] { + document.setFeatures(Seq("barcodes", "unsupported")) + } + } + + test("Form recognizer request entities build deterministic url and byte payloads") { + val urlInput = "https://contoso.example/layout.jpg" + val bytesInput = Array[Byte](1, 2, 3) + + val layoutFromUrl = new ExposedAnalyzeLayout("layout-url").setImageUrl(urlInput) + val urlPayload = entityToBody(layoutFromUrl.requestEntity(Row.empty).get).parseJson.asJsObject + assert(urlPayload.fields("source").convertTo[String] == urlInput) + + val layoutFromBytes = new ExposedAnalyzeLayout("layout-bytes").setImageBytes(bytesInput) + val bodyBytes = EntityUtils.toByteArray(layoutFromBytes.requestEntity(Row.empty).get) + assert(bodyBytes.sameElements(bytesInput)) + } + + test("AnalyzeDocument builds v3 request url and payload from local params") { + val imageUrl = "https://contoso.example/form.pdf" + val analyzeDocument = new ExposedAnalyzeDocument("analyze-document-url") + .setLocation("eastus") + .setPrebuiltModelId("prebuilt-layout") + .setImageUrl(imageUrl) + .setPages("1-2") + .setStringIndexType("utf16CodeUnit") + .setFeatures(Seq("barcodes", "languages")) + + val url = analyzeDocument.requestUrl(Row.empty) + val uri = new URI(url) + val query = queryParams(url) + assert(uri.getPath.endsWith("/formrecognizer/documentModels/prebuilt-layout:analyze")) + assert(query("api-version") == "2023-07-31") + assert(query("pages") == "1-2") + assert(query("stringIndexType") == "utf16CodeUnit") + assert(query("features") == "List(barcodes, languages)") + + val requestBody = entityToBody(analyzeDocument.requestEntity(Row.empty).get).parseJson.asJsObject + assert(requestBody.fields("urlSource").convertTo[String] == imageUrl) + } + + test("custom model endpoints append model id deterministically") { + val getModel = new ExposedGetCustomModel("get-custom-model-url") + .setLocation("eastus") + .setModelId("model-123") + .setIncludeKeys(true) + val getModelUrl = getModel.requestUrl(Row.empty) + assert(new URI(getModelUrl).getPath.endsWith("/formrecognizer/v2.1/custom/models/model-123")) + assert(queryParams(getModelUrl)("includeKeys") == "true") + + val analyzeCustom = new ExposedAnalyzeCustomModel("analyze-custom-model-url") + .setLocation("eastus") + .setModelId("model-123") + assert(new URI(analyzeCustom.requestUrl(Row.empty)) + .getPath.endsWith("/formrecognizer/v2.1/custom/models/model-123/analyze")) + } + + test("form recognizer schemas expose deterministic output and error columns") { + spark + val inputSchema = StructType(Seq(StructField("id", StringType, nullable = true))) + + val analyzeDocumentSchema = new AnalyzeDocument("analyze-document-schema") + .setLocation("eastus") + .setPrebuiltModelId("prebuilt-read") + .setImageUrl("https://contoso.example/doc.png") + .setOutputCol("documentResult") + .setErrorCol("documentError") + .transformSchema(inputSchema) + assert(analyzeDocumentSchema("documentResult").dataType == AnalyzeDocumentResponse.schema) + assert(analyzeDocumentSchema("documentError").dataType == ErrorUtils.ErrorSchema) + + val analyzeLayoutSchema = new AnalyzeLayout("analyze-layout-schema") + .setLocation("eastus") + .setImageUrl("https://contoso.example/doc.png") + .setOutputCol("layoutResult") + .setErrorCol("layoutError") + .transformSchema(inputSchema) + assert(analyzeLayoutSchema("layoutResult").dataType == AnalyzeResponse.schema) + assert(analyzeLayoutSchema("layoutError").dataType == ErrorUtils.ErrorSchema) + } + + test("field result helpers convert recursive values and simplify data types") { + import FormsJsonProtocol._ + + val numberValue = FieldResultRecursive( + `type` = "number", + page = None, + confidence = None, + boundingBox = None, + text = Some("7"), + valueString = None, + valuePhoneNumber = None, + valueNumber = Some(7.0), + valueDate = None, + valueTime = None, + valueObject = None, + valueArray = None + ) + val textValue = FieldResultRecursive( + `type` = "string", + page = None, + confidence = None, + boundingBox = None, + text = None, + valueString = Some("widget"), + valuePhoneNumber = None, + valueNumber = None, + valueDate = None, + valueTime = None, + valueObject = None, + valueArray = None + ) + + val mixedArray = FieldResultRecursive( + `type` = "array", + page = None, + confidence = None, + boundingBox = None, + text = None, + valueString = None, + valuePhoneNumber = None, + valueNumber = None, + valueDate = None, + valueTime = None, + valueObject = None, + valueArray = Some(Seq(textValue, numberValue)) + ) + assert(mixedArray.toSimplifiedDataType == ArrayType(StringType)) + + val objectValue = FieldResult( + `type` = "object", + page = None, + confidence = None, + boundingBox = None, + text = None, + valueString = None, + valuePhoneNumber = None, + valueNumber = None, + valueDate = None, + valueTime = None, + valueObject = Some(Map("count" -> numberValue, "label" -> textValue).toJson.compactPrint), + valueArray = None + ).toFieldResultRecursive + + val objectType = objectValue.toSimplifiedDataType.asInstanceOf[StructType] + assert(objectType.fieldNames.toSet == Set("count", "label")) + val objectRow = objectValue.viewAsDataType(objectType).asInstanceOf[Row] + val objectValues = objectRow.toSeq.toSet + assert(objectValues.contains(Some(7.0))) + assert(objectValues.contains(Some("widget"))) + assert(numberValue.viewAsDataType(StringType) == "7") + assert(numberValue.viewAsDataType(DoubleType) == 7.0) + } + + test("FormOntologyLearner.combineDataTypes merges nested schemas deterministically") { + val left = StructType(Seq( + StructField("shared", StringType, nullable = true), + StructField("leftOnly", DoubleType, nullable = true) + )) + val right = StructType(Seq( + StructField("shared", DoubleType, nullable = true), + StructField("rightOnly", StringType, nullable = true) + )) + + val merged = FormOntologyLearner.combineDataTypes(left, right).asInstanceOf[StructType] + assert(merged.fieldNames.toSet == Set("shared", "leftOnly", "rightOnly")) + assert(merged("shared").dataType == StringType) + assert(FormOntologyLearner.combineDataTypes(ArrayType(StringType), ArrayType(DoubleType)) == ArrayType(StringType)) + } +} diff --git a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/geospatial/GeospatialCoreSuite.scala b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/geospatial/GeospatialCoreSuite.scala new file mode 100644 index 00000000000..d7becb8b103 --- /dev/null +++ b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/geospatial/GeospatialCoreSuite.scala @@ -0,0 +1,167 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.services.geospatial + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.http.client.methods.{HttpGet, HttpPost} +import org.apache.http.util.EntityUtils +import org.apache.spark.sql.Row +import org.apache.spark.sql.types.ArrayType + +import java.net.{URI, URLDecoder} + +private[geospatial] class TestableAddressGeocoder extends AddressGeocoder { + def buildRequest(row: Row): Option[HttpPost] = + inputFunc(row).map(_.asInstanceOf[HttpPost]) +} + +private[geospatial] class TestableReverseAddressGeocoder extends ReverseAddressGeocoder { + def buildRequest(row: Row): Option[HttpPost] = + inputFunc(row).map(_.asInstanceOf[HttpPost]) +} + +private[geospatial] class TestableCheckPointInPolygon extends CheckPointInPolygon { + def buildRequest(row: Row): Option[HttpGet] = + inputFunc(row).map(_.asInstanceOf[HttpGet]) +} + +class GeospatialCoreSuite extends TestBase { + + import spark.implicits._ + + private def toQueryMap(uri: URI): Map[String, String] = { + Option(uri.getRawQuery).toSeq.flatMap(_.split("&")).map { kv => + val pair = kv.split("=", 2) + val key = URLDecoder.decode(pair(0), "UTF-8") + val value = if (pair.length > 1) URLDecoder.decode(pair(1), "UTF-8") else "" + key -> value + }.toMap + } + + test("address geocoder builds deterministic request payload and query params") { + val request = new TestableAddressGeocoder() + .setSubscriptionKey("fake-key") + .setAddress(Seq("One Microsoft Way, Redmond", "400 Broad St, Seattle")) + .buildRequest(Row.empty) + .get + + val query = toQueryMap(request.getURI) + assert(request.getURI.getPath.endsWith("/search/address/batch/json")) + assert(query("api-version") == "1.0") + assert(query("subscription-key") == "fake-key") + assert(request.getFirstHeader("Content-Type").getValue == "application/json") + + val payload = EntityUtils.toString(request.getEntity, "UTF-8") + assert(payload.contains("?query=One+Microsoft+Way%2C+Redmond&limit=1")) + assert(payload.contains("?query=400+Broad+St%2C+Seattle&limit=1")) + } + + test("reverse geocoder builds deterministic request payload and query params") { + val request = new TestableReverseAddressGeocoder() + .setSubscriptionKey("fake-key") + .setLatitude(Seq(48.858561, 47.639765)) + .setLongitude(Seq(2.294911, -122.127896)) + .buildRequest(Row.empty) + .get + + val query = toQueryMap(request.getURI) + assert(request.getURI.getPath.endsWith("/search/address/reverse/batch/json")) + assert(query("api-version") == "1.0") + assert(query("subscription-key") == "fake-key") + assert(request.getFirstHeader("Content-Type").getValue == "application/json") + + val payload = EntityUtils.toString(request.getEntity, "UTF-8") + assert(payload.contains("?query=48.858561,2.294911&limit=1")) + assert(payload.contains("?query=47.639765,-122.127896&limit=1")) + } + + test("address and reverse schema behavior is deterministic") { + val addressInput = Seq(Seq("One Microsoft Way, Redmond")).toDF("address") + val addressSchema = new AddressGeocoder() + .setAddressCol("address") + .setOutputCol("output") + .setErrorCol("addressError") + .transformSchema(addressInput.schema) + assert(addressSchema.fieldNames.toSet == Set("address", "output", "addressError")) + assert(addressSchema("output").dataType == ArrayType(SearchAddressBatchItem.schema)) + + val reverseInput = Seq((Seq(47.6418), Seq(-122.1275))).toDF("latitude", "longitude") + val reverseSchema = new ReverseAddressGeocoder() + .setLatitudeCol("latitude") + .setLongitudeCol("longitude") + .setOutputCol("output") + .setErrorCol("reverseError") + .transformSchema(reverseInput.schema) + assert(reverseSchema.fieldNames.toSet == Set("latitude", "longitude", "output", "reverseError")) + assert(reverseSchema("output").dataType == ArrayType(ReverseSearchAddressBatchItem.schema)) + } + + test("geospatial transformers validate missing input columns locally") { + val addressInput = Seq(Seq("One Microsoft Way, Redmond")).toDF("address") + val addressError = intercept[AssertionError] { + new AddressGeocoder().setAddressCol("missingAddress").transformSchema(addressInput.schema) + } + assert(addressError.getMessage.contains("Could not find dynamic columns")) + assert(addressError.getMessage.contains("missingAddress")) + + val reverseInput = Seq((Seq(47.6418), Seq(-122.1275))).toDF("latitude", "longitude") + val reverseError = intercept[AssertionError] { + new ReverseAddressGeocoder() + .setLatitudeCol("latitude") + .setLongitudeCol("missingLongitude") + .transformSchema(reverseInput.schema) + } + assert(reverseError.getMessage.contains("Could not find dynamic columns")) + assert(reverseError.getMessage.contains("missingLongitude")) + } + + test("checkpoint helper logic, schema behavior, and retired transform are deterministic") { + val transformer = new TestableCheckPointInPolygon() + .setSubscriptionKey("fake-key") + .setGeography("us") + .setUserDataIdentifier("udid-1") + .setLatitude(47.6418) + .setLongitude(-122.1275) + + assert(transformer.getUrl == "https://us.atlas.microsoft.com/spatial/pointInPolygon/json") + assert(transformer.getLatitude == Seq(47.6418)) + assert(transformer.getLongitude == Seq(-122.1275)) + assert(transformer.getUserDataIdentifier == "udid-1") + + val request = transformer.buildRequest(Row.empty).get + val query = toQueryMap(request.getURI) + assert(request.getURI.getPath.endsWith("/spatial/pointInPolygon/json")) + assert(query("api-version") == "1.0") + assert(query("subscription-key") == "fake-key") + assert(query("udid") == "udid-1") + assert(query("lat").contains("47.6418")) + assert(query("lon").contains("-122.1275")) + + val input = Seq((Seq(47.6418), Seq(-122.1275), "udid-1")).toDF("latitude", "longitude", "udid") + val schema = new CheckPointInPolygon() + .setLatitudeCol("latitude") + .setLongitudeCol("longitude") + .setUserDataIdentifierCol("udid") + .setOutputCol("pointInPolygon") + .setErrorCol("pointInPolygonError") + .transformSchema(input.schema) + assert(schema.fieldNames.toSet == Set("latitude", "longitude", "udid", "pointInPolygon", "pointInPolygonError")) + assert(schema("pointInPolygon").dataType == PointInPolygonProcessResult.schema) + + val missingColumnError = intercept[AssertionError] { + new CheckPointInPolygon() + .setLatitudeCol("latitude") + .setLongitudeCol("missingLongitude") + .setUserDataIdentifierCol("udid") + .transformSchema(input.schema) + } + assert(missingColumnError.getMessage.contains("Could not find dynamic columns")) + assert(missingColumnError.getMessage.contains("missingLongitude")) + + val retiredError = intercept[UnsupportedOperationException] { + transformer.transform(input) + } + assert(retiredError.getMessage.contains("retired on September 30, 2025")) + } +} diff --git a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/language/AnalyzeTextCoreOfflineSuite.scala b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/language/AnalyzeTextCoreOfflineSuite.scala new file mode 100644 index 00000000000..c54c64db460 --- /dev/null +++ b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/language/AnalyzeTextCoreOfflineSuite.scala @@ -0,0 +1,159 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.services.language + +import com.microsoft.azure.synapse.ml.io.http.{ + EntityData, HTTPResponseData, ProtocolVersionData, StatusLineData +} +import org.apache.http.entity.StringEntity +import org.apache.http.util.EntityUtils +import org.apache.spark.sql.Row +import org.apache.spark.sql.types.DataType +import org.scalatest.funsuite.AnyFunSuite +import spray.json._ +import spray.json.DefaultJsonProtocol._ + +import java.net.URI + +private[language] class TestableAnalyzeText extends AnalyzeText { + def buildEntity(row: Row): StringEntity = prepareEntity(row).get.asInstanceOf[StringEntity] + + def outputSchema: DataType = responseDataType +} + +private[language] class TestableAnalyzeTextLRO extends AnalyzeTextLongRunningOperations { + def buildEntity(row: Row): StringEntity = prepareEntity(row).get.asInstanceOf[StringEntity] + + def outputSchema: DataType = responseDataType + + def buildPollingURI(uri: URI): URI = modifyPollingURI(uri) +} + +class AnalyzeTextCoreOfflineSuite extends AnyFunSuite { + + private def parseEntity(entity: StringEntity): JsObject = { + EntityUtils.toString(entity, "UTF-8").parseJson.asJsObject + } + + private def responseWithEntity(json: String): HTTPResponseData = { + HTTPResponseData( + headers = Array.empty, + entity = Some(EntityData( + json.getBytes("UTF-8"), + None, + None, + None, + isChunked = false, + isRepeatable = true, + isStreaming = false)), + statusLine = StatusLineData(ProtocolVersionData("HTTP", 1, 1), 200, "OK"), + locale = "en_US") + } + + test("local parameter validation rejects invalid values") { + intercept[IllegalArgumentException] { + new AnalyzeText().setKind("UnknownTask") + } + intercept[IllegalArgumentException] { + new AnalyzeText().setStringIndexType("invalid-index") + } + intercept[IllegalArgumentException] { + new AnalyzeTextLongRunningOperations().setSentenceCount(0) + } + intercept[IllegalArgumentException] { + new AnalyzeTextLongRunningOperations().setSortBy("Score") + } + intercept[IllegalArgumentException] { + new AnalyzeTextLongRunningOperations().setSummaryLength("tiny") + } + } + + test("analyze text request-building is deterministic for language detection") { + val transformer = new TestableAnalyzeText() + .setKind("LanguageDetection") + .setText(Seq("Hello", "")) + .setCountryHint("US") + .setModelVersion("2024-10-01") + .setLoggingOptOut(true) + + val payload = parseEntity(transformer.buildEntity(Row.empty)) + assert(payload.fields("kind").convertTo[String] == "LanguageDetection") + + val analysisInput = payload.fields("analysisInput").asJsObject + val JsArray(documents) = analysisInput.fields("documents") + assert(documents.length == 2) + assert(documents.head.asJsObject.fields("countryHint").convertTo[String] == "US") + assert(documents(1).asJsObject.fields("countryHint").convertTo[String] == "US") + assert(documents(1).asJsObject.fields("text").convertTo[String] == "") + + val params = payload.fields("parameters").asJsObject + assert(params.fields("loggingOptOut").convertTo[Boolean]) + assert(params.fields("modelVersion").convertTo[String] == "2024-10-01") + } + + test("schema behavior follows selected task kind") { + val analyze = new TestableAnalyzeText().setKind("EntityLinking") + assert(analyze.outputSchema == EntityLinkingResponse.schema) + analyze.setKind("SentimentAnalysis") + assert(analyze.outputSchema == SentimentResponse.schema) + + val lro = new TestableAnalyzeTextLRO() + .setKind(AnalysisTaskKind.CustomMultiLabelClassification) + assert(lro.outputSchema == CustomLabelJobState.schema) + lro.setKind(AnalysisTaskKind.Healthcare) + assert(lro.outputSchema == HealthcareJobState.schema) + } + + test("lro request-building captures helper options deterministically") { + val transformer = new TestableAnalyzeTextLRO() + .setKind(AnalysisTaskKind.EntityRecognition) + .setText(Seq("John Doe")) + .setLanguage("en") + .setModelVersion("2024-06-01") + .setStringIndexType("UnicodeCodePoint") + .setLoggingOptOut(true) + .setInclusionList(Seq("Person")) + .setOverlapPolicy("allowOverlap") + .setExcludeNormalizedValues(true) + + val payload = parseEntity(transformer.buildEntity(Row.empty)) + val analysisInput = payload.fields("analysisInput").asJsObject + val JsArray(documents) = analysisInput.fields("documents") + assert(documents.head.asJsObject.fields("language").convertTo[String] == "en") + + val JsArray(tasks) = payload.fields("tasks") + val parameters = tasks.head.asJsObject.fields("parameters").asJsObject + assert(parameters.fields("modelVersion").convertTo[String] == "2024-06-01") + assert(parameters.fields("stringIndexType").convertTo[String] == "UnicodeCodePoint") + assert(parameters.fields("inclusionList").convertTo[Seq[String]] == Seq("Person")) + assert( + parameters.fields("overlapPolicy").asJsObject.fields("policyKind").convertTo[String] == "allowOverlap") + assert( + parameters.fields("inferenceOptions").asJsObject.fields("excludeNormalizedValues").convertTo[Boolean]) + } + + test("helper logic is deterministic for polling uri, kind mapping, and response rewrite") { + assert(AnalysisTaskKind.getKindFromString("Healthcare") == AnalysisTaskKind.Healthcare) + val ex = intercept[IllegalArgumentException] { + AnalysisTaskKind.getKindFromString("Nope") + } + assert(ex.getMessage.contains("Invalid kind")) + + val uri = new URI("https://example.test/jobs/1?api-version=2023-04-01") + val noStats = new TestableAnalyzeTextLRO() + assert(noStats.buildPollingURI(uri) == uri) + noStats.setShowStats(true) + assert(noStats.buildPollingURI(uri).toString.endsWith("&showStats=true")) + + val raw = responseWithEntity("""{"class":"Top","nested":{"class":"Secondary"}}""") + val rewritten = new TestableAnalyzeTextLRO() + .setKind(AnalysisTaskKind.CustomSingleLabelClassification) + .modifyResponse(Some(raw)) + .get + val rewrittenBody = new String(rewritten.entity.get.content, "UTF-8") + assert(rewrittenBody.contains("\"classifications\":\"Top\"")) + assert(rewrittenBody.contains("\"classifications\":\"Secondary\"")) + assert(!rewrittenBody.contains("\"class\":")) + } +} diff --git a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/openai/OpenAICoreOfflineSuite.scala b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/openai/OpenAICoreOfflineSuite.scala new file mode 100644 index 00000000000..d5af8139f20 --- /dev/null +++ b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/openai/OpenAICoreOfflineSuite.scala @@ -0,0 +1,186 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.services.openai + +import org.apache.http.entity.StringEntity +import org.apache.http.util.EntityUtils +import org.apache.spark.sql.Row +import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema +import org.apache.spark.sql.types.{ArrayType, IntegerType, MapType, StringType, StructField, StructType} +import org.scalatest.funsuite.AnyFunSuite +import spray.json._ +import spray.json.DefaultJsonProtocol._ + +class OpenAICoreOfflineSuite extends AnyFunSuite { + + private val stringMessageSchema = StructType(Seq( + StructField("role", StringType, nullable = false), + StructField("content", StringType, nullable = true), + StructField("name", StringType, nullable = true) + )) + + private val compositeMessageSchema = StructType(Seq( + StructField("role", StringType, nullable = false), + StructField( + "content", + ArrayType( + MapType(StringType, StringType, valueContainsNull = true), + containsNull = false + ), + nullable = true + ), + StructField("name", StringType, nullable = true) + )) + + private val unsupportedContentSchema = StructType(Seq( + StructField("role", StringType, nullable = false), + StructField("content", IntegerType, nullable = false) + )) + + private def messageRow(role: String, content: String): Row = + new GenericRowWithSchema(Array[Any](role, content, ""), stringMessageSchema) + + private def compositeMessageRow(role: String, parts: Seq[Map[String, String]]): Row = + new GenericRowWithSchema(Array[Any](role, parts, ""), compositeMessageSchema) + + private def unsupportedMessageRow(role: String, content: Int): Row = + new GenericRowWithSchema(Array[Any](role, content), unsupportedContentSchema) + + private def parseEntity(entity: StringEntity): JsObject = + EntityUtils.toString(entity).parseJson.asJsObject + + test("encodeMessagesToMap supports text and composite message shapes") { + val chat = new OpenAIChatCompletion() + val compositeParts = Seq( + Map("type" -> "text", "text" -> "first"), + Map("type" -> "input_file", "filename" -> "example.txt") + ) + + val mapped = chat.encodeMessagesToMap(Seq( + messageRow("user", "hello"), + compositeMessageRow("assistant", compositeParts) + )) + + assert(mapped.head("role") == "user") + assert(mapped.head("content") == "hello") + val secondContent = mapped(1)("content").asInstanceOf[Seq[Map[String, Any]]] + assert(secondContent.head("type") == "text") + assert(secondContent(1)("type") == "input_file") + } + + test("encodeMessagesToMap rejects unsupported content types") { + val chat = new OpenAIChatCompletion() + val ex = intercept[IllegalArgumentException] { + chat.encodeMessagesToMap(Seq(unsupportedMessageRow("user", 123))) + } + assert(ex.getMessage.contains("Unsupported content type")) + } + + test("OpenAIChatCompletion getStringEntity collapses content parts into text") { + val chat = new OpenAIChatCompletion() + val messageParts = Seq( + Map("type" -> "text", "text" -> "Line one"), + Map("type" -> "input_file", "filename" -> "example.txt"), + Map("type" -> "text", "text" -> "Line two") + ) + + val entity = chat.getStringEntity( + Seq(compositeMessageRow("user", messageParts)), + Map("temperature" -> 0.0) + ) + + val payload = parseEntity(entity) + val JsArray(messages) = payload.fields("messages") + val content = messages.head.asJsObject.fields("content").convertTo[String] + + assert(content == "Line one\nLine two") + } + + test("OpenAIChatCompletion response_format wraps bare schemas and exposes type") { + val chat = new OpenAIChatCompletion() + chat.setResponseFormat(Map( + "name" -> "answer_schema", + "strict" -> true, + "schema" -> Map( + "type" -> "object", + "properties" -> Map("answer" -> Map("type" -> "string")) + ) + )) + + val responseFormat = chat.getResponseFormat + assert(chat.getResponseFormatType == "json_schema") + assert(responseFormat("type") == "json_schema") + val jsonSchema = responseFormat("json_schema").asInstanceOf[Map[String, Any]] + assert(jsonSchema("name") == "answer_schema") + assert(jsonSchema.contains("schema")) + } + + test("OpenAIResponses optional params merge text/reasoning and drop gpt-5 sampling") { + val responses = new OpenAIResponses() + .setDeploymentName("gpt-5-mini") + .setTemperature(0.3) + .setTopP(0.7) + .setResponseFormat("json_object") + .setVerbosity("high") + .setReasoningEffort("medium") + + val params = responses.getOptionalParams(messageRow("user", "hello")) + + assert(params("model") == "gpt-5-mini") + assert(!params.contains("temperature")) + assert(!params.contains("top_p")) + assert(!params.contains("reasoning_effort")) + + val text = params("text").asInstanceOf[Map[String, Any]] + val format = text("format").asInstanceOf[Map[String, Any]] + assert(format("type") == "json_object") + assert(text("verbosity") == "high") + + val reasoning = params("reasoning").asInstanceOf[Map[String, Any]] + assert(reasoning("effort") == "medium") + } + + test("OpenAIResponses keeps sampling params for non-gpt5 deployments") { + val responses = new OpenAIResponses() + .setDeploymentName("gpt-4.1-mini") + .setTemperature(0.2) + .setTopP(0.6) + + val params = responses.getOptionalParams(messageRow("user", "hello")) + + assert(params("model") == "gpt-4.1-mini") + assert(params("temperature") == 0.2) + assert(params("top_p") == 0.6) + } + + test("OpenAIResponses getStringEntity wraps plain text and preserves composite parts") { + val responses = new OpenAIResponses() + val compositeParts = Seq( + Map("type" -> "input_file", "filename" -> "example.txt", "file_data" -> "AAA") + ) + + val entity = responses.getStringEntity( + Seq( + messageRow("user", "plain text"), + compositeMessageRow("user", compositeParts) + ), + Map("model" -> "gpt-4.1-mini") + ) + + val payload = parseEntity(entity) + val JsArray(inputs) = payload.fields("input") + + val JsArray(firstContent) = inputs.head.asJsObject.fields("content") + assert(firstContent.head.asJsObject.fields("type").convertTo[String] == "input_text") + assert(firstContent.head.asJsObject.fields("text").convertTo[String] == "plain text") + + val JsArray(secondContent) = inputs(1).asJsObject.fields("content") + assert(secondContent.head.asJsObject.fields("type").convertTo[String] == "input_file") + } + + test("OpenAI chat and responses stages expose expected response schemas") { + assert(new OpenAIChatCompletion().responseDataType == ChatModelResponse.schema) + assert(new OpenAIResponses().responseDataType == ResponsesModelResponse.schema) + } +} diff --git a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/openai/OpenAIPromptParamsSuite.scala b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/openai/OpenAIPromptParamsSuite.scala index 72631d70dd6..b4197f8fa7c 100644 --- a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/openai/OpenAIPromptParamsSuite.scala +++ b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/openai/OpenAIPromptParamsSuite.scala @@ -257,9 +257,6 @@ class OpenAIPromptParamsSuite extends TestBase { assert(generatedClass.contains("_post_processing_explicitly_set")) assert(generatedClass.contains("def clear(self, param)")) assert(generatedClass.contains("def copy(self, extra=None)")) - assert(generatedClass.contains("return super().clear(param)")) - assert(generatedClass.contains("result = super().copy(extra)")) - assert(!generatedClass.contains("super(OpenAIPrompt, self)")) assert(generatedClass.contains("self._set(postProcessingOptions=value)")) assert(!generatedClass.contains("_post_processing_validation")) assert(!generatedClass.contains("applyPrevalidated")) diff --git a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/openai/OpenAIPromptParserSuite.scala b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/openai/OpenAIPromptParserSuite.scala new file mode 100644 index 00000000000..5ce4fb8eb2c --- /dev/null +++ b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/openai/OpenAIPromptParserSuite.scala @@ -0,0 +1,126 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.services.openai + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.spark.sql.Row +import org.apache.spark.sql.functions.col +import org.apache.spark.sql.types.{ArrayType, DataType, StringType} + +import java.nio.charset.StandardCharsets +import java.nio.file.Files + +class OpenAIPromptParserSuite extends TestBase { + + import spark.implicits._ + + test("PassThroughParser returns input text and string schema") { + val parser = new PassThroughParser() + val parsed = Seq(" keep spacing ").toDF("response") + .select(parser.parse(col("response")).alias("parsed")) + .head() + .getString(0) + + assert(parsed == " keep spacing ") + assert(parser.outputSchema == StringType) + } + + test("DelimiterParser trims outer whitespace and splits values") { + val parser = new DelimiterParser(",") + val parsed = Seq(" apple, banana ,carrot ").toDF("response") + .select(parser.parse(col("response")).alias("parsed")) + .head() + .getSeq[String](0) + + assert(parsed == Seq("apple", " banana ", "carrot")) + assert(parser.outputSchema == ArrayType(StringType)) + } + + test("JsonParser removes code fences and parses JSON by schema") { + val schema = "name STRING, value INT" + val parser = new JsonParser(schema, Map.empty) + val parsed = Seq( + """```json + |{"name":"alpha","value":7} + |```""".stripMargin + ).toDF("response") + .select(parser.parse(col("response")).alias("parsed")) + .head() + .getAs[Row]("parsed") + + assert(parsed.getAs[String]("name") == "alpha") + assert(parsed.getAs[Int]("value") == 7) + assert(parser.outputSchema == DataType.fromDDL(schema)) + } + + test("RegexParser extracts configured group and uses string schema") { + val parser = new RegexParser("score=(\\d+)", 1) + val parsed = Seq("score=42 done").toDF("response") + .select(parser.parse(col("response")).alias("parsed")) + .head() + .getString(0) + + assert(parsed == "42") + assert(parser.outputSchema == StringType) + } + + test("stringMessageWrapper changes text type for responses API") { + val prompt = new OpenAIPrompt() + assert(prompt.stringMessageWrapper("hello") == Map("type" -> "text", "text" -> "hello")) + + prompt.setApiType("responses") + assert(prompt.stringMessageWrapper("hello") == Map("type" -> "input_text", "text" -> "hello")) + } + + test("createMessagesForRow returns null when path attachments are empty") { + val prompt = new OpenAIPrompt() + val messages = prompt.createMessagesForRow("Summarize", Map("filePath" -> " "), Seq("filePath")) + assert(messages == null) + } + + test("createMessagesForRow includes local text file contents for chat completions") { + val prompt = new OpenAIPrompt() + val tempFile = Files.createTempFile("synapseml-openai-local", ".txt") + + try { + Files.write(tempFile, "example content".getBytes(StandardCharsets.UTF_8)) + + val messages = prompt.createMessagesForRow( + "Summarize", + Map("filePath" -> tempFile.toString), + Seq("filePath") + ) + val userParts = messages.find(_.role == "user").get.content + + assert(userParts.head == Map("type" -> "text", "text" -> "Summarize")) + assert(userParts(1).get("type").contains("text")) + assert(userParts(1).get("text").exists(_.contains("Content: example content"))) + } finally { + Files.deleteIfExists(tempFile) + } + } + + test("createMessagesForRow formats text file content for responses API") { + val prompt = new OpenAIPrompt().setApiType("responses") + val tempFile = Files.createTempFile("synapseml-openai-local", ".txt") + + try { + Files.write(tempFile, "response content".getBytes(StandardCharsets.UTF_8)) + + val messages = prompt.createMessagesForRow( + "Summarize", + Map("filePath" -> tempFile.toString), + Seq("filePath") + ) + val systemParts = messages.find(_.role == "system").get.content + val userParts = messages.find(_.role == "user").get.content + + assert(systemParts.head.get("type").contains("input_text")) + assert(userParts.head == Map("type" -> "input_text", "text" -> "Summarize")) + assert(userParts(1) == Map("type" -> "input_text", "text" -> "response content")) + } finally { + Files.deleteIfExists(tempFile) + } + } +} diff --git a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/openai/UsageUtilsSuite.scala b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/openai/UsageUtilsSuite.scala new file mode 100644 index 00000000000..0d2c47f1a0f --- /dev/null +++ b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/openai/UsageUtilsSuite.scala @@ -0,0 +1,116 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.services.openai + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.spark.sql.{DataFrame, Row} +import org.apache.spark.sql.functions.{col, struct} + +class UsageUtilsSuite extends TestBase { + + import spark.implicits._ + + private def normalizeUsage(df: DataFrame, mapping: UsageUtils.UsageFieldMapping): Row = { + df.select(UsageUtils.normalize(col("usage"), mapping).alias("usage")).head().getAs[Row]("usage") + } + + test("normalize maps chat completion usage fields and nested details") { + val usageDf = Seq((1L, 2L, 3L, 10L, 11L, 21L, 22L, 23L, 24L)) + .toDF( + "prompt_tokens", + "completion_tokens", + "total_tokens", + "audio_tokens", + "cached_tokens", + "accepted_prediction_tokens", + "completion_audio_tokens", + "reasoning_tokens", + "rejected_prediction_tokens" + ) + .withColumn("usage", struct( + col("prompt_tokens"), + col("completion_tokens"), + col("total_tokens"), + struct(col("audio_tokens"), col("cached_tokens")).alias("prompt_tokens_details"), + struct( + col("accepted_prediction_tokens"), + col("completion_audio_tokens").alias("audio_tokens"), + col("reasoning_tokens"), + col("rejected_prediction_tokens") + ).alias("completion_tokens_details") + )) + .select("usage") + + val normalized = normalizeUsage(usageDf, UsageUtils.UsageMappings.ChatCompletions) + assert(normalized.getAs[Long]("input_tokens") == 1L) + assert(normalized.getAs[Long]("output_tokens") == 2L) + assert(normalized.getAs[Long]("total_tokens") == 3L) + assert( + normalized.getMap[String, Long](normalized.fieldIndex("input_token_details")) == + Map("audio_tokens" -> 10L, "cached_tokens" -> 11L) + ) + assert( + normalized.getMap[String, Long](normalized.fieldIndex("output_token_details")) == + Map( + "accepted_prediction_tokens" -> 21L, + "audio_tokens" -> 22L, + "reasoning_tokens" -> 23L, + "rejected_prediction_tokens" -> 24L + ) + ) + } + + test("normalize maps responses usage fields and details") { + val usageDf = Seq((4L, 5L, 9L, 2L, 3L)) + .toDF("input_tokens", "output_tokens", "total_tokens", "cached_tokens", "reasoning_tokens") + .withColumn("usage", struct( + col("input_tokens"), + col("output_tokens"), + col("total_tokens"), + struct(col("cached_tokens")).alias("input_tokens_details"), + struct(col("reasoning_tokens")).alias("output_tokens_details") + )) + .select("usage") + + val normalized = normalizeUsage(usageDf, UsageUtils.UsageMappings.Responses) + assert(normalized.getAs[Long]("input_tokens") == 4L) + assert(normalized.getAs[Long]("output_tokens") == 5L) + assert(normalized.getAs[Long]("total_tokens") == 9L) + assert(normalized.getMap[String, Long](normalized.fieldIndex("input_token_details")) == Map("cached_tokens" -> 2L)) + assert(normalized.getMap[String, Long](normalized.fieldIndex("output_token_details")) == + Map("reasoning_tokens" -> 3L)) + } + + test("normalize handles missing or empty detail mappings") { + val embeddingDf = Seq((7L, 7L)) + .toDF("prompt_tokens", "total_tokens") + .withColumn("usage", struct(col("prompt_tokens"), col("total_tokens"))) + .select("usage") + val embeddingUsage = normalizeUsage(embeddingDf, UsageUtils.UsageMappings.Embeddings) + assert(embeddingUsage.getAs[Long]("input_tokens") == 7L) + assert(embeddingUsage.isNullAt(embeddingUsage.fieldIndex("output_tokens"))) + assert(embeddingUsage.getMap[String, Long](embeddingUsage.fieldIndex("input_token_details")).isEmpty) + assert(embeddingUsage.getMap[String, Long](embeddingUsage.fieldIndex("output_token_details")).isEmpty) + + val customMapping = UsageUtils.UsageFieldMapping( + inputTokens = Some("input_tokens"), + outputTokens = None, + totalTokens = Some("total_tokens"), + inputDetails = Some("input_tokens_details" -> Seq.empty), + outputDetails = None + ) + val customDf = Seq((8L, 8L, 4L)) + .toDF("input_tokens", "total_tokens", "cached_tokens") + .withColumn("usage", struct( + col("input_tokens"), + col("total_tokens"), + struct(col("cached_tokens")).alias("input_tokens_details") + )) + .select("usage") + + val customUsage = normalizeUsage(customDf, customMapping) + assert(customUsage.getAs[Long]("input_tokens") == 8L) + assert(customUsage.getMap[String, Long](customUsage.fieldIndex("input_token_details")).isEmpty) + } +} diff --git a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/search/IndexSchemaLiveRoundTripSuite.scala b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/search/IndexSchemaLiveRoundTripSuite.scala new file mode 100644 index 00000000000..4961c86408b --- /dev/null +++ b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/search/IndexSchemaLiveRoundTripSuite.scala @@ -0,0 +1,96 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.services.search + +import com.microsoft.azure.synapse.ml.Secrets +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import com.microsoft.azure.synapse.ml.io.http.RESTHelpers._ +import org.apache.http.client.methods.HttpDelete +import spray.json._ + +import java.util.UUID + +/** End-to-end companion to [[IndexSchemaParsingSuite]]. + * + * The offline suite proves the parser accepts the payload shapes reported in issue #2143, but it + * asserts against hand-written JSON. This suite creates a real index that exercises every schema + * feature the service returns as an object, reads it back through the same production code path + * the writer uses, and parses it. That closes the gap where a hand-written fixture could drift + * from what the Search service actually emits. + */ +class IndexSchemaLiveRoundTripSuite extends TestBase with IndexJsonGetter with IndexParser { + + private lazy val azureSearchKey: String = sys.env.getOrElse("AZURE_SEARCH_KEY", Secrets.AzureSearchKey) + private val testServiceName = "mmlspark-azure-search" + + private def indexDefinition(name: String): String = + s""" + |{ + | "name": "$name", + | "fields": [ + | { "name": "id", "type": "Edm.String", "key": true, "searchable": true }, + | { "name": "text", "type": "Edm.String", "searchable": true, "analyzer": "sml_custom_analyzer" }, + | { "name": "title", "type": "Edm.String", "searchable": true } + | ], + | "corsOptions": { "allowedOrigins": ["*"], "maxAgeInSeconds": 300 }, + | "suggesters": [ + | { "name": "sml_suggester", "searchMode": "analyzingInfixMatching", "sourceFields": ["title"] } + | ], + | "analyzers": [ + | { + | "@odata.type": "#Microsoft.Azure.Search.CustomAnalyzer", + | "name": "sml_custom_analyzer", + | "tokenizer": "sml_tokenizer", + | "tokenFilters": ["sml_asciifolding"], + | "charFilters": ["sml_mapping"] + | } + | ], + | "tokenizers": [ + | { "@odata.type": "#Microsoft.Azure.Search.KeywordTokenizerV2", "name": "sml_tokenizer" } + | ], + | "tokenFilters": [ + | { "@odata.type": "#Microsoft.Azure.Search.AsciiFoldingTokenFilter", + | "name": "sml_asciifolding", "preserveOriginal": true } + | ], + | "charFilters": [ + | { "@odata.type": "#Microsoft.Azure.Search.MappingCharFilter", + | "name": "sml_mapping", "mappings": ["a=>b"] } + | ] + |} + |""".stripMargin + + private def deleteIndex(name: String): Unit = { + val apiVersion = AzureSearchAPIConstants.DefaultAPIVersion + val deleteRequest = new HttpDelete( + s"https://$testServiceName.search.windows.net/indexes/$name?api-version=$apiVersion") + deleteRequest.setHeader("api-key", azureSearchKey) + safeSend(deleteRequest) + () + } + + test("An index using object-valued schema features round-trips through the live service") { + val indexName = s"test-schema-${UUID.randomUUID().toString.take(8)}" + SearchIndex.createIfNoneExists(azureSearchKey, testServiceName, indexDefinition(indexName)) + try { + // Read back through the same path AzureSearchWriter uses, so this fails if the service + // emits a shape the production parser rejects. + val liveJson = getIndexJsonFromExistingIndex(azureSearchKey, testServiceName, indexName) + val info = parseIndexJson(liveJson) + + assert(info.name.contains(indexName)) + // The service returns these as objects; typing them as strings is what broke issue #2143. + assert(info.analyzers.exists(_.length == 1)) + assert(info.analyzers.get.head.asJsObject.fields("name") == JsString("sml_custom_analyzer")) + assert(info.tokenizers.exists(_.length == 1)) + assert(info.tokenFilters.exists(_.length == 1)) + assert(info.charFilters.exists(_.length == 1)) + assert(info.suggesters.exists(_.length == 1)) + // corsOptions is a single object rather than an array. + assert(info.corsOptions.exists(_.asJsObject.fields("maxAgeInSeconds") == JsNumber(300))) + } finally { + deleteIndex(indexName) + } + } + +} diff --git a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/search/IndexSchemaParsingSuite.scala b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/search/IndexSchemaParsingSuite.scala new file mode 100644 index 00000000000..6eb430fe7ce --- /dev/null +++ b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/search/IndexSchemaParsingSuite.scala @@ -0,0 +1,102 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.services.search + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import com.microsoft.azure.synapse.ml.services.search.AzureSearchProtocol._ +import spray.json._ + +/** Regression coverage for index definitions that use the schema features the Search service + * returns as objects rather than strings. See issue #2143: any index carrying a custom analyzer + * failed to deserialize with "Expected String as JsString" before the writer ever sent a document. + */ +class IndexSchemaParsingSuite extends TestBase with IndexParser { + + private val keyField = + """{"name": "Id", "type": "Edm.String", "key": true, "searchable": false}""" + + private def indexJson(extraMembers: String): String = + s"""{"name": "test-index", "fields": [$keyField]${if (extraMembers.isEmpty) "" else ", " + extraMembers}}""" + + // Verbatim from the issue #2143 report. + private val customAnalyzer = + """"analyzers": [{ + | "@odata.type": "#Microsoft.Azure.Search.CustomAnalyzer", + | "name": "keyword_analyzer", + | "tokenizer": "keyword_v2", + | "charFilters": [], + | "tokenFilters": ["lowercase"] + |}]""".stripMargin + + test("parseIndexJson accepts an index with a custom analyzer") { + val info = parseIndexJson(indexJson(customAnalyzer)) + assert(info.name.contains("test-index")) + assert(info.analyzers.exists(_.length == 1)) + val analyzer = info.analyzers.get.head.asJsObject + assert(analyzer.fields("name") == JsString("keyword_analyzer")) + assert(analyzer.fields("@odata.type") == JsString("#Microsoft.Azure.Search.CustomAnalyzer")) + assert(analyzer.fields("tokenFilters") == JsArray(JsString("lowercase"))) + } + + test("a custom analyzer survives a parse and re-serialize round trip") { + val original = indexJson(customAnalyzer) + val roundTripped = parseIndexJson(original).toJson.asJsObject + // The service rejects an analyzer it cannot identify, so every member has to come back intact. + assert(roundTripped.fields("analyzers") == original.parseJson.asJsObject.fields("analyzers")) + } + + test("parseIndexJson accepts object-valued charFilters, tokenizers and tokenFilters") { + val members = + """"charFilters": [{"@odata.type": "#Microsoft.Azure.Search.MappingCharFilter", + | "name": "cf", "mappings": ["a=>b"]}], + |"tokenizers": [{"@odata.type": "#Microsoft.Azure.Search.KeywordTokenizerV2", "name": "kw"}], + |"tokenFilters": [{"@odata.type": "#Microsoft.Azure.Search.AsciiFoldingTokenFilter", + | "name": "af", "preserveOriginal": true}]""".stripMargin + val info = parseIndexJson(indexJson(members)) + assert(info.charFilters.exists(_.length == 1)) + assert(info.tokenizers.exists(_.length == 1)) + assert(info.tokenFilters.exists(_.length == 1)) + assert(info.tokenFilters.get.head.asJsObject.fields("preserveOriginal") == JsTrue) + } + + test("parseIndexJson accepts object-valued suggesters") { + val members = + """"suggesters": [{"name": "sg", "searchMode": "analyzingInfixMatching", "sourceFields": ["Id"]}]""" + val info = parseIndexJson(indexJson(members)) + assert(info.suggesters.exists(_.length == 1)) + assert(info.suggesters.get.head.asJsObject.fields("name") == JsString("sg")) + } + + test("parseIndexJson accepts corsOptions, which the service returns as an object not an array") { + val members = """"corsOptions": {"allowedOrigins": ["*"], "maxAgeInSeconds": 300}""" + val info = parseIndexJson(indexJson(members)) + assert(info.corsOptions.exists(_.asJsObject.fields("maxAgeInSeconds") == JsNumber(300))) + } + + test("an index using every object-valued feature at once still parses") { + val members = Seq( + customAnalyzer, + """"charFilters": [{"@odata.type": "#Microsoft.Azure.Search.MappingCharFilter", + | "name": "cf", "mappings": ["a=>b"]}]""".stripMargin, + """"tokenizers": [{"@odata.type": "#Microsoft.Azure.Search.KeywordTokenizerV2", "name": "kw"}]""", + """"tokenFilters": [{"@odata.type": "#Microsoft.Azure.Search.AsciiFoldingTokenFilter", "name": "af"}]""", + """"suggesters": [{"name": "sg", "searchMode": "analyzingInfixMatching", "sourceFields": ["Id"]}]""", + """"corsOptions": {"allowedOrigins": ["*"]}""" + ).mkString(", ") + val info = parseIndexJson(indexJson(members)) + assert(info.fields.length == 1) + assert(info.analyzers.isDefined && info.charFilters.isDefined && info.tokenizers.isDefined) + assert(info.tokenFilters.isDefined && info.suggesters.isDefined && info.corsOptions.isDefined) + } + + test("omitting the optional members leaves them empty rather than failing") { + val info = parseIndexJson(indexJson("")) + assert(info.analyzers.isEmpty) + assert(info.charFilters.isEmpty) + assert(info.tokenizers.isEmpty) + assert(info.tokenFilters.isEmpty) + assert(info.suggesters.isEmpty) + assert(info.corsOptions.isEmpty) + } +} diff --git a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/search/SearchIndexRetention.scala b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/search/SearchIndexRetention.scala new file mode 100644 index 00000000000..3e7da994981 --- /dev/null +++ b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/search/SearchIndexRetention.scala @@ -0,0 +1,96 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.services.search + +import java.time.format.{DateTimeFormatter, DateTimeFormatterBuilder, DateTimeParseException, SignStyle} +import java.time.temporal.ChronoField +import java.time.{Duration, LocalDateTime} +import scala.util.matching.Regex + +/** Decides which of the shared search service's test indexes a run is allowed to delete. + * + * The service will only hold [[MaxIndexes]] indexes and the search suites create one per test, so + * a run that deletes only what it created will eventually find the service full and fail in + * `beforeAll` before it can create anything. Collecting purely by age does not save it either + * once the cap is reached faster than the retention window expires. That is what happened in + * issue #2639: the service sat at 48 of 50 indexes while the oldest was barely 40 hours old, so a + * two day cutoff selected nothing and every build in the repository, `master` included, failed + * until the indexes were deleted by hand. + * + * The policy here is therefore driven by pressure rather than by age alone. Anything past + * [[RoutineAge]] is collected on every sweep, and when that still does not leave + * [[DesiredFreeSlots]] free the sweep keeps taking the next oldest index until it does. + * [[MinimumAge]] is the floor that stops it from ever touching an index a concurrently running + * build might still be using. + */ +object SearchIndexRetention { + + /** Maximum number of indexes the shared service will hold. */ + val MaxIndexes: Int = 50 + + /** Slots a run wants free before it starts creating indexes of its own. */ + val DesiredFreeSlots: Int = 10 + + /** Never collect an index younger than this. A full pipeline run finishes well inside an hour, + * so an index older than this cannot still belong to a build that is running. + * + * This holds only while every run stamps names on the same clock. [[age]] can do no better + * than subtract the name's timestamp from the caller's `now`, so a name written in local time + * and read from another zone is wrong by the offset between them, and an offset larger than + * this floor would let a sweep collect an index a live build had just created. Producers and + * callers therefore both work in UTC. + */ + val MinimumAge: Duration = Duration.ofHours(3) + + /** Collected on every sweep, however much room happens to be left. */ + val RoutineAge: Duration = Duration.ofHours(12) + + /** Test index names are the ones `generateIndexName` writes: a `test-` prefix, then a hash, then + * the 17 digit timestamp [[Formatter]] produces. + * + * The prefix is part of the match on purpose. The sweep reads every index in the shared + * service, not just this suite's, so matching on the timestamp alone would let it delete a + * foreign index that happened to end in 17 digits. That was survivable while collection only + * touched indexes older than two days; now that a sweep routinely collects at [[RoutineAge]] + * and drops to [[MinimumAge]] under pressure, it is not. + */ + private val TimestampedName: Regex = "^test-.*-(\\d{17})$".r + + /** When a date pattern starts with 'yyyy' and has no separator following, the parser can + * sometimes decide to take the whole string to match the year, which results in an exception. + * The following is a hackaround. + */ + val Formatter: DateTimeFormatter = new DateTimeFormatterBuilder() + .appendValue(ChronoField.YEAR_OF_ERA, 4, 4, SignStyle.EXCEEDS_PAD) + .appendPattern("MMddHHmmssSSS").toFormatter() + + /** How old the index its name says it is, or None when the name carries no usable timestamp. */ + def age(name: String, now: LocalDateTime): Option[Duration] = name match { + case TimestampedName(stamp) => + try Some(Duration.between(LocalDateTime.parse(stamp, Formatter), now)) + catch { case _: DateTimeParseException => None } + case _ => None + } + + /** Every index this run is allowed to delete, oldest first. + * + * An index whose name carries no timestamp is never collected: without an age there is no way + * to tell it apart from one a running build just created. + */ + def collectable(existing: Seq[String], now: LocalDateTime): Seq[(String, Duration)] = + existing.flatMap(name => age(name, now).map(name -> _)) + .filter { case (_, indexAge) => indexAge.compareTo(MinimumAge) > 0 } + .sortBy { case (_, indexAge) => -indexAge.toMillis } + + /** The indexes to delete on this sweep, oldest first. */ + def select(existing: Seq[String], now: LocalDateTime): Seq[String] = { + val candidates = collectable(existing, now) + // candidates is oldest first, so the routine ones are exactly its first `routine` entries and + // taking any more than that walks steadily towards the youngest index still safe to collect. + val routine = candidates.count { case (_, indexAge) => indexAge.compareTo(RoutineAge) > 0 } + val freeAfterRoutine = MaxIndexes - existing.size + routine + val shortfall = math.max(0, DesiredFreeSlots - freeAfterRoutine) + candidates.take(routine + shortfall).map { case (name, _) => name } + } +} diff --git a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/search/SearchIndexRetentionSuite.scala b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/search/SearchIndexRetentionSuite.scala new file mode 100644 index 00000000000..fc436116109 --- /dev/null +++ b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/search/SearchIndexRetentionSuite.scala @@ -0,0 +1,143 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.services.search + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase + +import java.time.LocalDateTime + +/** Covers the retention policy that keeps the shared search service from filling up (issue #2639). + * + * None of this talks to the service, so it runs on every build rather than only on the ones that + * have search credentials, which matters because the bug it guards against took down `master`. + */ +class SearchIndexRetentionSuite extends TestBase { + + import SearchIndexRetention._ + + private val now: LocalDateTime = LocalDateTime.of(2026, 8, 15, 12, 0, 0) + + /** A name shaped exactly like the ones generateIndexName produces. */ + private def index(hoursOld: Long, tag: String = "1534278552"): String = + s"test-$tag-${Formatter.format(now.minusHours(hoursOld))}" + + /** Enough young indexes to sit `holding` short of the cap without any being collectable. */ + private def young(count: Int): Seq[String] = + (1 to count).map(i => index(1, s"young$i")) + + test("An index younger than the minimum age is never collected") { + val recent = Seq(index(MinimumAge.toHours - 1)) + assert(collectable(recent, now).isEmpty) + assert(select(recent, now).isEmpty) + } + + test("An index past the routine age is collected even when the service is nearly empty") { + val old = index(RoutineAge.toHours + 1) + assert(select(Seq(old), now) == Seq(old)) + } + + test("An index between the minimum and routine ages is kept while there is room") { + val middling = index(RoutineAge.toHours - 1) + assert(collectable(Seq(middling), now).map { case (n, _) => n } == Seq(middling)) + assert(select(Seq(middling), now).isEmpty) + } + + test("Pressure reclaims indexes that are too young for the routine sweep") { + // This is issue #2639: the service one index below its cap with nothing past the old two day + // cutoff. A purely age based sweep selected nothing here and the suite then failed to create. + val reclaimable = (1 to 20).map(i => index(MinimumAge.toHours + i, s"mid$i")) + val existing = reclaimable ++ young(MaxIndexes - 2 - reclaimable.size) + assert(existing.size == MaxIndexes - 2) + val collected = select(existing, now) + assert(collected.nonEmpty) + assert(existing.size - collected.size <= MaxIndexes - DesiredFreeSlots) + } + + test("Pressure reclaims the oldest indexes first") { + val existing = (1 to 20).map(i => index(MinimumAge.toHours + i, s"mid$i")) ++ + young(MaxIndexes - 2 - 20) + val collected = select(existing, now) + val ages = collected.map(name => age(name, now).get.toHours) + assert(ages == ages.sorted.reverse, "expected oldest first") + assert(ages.min > MinimumAge.toHours, "must not cross the safety floor") + } + + test("Pressure never collects an index a running build could still own") { + // Every index is under the floor, so even at the cap there is nothing safe to take. + val existing = (1 to MaxIndexes).map(i => index(MinimumAge.toHours - 1, s"busy$i")) + assert(select(existing, now).isEmpty) + } + + test("The pressure sweep stops once it has freed enough room") { + // Every index sits between the two ages, so nothing is collected routinely and the whole + // selection is pressure driven. It should stop the moment there is enough room, not keep going. + val existing = (1 to MaxIndexes).map(i => + index(MinimumAge.toHours + 1 + (i % (RoutineAge.toHours - MinimumAge.toHours - 1)), s"mid$i")) + assert(collectable(existing, now).size == MaxIndexes, "expected every index to be collectable") + assert(select(existing, now).size == DesiredFreeSlots) + } + + test("The routine sweep is not capped by the free slot target") { + // Anything past the routine age is garbage, so it all goes even though far fewer would do. + val existing = (1 to MaxIndexes).map(i => index(RoutineAge.toHours + i, s"old$i")) + assert(select(existing, now).size == MaxIndexes) + } + + test("An index with no timestamp in its name is left alone") { + // Nothing can be inferred about its age, so it is not this sweep's to delete. + val untimestamped = Seq("test-website", "test-33467690", "examplevectorindex") + assert(select(untimestamped, now).isEmpty) + assert(untimestamped.flatMap(age(_, now)).isEmpty) + } + + test("An index whose timestamp is not a real date is left alone") { + assert(age("test-1-99999999999999999", now).isEmpty) + assert(select(Seq("test-1-99999999999999999"), now).isEmpty) + } + + /** The sweep reads every index in the shared service, not just the ones these suites made, so a + * foreign name that happens to end in 17 digits must never be collected however old it looks + * or however full the service is. + */ + test("An index that is not one of ours is never collected") { + val foreign = Seq( + index(RoutineAge.toHours * 10, "x").stripPrefix("test-"), + s"prod-catalog-${Formatter.format(now.minusDays(30))}") + assert(foreign.flatMap(age(_, now)).isEmpty) + assert(select(foreign, now).isEmpty) + assert(select(foreign ++ young(MaxIndexes - foreign.size), now).isEmpty) + } + + test("Age is read back from the name generateIndexName would have written") { + assert(age(index(5), now).map(_.toHours).contains(5L)) + // UUID.randomUUID().hashCode() is signed, so the hash segment is sometimes negative. + assert(age(index(5, "-1534278552"), now).map(_.toHours).contains(5L)) + } + + /** Guards the clock convention [[SearchIndexRetention.MinimumAge]] depends on. The shared + * service is written to both by the UTC build agents and from workstations, so a name stamped + * in the writer's local time reads as wrong by the offset between the two zones. Offsets reach + * well past the floor -- the live service showed indexes apparently seven hours in the future + * when read from a UTC-7 machine -- which would be enough for a sweep to collect an index a + * running build had just created. + */ + test("An index is aged on the same clock it was named on") { + val stamped = s"test-utc-${Formatter.format(now)}" + assert(age(stamped, now).map(_.toHours).contains(0L)) + assert(select(Seq(stamped), now).isEmpty) + + // Reading the same name against a clock seven hours ahead ages it past the floor, which is + // what naming in local time would do to a build that is still using the index. + val skewed = age(stamped, now.plusHours(7)) + assert(skewed.map(_.toHours).contains(7L)) + assert(skewed.exists(_.compareTo(MinimumAge) > 0)) + } + + test("The safety floor leaves room for a pipeline run to finish") { + // A full run is well under an hour; anything near that would make the sweep race live builds. + assert(MinimumAge.toHours >= 2) + assert(RoutineAge.compareTo(MinimumAge) > 0) + assert(DesiredFreeSlots > 0 && DesiredFreeSlots < MaxIndexes) + } +} diff --git a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/search/split1/SearchWriterSuitePart1.scala b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/search/split1/SearchWriterSuitePart1.scala index 4003d0b61ce..dc28bbc0205 100644 --- a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/search/split1/SearchWriterSuitePart1.scala +++ b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/search/split1/SearchWriterSuitePart1.scala @@ -17,12 +17,12 @@ import org.apache.spark.ml.util.MLReadable import org.apache.spark.sql.DataFrame import org.scalatest.{Outcome, TestData} -import java.time.LocalDateTime -import java.time.format.{DateTimeFormatter, DateTimeFormatterBuilder, DateTimeParseException, SignStyle} -import java.time.temporal.ChronoField +import java.time.{LocalDateTime, ZoneOffset} +import java.time.format.DateTimeFormatter import java.util.UUID import scala.collection.mutable import scala.concurrent.blocking +import scala.util.control.NonFatal trait AzureSearchKey { lazy val azureSearchKey: String = sys.env.getOrElse("AZURE_SEARCH_KEY", Secrets.AzureSearchKey) @@ -35,11 +35,15 @@ class SearchWriterSuiteUtilities extends TestBase with AzureSearchKey private[ml] val testServiceName = "mmlspark-azure-search" - // When a date pattern starts with 'yyyy' and has no separator following, the parser can sometimes decide - // to take the whole string to match the year, which results in an exception. The following is a hackaround. - val formatter: DateTimeFormatter = new DateTimeFormatterBuilder() - .appendValue(ChronoField.YEAR_OF_ERA, 4, 4, SignStyle.EXCEEDS_PAD) - .appendPattern("MMddHHmmssSSS").toFormatter() + /** Status the service returns for an index it has just deleted. */ + private val deletedStatus: Int = 204 + /** Status for an index that is already gone. */ + private val notFoundStatus: Int = 404 + /** Stand-in status for a delete that threw before it got a response. */ + private val unknownDeleteStatus: Int = -1 + + // Shared with SearchIndexRetention, which has to read back the timestamps written here. + val formatter: DateTimeFormatter = SearchIndexRetention.Formatter private[ml] def createTestData(numDocs: Int): DataFrame = { (0 until numDocs) @@ -140,7 +144,12 @@ class SearchWriterSuiteUtilities extends TestBase with AzureSearchKey testName.get } - val date = formatter.format(LocalDateTime.now()) + // UTC, not the default zone: the timestamp in the name is the only record of when an index + // was made, and `cleanOldIndexes` will not collect one until it is `MinimumAge` old. Naming + // in local time makes that age wrong by the reader's offset whenever the two runs sit in + // different zones, which is the normal case here because the service is shared between the + // UTC build agents and whoever runs these suites from a workstation. + val date = formatter.format(LocalDateTime.now(ZoneOffset.UTC)) val name = s"test-${UUID.randomUUID().hashCode()}-${date}" createdIndexes.getOrElseUpdate(testNameNormalized, mutable.HashSet[String]()).+=(name) name @@ -171,9 +180,14 @@ class SearchWriterSuiteUtilities extends TestBase with AzureSearchKey println("Cleaning up services") val indexNames = this.createdIndexes.values.flatten println(s"Remaining indices: ${indexNames.mkString(",")}") - cleanTestIndices(indexNames) - cleanOldIndexes() - super.afterAll() + try { + cleanTestIndices(indexNames) + } finally { + // Still sweep and tear down Spark even when this run could not delete its own indexes, + // since skipping the sweep is what leaves the service full for the run after this one. + cleanOldIndexes() + super.afterAll() + } () } @@ -193,34 +207,49 @@ class SearchWriterSuiteUtilities extends TestBase with AzureSearchKey } private[ml] def cleanTestIndices(indices: Iterable[String]): Unit = { - val successfulCleanup = getExisting(azureSearchKey, testServiceName) - .intersect(indices.toSeq).map { n => - println(s"Deleting index $n") - deleteIndex(n) - }.forall(_ == 204) - assert(successfulCleanup) + val existing = getExisting(azureSearchKey, testServiceName).toSet + val failed = deleteQuietly(indices.filter(existing.contains)) + assert(failed.isEmpty, s"Could not delete test indices ${failed.mkString(",")}") () } - def cleanOldIndexes(): Unit = { - import scala.util.matching.Regex - - val twoDaysAgo = LocalDateTime.now().minusDays(2) - val endingDatePattern: Regex = "^.*-(\\d{17})$".r - val e = getExisting(azureSearchKey, testServiceName) - e.foreach { - case name@endingDatePattern(dateString) => - try { - val date = LocalDateTime.parse(dateString, formatter) - if (date.isBefore(twoDaysAgo)) { - deleteIndex(name) - } - } catch { - case _: DateTimeParseException => {} - case t: Throwable => throw t + /** Deletes every index named and returns the ones that survived. + * + * Nothing is thrown, because a sweep that stops at its first failure is how the service fills + * up: the indexes behind the failure are left behind and the run that would have collected + * them never gets that far. + */ + private[ml] def deleteQuietly(indices: Iterable[String]): Seq[String] = + indices.toSeq.filter { name => + println(s"Deleting index $name") + val status = + try deleteIndex(name) + catch { + case NonFatal(t) => + println(s"Failed to delete index $name: $t") + unknownDeleteStatus } - case _ => {} + // A 404 means something else already collected it, which does just as well. + status != deletedStatus && status != notFoundStatus + } + + /** Frees room on the shared service, deleting the oldest test indexes first. + * + * Never throws. This runs at the start of `beforeAll`, so letting it fail would take down the + * whole suite over indexes that another run is free to collect later. + */ + def cleanOldIndexes(): Unit = { + try { + val existing = getExisting(azureSearchKey, testServiceName) + val collected = SearchIndexRetention.select(existing, LocalDateTime.now(ZoneOffset.UTC)) + println(s"Service $testServiceName holds ${existing.size} of ${SearchIndexRetention.MaxIndexes} indexes, " + + s"reclaiming ${collected.size}") + val failed = deleteQuietly(collected) + if (failed.nonEmpty) println(s"Could not reclaim ${failed.mkString(",")}") + } catch { + case NonFatal(t) => println(s"Could not reclaim old indexes: $t") } + () } private[ml] def retryWithBackoff[T](f: => T, @@ -510,7 +539,7 @@ class SearchWriterSuitePart1 extends SearchWriterSuiteUtilities val indexJson = retryWithBackoff(getIndexJsonFromExistingIndex(azureSearchKey, testServiceName, in1)) // assert if vectorCol is a vector field - assert(parseIndexJson(indexJson).fields.find(_.name == "vectorCol").get.vectorSearchConfiguration.nonEmpty) + assert(parseIndexJson(indexJson).fields.find(_.name == "vectorCol").get.vectorReference.nonEmpty) } test("Infer the structure of the index from the dataframe with vector columns") { @@ -546,9 +575,9 @@ class SearchWriterSuitePart1 extends SearchWriterSuiteUtilities // assert if vectorCols are a vector field val indexJson = retryWithBackoff(getIndexJsonFromExistingIndex(azureSearchKey, testServiceName, in)) - assert(parseIndexJson(indexJson).fields.find(_.name == "vectorCol1").get.vectorSearchConfiguration.nonEmpty) - assert(parseIndexJson(indexJson).fields.find(_.name == "vectorCol2").get.vectorSearchConfiguration.nonEmpty) - assert(parseIndexJson(indexJson).fields.find(_.name == "vectorCol3").get.vectorSearchConfiguration.nonEmpty) + assert(parseIndexJson(indexJson).fields.find(_.name == "vectorCol1").get.vectorReference.nonEmpty) + assert(parseIndexJson(indexJson).fields.find(_.name == "vectorCol2").get.vectorReference.nonEmpty) + assert(parseIndexJson(indexJson).fields.find(_.name == "vectorCol3").get.vectorReference.nonEmpty) } test("Throw useful error when given vector columns in nested fields") { @@ -631,6 +660,45 @@ class SearchWriterSuitePart1 extends SearchWriterSuiteUtilities } } + test("Write to a vector index created by an earlier release using the legacy schema") { + // Regression guard: an index created by a previous SynapseML release uses the legacy + // vector schema. Reading it back under the current default api-version returns + // `dimensions` with a null `vectorSearchProfile` and an empty `profiles` list, because + // the service normalizes the response. Appending documents must keep working for those + // users without them having to pin an apiVersion. + val in = generateIndexName() + val legacyApiVersion = "2023-07-01-Preview" + val vectorDF = createTestDataWithVector(4) + + // Create the index exactly as an older release would have. + AzureSearchWriter.write(vectorDF.limit(2), + Map("subscriptionKey" -> azureSearchKey, + "actionCol" -> "searchAction", + "serviceName" -> testServiceName, + "apiVersion" -> legacyApiVersion, + "indexJson" -> createSimpleIndexJsonWithVector(in))) + + val normalized = retryWithBackoff(getIndexJsonFromExistingIndex(azureSearchKey, testServiceName, in)) + val normalizedVectorField = parseIndexJson(normalized).fields.find(_.name == "vectorCol").get + assert(normalizedVectorField.isVectorField, + "a field with dimensions must still be recognised as a vector field after normalization") + + // Now append with the current default api-version, as an upgraded user would. + retryWithBackoff({ + if (getExisting(azureSearchKey, testServiceName).contains(in)) { + AzureSearchWriter.write(vectorDF.except(vectorDF.limit(2)), + Map("subscriptionKey" -> azureSearchKey, + "actionCol" -> "searchAction", + "serviceName" -> testServiceName, + "indexName" -> in)) + } else { + throw new RuntimeException("No existing service found") + } + }) + + retryWithBackoff(assertSize(in, 4)) + } + test("Handle non-existent vector column specified in vectorCols option") { val in = generateIndexName() val phraseDF = Seq( diff --git a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/search/split2/SearchWriterSuitePart2.scala b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/search/split2/SearchWriterSuitePart2.scala index 54ace956e1a..dec4874cb81 100644 --- a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/search/split2/SearchWriterSuitePart2.scala +++ b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/search/split2/SearchWriterSuitePart2.scala @@ -49,7 +49,7 @@ class SearchWriterSuite extends SearchWriterSuiteUtilities { retryWithBackoff(assertSize(in, 2)) val indexJson = retryWithBackoff(getIndexJsonFromExistingIndex(azureSearchKey, testServiceName, in)) - assert(parseIndexJson(indexJson).fields.find(_.name == "vectorContent").get.vectorSearchConfiguration.nonEmpty) + assert(parseIndexJson(indexJson).fields.find(_.name == "vectorContent").get.vectorReference.nonEmpty) } test("Handle Azure Search index with scoring profiles") { @@ -171,4 +171,197 @@ class SearchWriterSuite extends SearchWriterSuiteUtilities { } + test("Handle GeoJSON GeographyPoint fields supplied as strings") { + + val in = generateIndexName() + val df = spark.createDataFrame(Seq( + ("upload", "0", """{"type":"Point","coordinates":[-122.3493, 47.6205]}"""), + ("upload", "1", """{"type":"Point","coordinates":[-122.3351, 47.6080]}""") + )).toDF("searchAction", "id", "location") + + val indexJson = + s""" + |{ + | "name": "$in", + | "fields": [ + | { "name": "id", "type": "Edm.String", "key": true, "searchable": true, "retrievable": true }, + | { "name": "location", "type": "Edm.GeographyPoint", "searchable": false, + | "filterable": true, "retrievable": true, "sortable": true } + | ] + |} + |""".stripMargin + + AzureSearchWriter.write(df, + Map( + "subscriptionKey" -> azureSearchKey, + "actionCol" -> "searchAction", + "serviceName" -> testServiceName, + "indexJson" -> indexJson + ) + ) + + // With fatalErrors=true (default) any 400 from Azure Search becomes a thrown + // RuntimeException, so reaching this `assertSize` proves the documents were + // accepted as valid spatial objects -- a count of 2 is only achievable if the + // GeoJSON strings were correctly parsed and serialized as GeoJSON objects. + retryWithBackoff(assertSize(in, 2)) + + } + + test("convertGeographyPointToStruct parses GeoJSON strings into structs") { + val df = spark.createDataFrame(Seq( + ("0", """{"type":"Point","coordinates":[-122.3493, 47.6205]}"""), + ("1", null) + )).toDF("id", "location") + + val indexJson = + """ + |{ + | "name": "unit-test-geo", + | "fields": [ + | { "name": "id", "type": "Edm.String", "key": true }, + | { "name": "location", "type": "Edm.GeographyPoint" } + | ] + |} + |""".stripMargin + + val converted = AzureSearchWriter.convertGeographyPointToStruct(df, indexJson) + val expected = StructType(Seq( + StructField("type", StringType), + StructField("coordinates", ArrayType(DoubleType)) + )) + assert(converted.schema("location").dataType == expected) + + val rows = converted.orderBy("id").collect() + val parsed = rows.head.getStruct(rows.head.fieldIndex("location")) + assert(parsed.getString(0) == "Point") + assert(parsed.getSeq[Double](1) == Seq(-122.3493, 47.6205)) + assert(rows(1).isNullAt(rows(1).fieldIndex("location"))) + } + + test("convertGeographyPointToStruct leaves struct columns untouched") { + val schema = StructType(Seq( + StructField("id", StringType), + StructField("location", StructType(Seq( + StructField("type", StringType, nullable = false), + StructField("coordinates", ArrayType(DoubleType, containsNull = false), nullable = false) + ))) + )) + val df = spark.createDataFrame( + spark.sparkContext.parallelize(Seq(Row("0", Row("Point", Seq(-122.3493, 47.6205))))), + schema + ) + + val indexJson = + """ + |{ + | "name": "unit-test-geo", + | "fields": [ + | { "name": "id", "type": "Edm.String", "key": true }, + | { "name": "location", "type": "Edm.GeographyPoint" } + | ] + |} + |""".stripMargin + + val converted = AzureSearchWriter.convertGeographyPointToStruct(df, indexJson) + assert(converted.schema("location").dataType == schema("location").dataType) + } + + test("convertGeographyPointToStruct fails fast on malformed GeoJSON instead of silently nulling") { + val df = spark.createDataFrame(Seq( + ("0", "{not valid json") + )).toDF("id", "location") + + val indexJson = + """ + |{ + | "name": "unit-test-geo", + | "fields": [ + | { "name": "id", "type": "Edm.String", "key": true }, + | { "name": "location", "type": "Edm.GeographyPoint" } + | ] + |} + |""".stripMargin + + val converted = AzureSearchWriter.convertGeographyPointToStruct(df, indexJson) + // FAILFAST surfaces parse errors when the row is materialized, not at plan time. + // The concrete wrapper type varies (SparkException, ExecutionException, or a bare + // RuntimeException when Spark folds the LocalRelation on the driver), so assert on + // the flattened cause chain instead. + val caught = intercept[Exception] { + converted.collect() + } + assert(causeChain(caught).contains("Malformed records are detected"), + s"expected a FAILFAST parse failure but got: ${causeChain(caught)}") + } + + test("convertGeographyPointToStruct rejects valid JSON that is not a GeoJSON Point") { + val indexJson = + """ + |{ + | "name": "unit-test-geo", + | "fields": [ + | { "name": "id", "type": "Edm.String", "key": true }, + | { "name": "location", "type": "Edm.GeographyPoint" } + | ] + |} + |""".stripMargin + + // Every one of these is syntactically valid JSON (or blank), so Spark's FAILFAST parser + // accepts it and yields a partially-null struct. Without explicit shape validation these + // would be silently indexed as a null location instead of failing the write. + val wrongShapes = Seq( + """{"foo":"bar"}""", + """{"type":"Point"}""", + """{"coordinates":[-122.3493, 47.6205]}""", + """{"type":"Polygon","coordinates":[-122.3493, 47.6205]}""", + """{"type":"Point","coordinates":[-122.3493]}""", + """{"type":"Point","coordinates":[-122.3493, 47.6205, 12.0]}""", + """{"type":"Point","coordinates":[-122.3493, null]}""", + "", + " " + ) + + wrongShapes.foreach { badValue => + val df = spark.createDataFrame(Seq(("0", badValue))).toDF("id", "location") + val converted = AzureSearchWriter.convertGeographyPointToStruct(df, indexJson) + val caught = intercept[Exception] { + converted.collect() + } + val message = causeChain(caught) + assert(message.contains("not a valid GeoJSON Point"), + s"expected a GeoJSON validation failure for '$badValue' but got: $message") + assert(message.contains("location"), + s"expected the error to name the offending column for '$badValue'") + } + } + + test("convertGeographyPointToStruct preserves nulls without raising") { + val df = spark.createDataFrame(Seq( + ("0", """{"type":"Point","coordinates":[-122.3493, 47.6205]}"""), + ("1", null), + ("2", null) + )).toDF("id", "location") + + val indexJson = + """ + |{ + | "name": "unit-test-geo", + | "fields": [ + | { "name": "id", "type": "Edm.String", "key": true }, + | { "name": "location", "type": "Edm.GeographyPoint" } + | ] + |} + |""".stripMargin + + val rows = AzureSearchWriter.convertGeographyPointToStruct(df, indexJson).orderBy("id").collect() + assert(rows.length == 3) + assert(!rows.head.isNullAt(rows.head.fieldIndex("location"))) + assert(rows(1).isNullAt(rows(1).fieldIndex("location"))) + assert(rows(2).isNullAt(rows(2).fieldIndex("location"))) + } + + private def causeChain(t: Throwable): String = + Iterator.iterate(t)(_.getCause).takeWhile(_ != null).map(_.toString).mkString(" | ") + } diff --git a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/search/split2/VectorSchemaMigrationSuite.scala b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/search/split2/VectorSchemaMigrationSuite.scala new file mode 100644 index 00000000000..a9509c405bf --- /dev/null +++ b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/search/split2/VectorSchemaMigrationSuite.scala @@ -0,0 +1,445 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.services.search.split2 + +import com.microsoft.azure.synapse.ml.services.search._ +import com.microsoft.azure.synapse.ml.services.search.AzureSearchProtocol._ +import org.apache.http.{HttpEntity, HttpVersion} +import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet} +import org.apache.http.entity.{BasicHttpEntity, StringEntity} +import org.apache.http.message.{BasicHttpResponse, BasicStatusLine} +import org.scalatest.funsuite.AnyFunSuite +import spray.json._ + +import java.io.{IOException, InputStream} + +/** Secret-free coverage for the Azure AI Search vector schema migration. */ +class VectorSchemaMigrationSuite extends AnyFunSuite { + + private val legacyIndexJson = + """ + |{ + | "name": "legacy-index", + | "fields": [ + | { "name": "id", "type": "Edm.String", "key": true }, + | { + | "name": "vectorCol", + | "type": "Collection(Edm.Single)", + | "dimensions": 3, + | "vectorSearchConfiguration": "hnswConfig", + | "futureFieldOption": { "mode": "kept" } + | } + | ], + | "vectorSearch": { + | "algorithmConfigurations": [ + | { + | "name": "hnswConfig", + | "kind": "hnsw", + | "parameters": { "m": 4, "efConstruction": 400, "efSearch": 500, "metric": "cosine" }, + | "futureAlgorithmOption": "kept" + | }, + | { + | "name": "exhaustiveConfig", + | "kind": "exhaustiveKnn", + | "parameters": { "metric": "euclidean" } + | } + | ], + | "futureVectorSearchOption": { "enabled": true } + | }, + | "futureRootOption": [1, 2, 3] + |} + """.stripMargin + + private val modernIndexJson = + """ + |{ + | "name": "modern-index", + | "fields": [ + | { "name": "id", "type": "Edm.String", "key": true }, + | { + | "name": "vectorCol", + | "type": "Collection(Edm.Single)", + | "searchable": true, + | "dimensions": 3, + | "vectorSearchProfile": "vectorProfile", + | "futureFieldOption": { "mode": "kept" } + | } + | ], + | "vectorSearch": { + | "algorithms": [ + | { + | "name": "hnswConfig", + | "kind": "hnsw", + | "parameters": { "m": 8, "efConstruction": 500, "efSearch": 700, "metric": "cosine" }, + | "futureAlgorithmOption": "kept" + | }, + | { + | "name": "exhaustiveConfig", + | "kind": "exhaustiveKnn", + | "parameters": { "metric": "dotProduct" } + | } + | ], + | "profiles": [ + | { + | "name": "vectorProfile", + | "algorithm": "hnswConfig", + | "vectorizer": "aoaiVectorizer", + | "compression": "scalarCompression", + | "futureProfileOption": 7 + | } + | ], + | "vectorizers": [ + | { + | "name": "aoaiVectorizer", + | "kind": "azureOpenAI", + | "azureOpenAIParameters": { + | "resourceUri": "https://example.openai.azure.com", + | "deploymentId": "embedding", + | "modelName": "text-embedding-3-small" + | }, + | "futureVectorizerOption": true + | } + | ], + | "compressions": [ + | { + | "name": "scalarCompression", + | "kind": "scalarQuantization", + | "rescoringOptions": { "enableRescoring": true, "defaultOversampling": 4.0 }, + | "futureCompressionOption": "kept" + | } + | ], + | "futureVectorSearchOption": { "enabled": true } + | }, + | "futureRootOption": { "mode": "kept" } + |} + """.stripMargin + + private def parse(json: String): JsValue = json.parseJson + + private def objectAt(value: JsValue, key: String): JsObject = + value.asJsObject.fields(key).asJsObject + + private def vectorField(value: JsValue): JsObject = + value.asJsObject.fields("fields").asInstanceOf[JsArray].elements + .map(_.asJsObject) + .find(_.fields.get("name").contains(JsString("vectorCol"))) + .get + + private class RecordingSearchIndexClient(existingIndexName: String, + remoteIndexJson: String) extends SearchIndexClient { + var listCalls = 0 + var getCalls = 0 + var createCalls = 0 + + override def getExisting(auth: AzureSearchAuth, + serviceName: String, + apiVersion: String): Seq[String] = { + listCalls += 1 + Seq(existingIndexName) + } + + override def getIndexJson(auth: AzureSearchAuth, + serviceName: String, + indexName: String, + apiVersion: String): String = { + getCalls += 1 + remoteIndexJson + } + + override def createIndex(auth: AzureSearchAuth, + serviceName: String, + indexJson: String, + apiVersion: String): Int = { + createCalls += 1 + 201 // scalastyle:ignore magic.number + } + } + + private class TrackingResponse(responseEntity: HttpEntity) + extends BasicHttpResponse(new BasicStatusLine(HttpVersion.HTTP_1_1, 200, "OK")) + with CloseableHttpResponse { // scalastyle:ignore magic.number + + @volatile var closed = false + setEntity(responseEntity) + + override def close(): Unit = closed = true + } + + test("api version gate uses the 2023-10-01-Preview boundary") { + Seq("2023-07-01-Preview", "2023-09-30", "2020-06-30") + .foreach(v => assert(!AzureSearchAPIConstants.supportsVectorProfiles(v), s"$v should use legacy schema")) + + Seq("2023-10-01-Preview", "2023-10-01", "2023-11-01", "2024-03-01-preview", "2026-04-01") + .foreach(v => assert(AzureSearchAPIConstants.supportsVectorProfiles(v), s"$v should use profiles")) + + assert(AzureSearchAPIConstants.supportsVectorProfiles(AzureSearchAPIConstants.DefaultAPIVersion)) + } + + test("invalid api versions fail explicitly") { + Seq("", "not-a-version", "2023-13-01", "2023-10").foreach { version => + val error = intercept[IllegalArgumentException] { + AzureSearchAPIConstants.supportsVectorProfiles(version) + } + assert(error.getMessage.contains("apiVersion")) + } + } + + test("published VectorSearch and IndexField case class shapes remain source compatible") { + val algorithms = Seq(AlgorithmConfigs("vectorConfig", "hnsw")) + val vectorSearch = VectorSearch(algorithms) + val VectorSearch(extractedAlgorithms) = vectorSearch + assert(extractedAlgorithms == algorithms) + assert(vectorSearch.productArity == 1) + + val field = IndexField( + "vectorCol", "Collection(Edm.Single)", None, None, None, None, None, None, + None, None, None, None, None, Some(3), Some("vectorConfig")) + val IndexField(name, fieldType, searchable, filterable, sortable, facetable, retrievable, key, + analyzer, searchAnalyzer, indexAnalyzer, synonymMap, fields, dimensions, vectorConfiguration) = field + + assert(name == "vectorCol") + assert(fieldType == "Collection(Edm.Single)") + assert(Seq(searchable, filterable, sortable, facetable, retrievable, key).forall(_.isEmpty)) + assert(Seq(analyzer, searchAnalyzer, indexAnalyzer).forall(_.isEmpty)) + assert(synonymMap.isEmpty && fields.isEmpty) + assert(dimensions.contains(3) && vectorConfiguration.contains("vectorConfig")) + assert(field.copy(name = "copy").productArity == 15) + } + + test("public parsers accept modern aliases without changing the published model") { + val parsed = new IndexParser {}.parseIndexJson(modernIndexJson) + val field = parsed.fields.find(_.name == "vectorCol").get + + assert(field.vectorSearchConfiguration.contains("vectorProfile")) + assert(parsed.vectorSearch.get.algorithmConfigurations.map(_.name) == + Seq("hnswConfig", "exhaustiveConfig")) + + val publicJson = parsed.toJson.asJsObject + assert(objectAt(publicJson, "vectorSearch").fields.contains("algorithmConfigurations")) + assert(vectorField(publicJson).fields.contains("vectorSearchConfiguration")) + } + + test("legacy JSON is modernized by renaming only required keys") { + val original = parse(legacyIndexJson) + val aligned = VectorSchema.align(original, "2023-10-01-Preview") + val originalVectorSearch = objectAt(original, "vectorSearch") + val alignedVectorSearch = objectAt(aligned, "vectorSearch") + val alignedField = vectorField(aligned) + + assert(alignedVectorSearch.fields("algorithms") == + originalVectorSearch.fields("algorithmConfigurations")) + assert(alignedVectorSearch.fields("futureVectorSearchOption") == + originalVectorSearch.fields("futureVectorSearchOption")) + assert(alignedVectorSearch.fields("profiles") == JsArray(Vector( + JsObject("name" -> JsString("hnswConfig"), "algorithm" -> JsString("hnswConfig")), + JsObject("name" -> JsString("exhaustiveConfig"), "algorithm" -> JsString("exhaustiveConfig"))))) + assert(!alignedVectorSearch.fields.contains("algorithmConfigurations")) + + assert(alignedField.fields("vectorSearchProfile") == JsString("hnswConfig")) + assert(alignedField.fields("searchable").convertTo[Boolean]) + assert(alignedField.fields("futureFieldOption") == vectorField(original).fields("futureFieldOption")) + assert(!alignedField.fields.contains("vectorSearchConfiguration")) + assert(aligned.asJsObject.fields("futureRootOption") == original.asJsObject.fields("futureRootOption")) + } + + test("the actual REST entity preserves complete modern vector JSON") { + val original = parse(modernIndexJson) + val entity = parse(SearchIndex.prepareEntity(modernIndexJson, "2026-04-01")) + + assert(entity == original) + val vectorSearchKeys = objectAt(entity, "vectorSearch").fields.keySet + assert(Seq("algorithms", "profiles", "vectorizers", "compressions").forall(vectorSearchKeys)) + } + + test("legacy REST preparation preserves parameters and unknown fields") { + val prepared = parse(SearchIndex.prepareEntity(legacyIndexJson, "2026-04-01")) + val originalVectorSearch = objectAt(parse(legacyIndexJson), "vectorSearch") + val preparedVectorSearch = objectAt(prepared, "vectorSearch") + + assert(preparedVectorSearch.fields("algorithms") == + originalVectorSearch.fields("algorithmConfigurations")) + assert(preparedVectorSearch.fields("futureVectorSearchOption") == + originalVectorSearch.fields("futureVectorSearchOption")) + assert(prepared.asJsObject.fields("futureRootOption") == + parse(legacyIndexJson).asJsObject.fields("futureRootOption")) + } + + test("legacy api path is explicit and refuses lossy modern downgrades") { + val legacy = parse(legacyIndexJson) + assert(VectorSchema.align(legacy, "2023-07-01-Preview") == legacy) + + val error = intercept[IllegalArgumentException] { + VectorSchema.align(parse(modernIndexJson), "2023-07-01-Preview") + } + assert(error.getMessage.contains("cannot be losslessly sent")) + assert(error.getMessage.contains("2023-10-01-Preview")) + } + + test("existing legacy indexes fail early instead of implying an automatic migration") { + val error = intercept[IllegalArgumentException] { + VectorSchema.requireCompatibleExistingIndex(parse(legacyIndexJson), "2026-04-01") + } + assert(error.getMessage.contains("createIfNoneExists does not update or migrate")) + assert(error.getMessage.contains("earlier than 2023-10-01-Preview")) + assert(error.getMessage.contains("Create or Update Index")) + + VectorSchema.requireCompatibleExistingIndex(parse(legacyIndexJson), "2023-07-01-Preview") + VectorSchema.requireCompatibleExistingIndex(parse(modernIndexJson), "2026-04-01") + } + + test("modern API responses that normalize legacy indexes are accepted") { + // This is the verbatim shape the live service returns when an index created with the + // legacy vector schema (algorithmConfigurations + vectorSearchConfiguration) is read + // under api-version 2026-04-01: the algorithm survives, but `profiles` is empty and the + // field's `vectorSearchProfile` is null. Rejecting this would break every existing + // SynapseML user who created a vector index with an earlier release and never pinned + // an apiVersion. SynapseML never rewrites an index that already exists, so writing + // documents to it is safe. + val normalizedLegacyIndex = + """ + |{ + | "name": "legacy-index", + | "fields": [ + | { "name": "id", "type": "Edm.String", "key": true, "dimensions": null, + | "vectorSearchProfile": null }, + | { + | "name": "vectorCol", + | "type": "Collection(Edm.Single)", + | "dimensions": 3, + | "vectorSearchProfile": null + | } + | ], + | "vectorSearch": { + | "algorithms": [ { "name": "vectorConfig", "kind": "hnsw" } ], + | "profiles": [], + | "vectorizers": [], + | "compressions": [] + | } + |} + """.stripMargin + + VectorSchema.requireCompatibleExistingIndex(parse(normalizedLegacyIndex), "2026-04-01") + + // The vector column must still be recognised as a vector, otherwise makeColsCompatible + // would stop converting Spark ML Vector columns and schema parity would fail. + val fields = normalizedLegacyIndex.parseJson.convertTo[IndexInfo].fields + assert(fields.find(_.name == "vectorCol").exists(_.isVectorField)) + assert(fields.find(_.name == "id").exists(!_.isVectorField)) + } + + test("existing modern indexes reject a legacy api version") { + val error = intercept[IllegalArgumentException] { + VectorSchema.requireCompatibleExistingIndex(parse(modernIndexJson), "2023-07-01-Preview") + } + assert(error.getMessage.contains("profile-based vector schema")) + assert(error.getMessage.contains("2023-10-01-Preview or later")) + } + + test("createIfNoneExists validates the actual remote index with one list and one GET") { + val remoteLegacyJson = legacyIndexJson.replace("\"legacy-index\"", "\"modern-index\"") + val client = new RecordingSearchIndexClient("modern-index", remoteLegacyJson) + + val error = intercept[IllegalArgumentException] { + SearchIndex.createIfNoneExists( + AzureSearchAuth(), "service", modernIndexJson, "2026-04-01", client) + } + + assert(error.getMessage.contains("legacy vector schema")) + assert(client.listCalls == 1) + assert(client.getCalls == 1) + assert(client.createCalls == 0) + } + + test("internal index GET closes responses after successful and failed reads") { + val successResponse = new TrackingResponse(new StringEntity("""{"name":"index"}""")) + val successJson = IndexJsonReader.read( + new HttpGet("https://example.test/index"), _ => successResponse) + + assert(successJson == """{"name":"index"}""") + assert(successResponse.closed) + + val failingEntity = new BasicHttpEntity() + failingEntity.setContent(new InputStream { + override def read(): Int = throw new IOException("expected read failure") + }) + val failingResponse = new TrackingResponse(failingEntity) + + assertThrows[IOException] { + IndexJsonReader.read(new HttpGet("https://example.test/failing-index"), _ => failingResponse) + } + assert(failingResponse.closed) + } + + test("nested fields are modernized without dropping unrelated content") { + val nested = + """ + |{ + | "name": "nested-index", + | "fields": [ + | { + | "name": "parent", + | "type": "Edm.ComplexType", + | "futureParentOption": true, + | "fields": [ + | { + | "name": "vectorCol", + | "type": "Collection(Edm.Single)", + | "dimensions": 3, + | "vectorSearchConfiguration": "vectorConfig", + | "futureChildOption": "kept" + | } + | ] + | } + | ], + | "vectorSearch": { + | "algorithmConfigurations": [ { "name": "vectorConfig", "kind": "hnsw" } ] + | } + |} + """.stripMargin + + val aligned = VectorSchema.align(parse(nested), "2026-04-01") + val parent = aligned.asJsObject.fields("fields").asInstanceOf[JsArray].elements.head.asJsObject + val child = parent.fields("fields").asInstanceOf[JsArray].elements.head.asJsObject + + assert(parent.fields("futureParentOption").convertTo[Boolean]) + assert(child.fields("futureChildOption") == JsString("kept")) + assert(child.fields("vectorSearchProfile") == JsString("vectorConfig")) + assert(child.fields("searchable").convertTo[Boolean]) + } + + test("a null dimensions placeholder is not treated as a vector field") { + val nullDimensions = parse( + """ + |{ + | "name": "null-dimensions-index", + | "fields": [ + | { + | "name": "notAVector", + | "type": "Edm.String", + | "dimensions": null, + | "vectorSearchConfiguration": "vectorConfig" + | } + | ], + | "vectorSearch": { + | "algorithmConfigurations": [ { "name": "vectorConfig", "kind": "hnsw" } ] + | } + |} + """.stripMargin) + + val aligned = VectorSchema.align(nullDimensions, "2026-04-01") + val field = aligned.asJsObject.fields("fields").asInstanceOf[JsArray].elements.head.asJsObject + + // The reference is still modernized, but a null placeholder must not force searchable=true. + assert(field.fields("vectorSearchProfile") == JsString("vectorConfig")) + assert(!field.fields.contains("searchable")) + } + + test("non-vector indexes are unchanged for both schema generations") { + val plain = parse( + """{"name":"plain-index","fields":[{"name":"id","type":"Edm.String","key":true}],"future":7}""") + + assert(VectorSchema.align(plain, "2026-04-01") == plain) + assert(VectorSchema.align(plain, "2023-07-01-Preview") == plain) + } +} diff --git a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/translate/TextTranslatorCoreSuite.scala b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/translate/TextTranslatorCoreSuite.scala new file mode 100644 index 00000000000..25abe5f1091 --- /dev/null +++ b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/translate/TextTranslatorCoreSuite.scala @@ -0,0 +1,340 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.services.translate + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.http.client.methods.HttpPost +import org.apache.http.util.EntityUtils +import org.apache.spark.sql.Row +import org.apache.spark.sql.types.ArrayType +import org.apache.spark.sql.types.StructType + +import java.net.URLDecoder + +private[translate] class TestableTranslate extends Translate { + def buildRequest(schema: StructType, row: Row): Option[HttpPost] = + inputFunc(schema)(row).map(_.asInstanceOf[HttpPost]) +} + +private[translate] class TestableTransliterate extends Transliterate { + def buildRequest(schema: StructType, row: Row): Option[HttpPost] = + inputFunc(schema)(row).map(_.asInstanceOf[HttpPost]) +} + +private[translate] class TestableDetect extends Detect { + def buildRequest(schema: StructType, row: Row): Option[HttpPost] = + inputFunc(schema)(row).map(_.asInstanceOf[HttpPost]) +} + +private[translate] class TestableBreakSentence extends BreakSentence { + def buildRequest(schema: StructType, row: Row): Option[HttpPost] = + inputFunc(schema)(row).map(_.asInstanceOf[HttpPost]) +} + +private[translate] class TestableDictionaryLookup extends DictionaryLookup { + def buildRequest(schema: StructType, row: Row): Option[HttpPost] = + inputFunc(schema)(row).map(_.asInstanceOf[HttpPost]) +} + +private[translate] class TestableDictionaryExamples extends DictionaryExamples { + def buildRequest(schema: StructType, row: Row): Option[HttpPost] = + inputFunc(schema)(row).map(_.asInstanceOf[HttpPost]) +} + +class TextTranslatorCoreSuite extends TestBase { + + import spark.implicits._ + + private def toQueryMap(post: HttpPost): Map[String, String] = { + Option(post.getURI.getRawQuery).toSeq.flatMap(_.split("&")).map { kv => + val pair = kv.split("=", 2) + val key = URLDecoder.decode(pair(0), "UTF-8") + val value = if (pair.length > 1) URLDecoder.decode(pair(1), "UTF-8") else "" + key -> value + }.toMap + } + + test("setLocation sets translator endpoint and subscription region") { + val global = new Translate().setLocation("eastus") + assert(global.getSubscriptionRegion == "eastus") + assert(global.getUrl == "https://api.cognitive.microsofttranslator.com/translate") + + val usGov = new Translate().setLocation("usgovarizona") + assert(usGov.getUrl == "https://api.cognitive.microsofttranslator.us/translate") + + val china = new Translate().setLocation("chinanorth") + assert(china.getUrl == "https://api.cognitive.microsofttranslator.cn/translate") + } + + test("translate defaults are deterministic") { + val t = new Translate() + assert(t.getOrDefault(t.textType) == Left("plain")) + assert(t.getOrDefault(t.category) == Left("general")) + assert(t.getOrDefault(t.profanityAction) == Left("NoAction")) + assert(t.getOrDefault(t.profanityMarker) == Left("Asterisk")) + assertResult(Left(false))(t.getOrDefault(t.includeAlignment)) + assertResult(Left(false))(t.getOrDefault(t.includeSentenceLength)) + assertResult(Left(true))(t.getOrDefault(t.allowFallback)) + } + + test("translate rejects invalid enum parameters") { + intercept[IllegalArgumentException] { + new Translate().setTextType("markdown") + } + intercept[IllegalArgumentException] { + new Translate().setProfanityAction("Mask") + } + intercept[IllegalArgumentException] { + new Translate().setProfanityMarker("Bracket") + } + } + + test("translate request building maps query params and body deterministically") { + val df = Seq((Seq("hello", "world"), Seq("de", "fr"), "en")) + .toDF("text", "toLanguage", "fromLanguage") + + val t = new TestableTranslate() + .setSubscriptionKey("fake-key") + .setLocation("eastus") + .setTextCol("text") + .setToLanguageCol("toLanguage") + .setFromLanguageCol("fromLanguage") + + val request = t.buildRequest(df.schema, df.head()).get + val query = toQueryMap(request) + assert(query("api-version") == "3.0") + assert(query("from") == "en") + assert(query("to") == "de,fr") + assert(query("textType") == "plain") + assert(query("category") == "general") + assert(query("profanityAction") == "NoAction") + assert(query("profanityMarker") == "Asterisk") + assert(query("includeAlignment") == "false") + assert(query("includeSentenceLength") == "false") + assert(query("allowFallback") == "true") + assert(request.getFirstHeader("Ocp-Apim-Subscription-Key").getValue == "fake-key") + assert(request.getFirstHeader("Ocp-Apim-Subscription-Region").getValue == "eastus") + assert(request.getFirstHeader("Content-Type").getValue == "application/json; charset=UTF-8") + assert(EntityUtils.toString(request.getEntity, "UTF-8") == """[{"Text":"hello"},{"Text":"world"}]""") + } + + test("translate request building skips empty or missing text and targets") { + val t = new TestableTranslate() + .setLocation("eastus") + .setTextCol("text") + .setToLanguageCol("toLanguage") + + val emptyTextDf = Seq((Seq.empty[String], Seq("de"))).toDF("text", "toLanguage") + assert(t.buildRequest(emptyTextDf.schema, emptyTextDf.head()).isEmpty) + + val emptyToDf = Seq((Seq("hello"), Seq.empty[String])).toDF("text", "toLanguage") + assert(t.buildRequest(emptyToDf.schema, emptyToDf.head()).isEmpty) + + val nullToDf = Seq((Seq("hello"), Option.empty[Seq[String]])).toDF("text", "toLanguage") + assert(t.buildRequest(nullToDf.schema, nullToDf.head()).isEmpty) + } + + test("translate transformSchema adds output and error columns without temp columns") { + val input = Seq(("hello", "de")).toDF("text", "toLanguage") + val t = new Translate() + .setTextCol("text") + .setToLanguageCol("toLanguage") + .setOutputCol("translation") + .setErrorCol("translationError") + + val schema = t.transformSchema(input.schema) + assert(schema.fieldNames.toSet == Set("text", "toLanguage", "translation", "translationError")) + assert(schema("translation").dataType == ArrayType(TranslateResponse.schema)) + } + + test("translate validates required parameters during schema creation") { + val textOnly = Seq("hello").toDF("text") + val err = intercept[AssertionError] { + new Translate().setTextCol("text").transformSchema(textOnly.schema) + } + assert(err.getMessage.contains("Missing required params")) + assert(err.getMessage.contains("toLanguage")) + } + + test("transliterate request building maps required params and body deterministically") { + val df = Seq((Seq("こんにちは"), "ja", "Jpan", "Latn")).toDF("text", "language", "fromScript", "toScript") + + val t = new TestableTransliterate() + .setSubscriptionKey("fake-key") + .setLocation("eastus") + .setTextCol("text") + .setLanguageCol("language") + .setFromScriptCol("fromScript") + .setToScriptCol("toScript") + + val request = t.buildRequest(df.schema, df.head()).get + val query = toQueryMap(request) + assert(request.getURI.getPath.endsWith("/transliterate")) + assert(query("api-version") == "3.0") + assert(query("language") == "ja") + assert(query("fromScript") == "Jpan") + assert(query("toScript") == "Latn") + assert(request.getFirstHeader("Ocp-Apim-Subscription-Key").getValue == "fake-key") + assert(request.getFirstHeader("Ocp-Apim-Subscription-Region").getValue == "eastus") + assert(EntityUtils.toString(request.getEntity, "UTF-8") == """[{"Text":"こんにちは"}]""") + } + + test("detect and breaksentence request building is deterministic offline") { + val detectDf = Seq(Seq("hello", "world")).toDF("text") + val detectRequest = new TestableDetect() + .setLocation("eastus") + .setTextCol("text") + .buildRequest(detectDf.schema, detectDf.head()) + .get + assert(detectRequest.getURI.getPath.endsWith("/detect")) + assert(toQueryMap(detectRequest) == Map("api-version" -> "3.0")) + assert(EntityUtils.toString(detectRequest.getEntity, "UTF-8") == """[{"Text":"hello"},{"Text":"world"}]""") + + val breakDf = Seq((Seq("hello"), "en", "Latn")).toDF("text", "language", "script") + val breakRequest = new TestableBreakSentence() + .setLocation("eastus") + .setTextCol("text") + .setLanguageCol("language") + .setScriptCol("script") + .buildRequest(breakDf.schema, breakDf.head()) + .get + val breakQuery = toQueryMap(breakRequest) + assert(breakRequest.getURI.getPath.endsWith("/breaksentence")) + assert(breakQuery("api-version") == "3.0") + assert(breakQuery("language") == "en") + assert(breakQuery("script") == "Latn") + assert(EntityUtils.toString(breakRequest.getEntity, "UTF-8") == """[{"Text":"hello"}]""") + } + + test("dictionary lookup and examples request building maps query params and body") { + val lookupDf = Seq((Seq("fly"), "en", "es")).toDF("text", "fromLanguage", "toLanguage") + val lookupRequest = new TestableDictionaryLookup() + .setSubscriptionKey("fake-key") + .setLocation("eastus") + .setTextCol("text") + .setFromLanguageCol("fromLanguage") + .setToLanguageCol("toLanguage") + .buildRequest(lookupDf.schema, lookupDf.head()) + .get + val lookupQuery = toQueryMap(lookupRequest) + assert(lookupRequest.getURI.getPath.endsWith("/dictionary/lookup")) + assert(lookupQuery("api-version") == "3.0") + assert(lookupQuery("from") == "en") + assert(lookupQuery("to") == "es") + assert(EntityUtils.toString(lookupRequest.getEntity, "UTF-8") == """[{"Text":"fly"}]""") + + val examplesDf = Seq((Seq(TextAndTranslation("fly", "volar")), "en", "es")) + .toDF("textAndTranslation", "fromLanguage", "toLanguage") + val examplesRequest = new TestableDictionaryExamples() + .setLocation("eastus") + .setTextAndTranslationCol("textAndTranslation") + .setFromLanguageCol("fromLanguage") + .setToLanguageCol("toLanguage") + .buildRequest(examplesDf.schema, examplesDf.head()) + .get + val examplesQuery = toQueryMap(examplesRequest) + assert(examplesRequest.getURI.getPath.endsWith("/dictionary/examples")) + assert(examplesQuery("api-version") == "3.0") + assert(examplesQuery("from") == "en") + assert(examplesQuery("to") == "es") + assert(EntityUtils.toString(examplesRequest.getEntity, "UTF-8") == """[{"Text":"fly","Translation":"volar"}]""") + } + + test("dictionary examples request building supports scalar text and translation input") { + val request = new TestableDictionaryExamples() + .setLocation("eastus") + .setFromLanguage("en") + .setToLanguage("es") + .setTextAndTranslation(TextAndTranslation("fly", "volar")) + .buildRequest(StructType(Seq.empty), Row.empty) + .get + val query = toQueryMap(request) + assert(request.getURI.getPath.endsWith("/dictionary/examples")) + assert(query("api-version") == "3.0") + assert(query("from") == "en") + assert(query("to") == "es") + assert(EntityUtils.toString(request.getEntity, "UTF-8") == """[{"Text":"fly","Translation":"volar"}]""") + } + + test("non-translate transformSchema adds deterministic output and error columns") { + val textOnly = Seq("hello").toDF("text") + val textAndTranslationOnly = Seq(Seq(TextAndTranslation("fly", "volar"))).toDF("textAndTranslation") + + val transliterateSchema = new Transliterate() + .setTextCol("text") + .setLanguage("ja") + .setFromScript("Jpan") + .setToScript("Latn") + .setOutputCol("transliteration") + .setErrorCol("transliterationError") + .transformSchema(textOnly.schema) + assert(transliterateSchema.fieldNames.toSet == Set("text", "transliteration", "transliterationError")) + assert(transliterateSchema("transliteration").dataType == ArrayType(TransliterateResponse.schema)) + + val detectSchema = new Detect() + .setTextCol("text") + .setOutputCol("detection") + .setErrorCol("detectionError") + .transformSchema(textOnly.schema) + assert(detectSchema.fieldNames.toSet == Set("text", "detection", "detectionError")) + assert(detectSchema("detection").dataType == ArrayType(DetectResponse.schema)) + + val breakSentenceSchema = new BreakSentence() + .setTextCol("text") + .setOutputCol("sentenceBreaks") + .setErrorCol("breakError") + .transformSchema(textOnly.schema) + assert(breakSentenceSchema.fieldNames.toSet == Set("text", "sentenceBreaks", "breakError")) + assert(breakSentenceSchema("sentenceBreaks").dataType == ArrayType(BreakSentenceResponse.schema)) + + val lookupSchema = new DictionaryLookup() + .setTextCol("text") + .setFromLanguage("en") + .setToLanguage("es") + .setOutputCol("lookup") + .setErrorCol("lookupError") + .transformSchema(textOnly.schema) + assert(lookupSchema.fieldNames.toSet == Set("text", "lookup", "lookupError")) + assert(lookupSchema("lookup").dataType == ArrayType(DictionaryLookupResponse.schema)) + + val examplesSchema = new DictionaryExamples() + .setTextAndTranslationCol("textAndTranslation") + .setFromLanguage("en") + .setToLanguage("es") + .setOutputCol("examples") + .setErrorCol("examplesError") + .transformSchema(textAndTranslationOnly.schema) + assert(examplesSchema.fieldNames.toSet == Set("textAndTranslation", "examples", "examplesError")) + assert(examplesSchema("examples").dataType == ArrayType(DictionaryExamplesResponse.schema)) + } + + test("non-translate classes validate required parameters during schema creation") { + val textOnly = Seq("hello").toDF("text") + val textAndTranslationOnly = Seq(Seq(TextAndTranslation("fly", "volar"))).toDF("textAndTranslation") + + val transliterateError = intercept[AssertionError] { + new Transliterate().setTextCol("text").transformSchema(textOnly.schema) + } + assert(transliterateError.getMessage.contains("Missing required params")) + assert(transliterateError.getMessage.contains("language")) + assert(transliterateError.getMessage.contains("fromScript")) + assert(transliterateError.getMessage.contains("toScript")) + + val dictionaryLookupError = intercept[AssertionError] { + new DictionaryLookup().setTextCol("text").transformSchema(textOnly.schema) + } + assert(dictionaryLookupError.getMessage.contains("Missing required params")) + assert(dictionaryLookupError.getMessage.contains("fromLanguage")) + assert(dictionaryLookupError.getMessage.contains("toLanguage")) + + val dictionaryExamplesError = intercept[AssertionError] { + new DictionaryExamples() + .setTextAndTranslationCol("textAndTranslation") + .transformSchema(textAndTranslationOnly.schema) + } + assert(dictionaryExamplesError.getMessage.contains("Missing required params")) + assert(dictionaryExamplesError.getMessage.contains("fromLanguage")) + assert(dictionaryExamplesError.getMessage.contains("toLanguage")) + } +} diff --git a/core/src/main/python/synapse/ml/stages/EnsembleByKey.py b/core/src/main/python/synapse/ml/stages/EnsembleByKey.py new file mode 100644 index 00000000000..4de593ac0b2 --- /dev/null +++ b/core/src/main/python/synapse/ml/stages/EnsembleByKey.py @@ -0,0 +1,13 @@ +# Copyright (C) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE in project root for information. + +from pyspark.ml.common import inherit_doc +from synapse.ml.stages._EnsembleByKey import _EnsembleByKey + + +@inherit_doc +class EnsembleByKey(_EnsembleByKey): + def getColNames(self): + if self.isSet(self.colNames): + return self.getOrDefault(self.colNames) + return [f"{self.getStrategy()}({name})" for name in self.getCols()] diff --git a/core/src/main/python/synapse/ml/stages/LumpFeaturesModel.py b/core/src/main/python/synapse/ml/stages/LumpFeaturesModel.py new file mode 100644 index 00000000000..767d04aa01c --- /dev/null +++ b/core/src/main/python/synapse/ml/stages/LumpFeaturesModel.py @@ -0,0 +1,29 @@ +# Copyright (C) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE in project root for information. + +import json +import sys + +if sys.version >= "3": + basestring = str + +from synapse.ml.core.schema.Utils import * +from synapse.ml.stages._LumpFeaturesModel import _LumpFeaturesModel + + +@inherit_doc +class LumpFeaturesModel(_LumpFeaturesModel): + def getKeptValues(self): + """Returns the values LumpFeatures learned to retain, per rule column. + + Every value not listed here is replaced with otherValue at transform time, so this is the + authoritative view of what the fitted model will keep. Use it to audit a fit before scoring: + a column whose list is much shorter than its top-K means the frequency filters (minCount, + minFreq) removed the tail, and an empty list means every value gets lumped. + + Returns: + + dict: map from rule column name to its retained values, ordered by descending + frequency in the fitting data and then ascending value. + """ + return json.loads(self._call_java("getKeptValuesAsJson")) diff --git a/core/src/main/scala/com/microsoft/azure/synapse/ml/automl/EvaluationUtils.scala b/core/src/main/scala/com/microsoft/azure/synapse/ml/automl/EvaluationUtils.scala index f2c0cd17141..c2c06e0bac8 100644 --- a/core/src/main/scala/com/microsoft/azure/synapse/ml/automl/EvaluationUtils.scala +++ b/core/src/main/scala/com/microsoft/azure/synapse/ml/automl/EvaluationUtils.scala @@ -56,7 +56,10 @@ object EvaluationUtils { case _ => throw new Exception("Metric is not supported for regressors") } case SchemaConstants.ClassificationKind => evaluationMetric match { - case MetricConstants.AucSparkMetric => (MetricConstants.AucColumnName, chooseHighest) + case MetricConstants.AucSparkMetric | MetricConstants.AreaUnderROCMetric => + (MetricConstants.AucColumnName, chooseHighest) + case MetricConstants.AreaUnderPRMetric => + (MetricConstants.AreaUnderPRColumnName, chooseHighest) case MetricConstants.PrecisionSparkMetric => (MetricConstants.PrecisionColumnName, chooseHighest) case MetricConstants.RecallSparkMetric => (MetricConstants.RecallColumnName, chooseHighest) case MetricConstants.AccuracySparkMetric => (MetricConstants.AccuracyColumnName, chooseHighest) diff --git a/core/src/main/scala/com/microsoft/azure/synapse/ml/automl/FindBestModel.scala b/core/src/main/scala/com/microsoft/azure/synapse/ml/automl/FindBestModel.scala index e43bf427f2a..ce2a00b6e75 100644 --- a/core/src/main/scala/com/microsoft/azure/synapse/ml/automl/FindBestModel.scala +++ b/core/src/main/scala/com/microsoft/azure/synapse/ml/automl/FindBestModel.scala @@ -29,8 +29,9 @@ trait FindBestModelParams extends Wrappable with ComplexParamsWritable with HasE * The metrics that can be chosen are: * * For Binary Classifiers: - * - AreaUnderROC + * - areaUnderROC (reported in the AUC column) * - AUC + * - areaUnderPR * - accuracy * - precision * - recall diff --git a/core/src/main/scala/com/microsoft/azure/synapse/ml/automl/HyperparamBuilder.scala b/core/src/main/scala/com/microsoft/azure/synapse/ml/automl/HyperparamBuilder.scala index 475466bf658..31e9f20b13d 100644 --- a/core/src/main/scala/com/microsoft/azure/synapse/ml/automl/HyperparamBuilder.scala +++ b/core/src/main/scala/com/microsoft/azure/synapse/ml/automl/HyperparamBuilder.scala @@ -5,6 +5,7 @@ package com.microsoft.azure.synapse.ml.automl import org.apache.spark.ml.param._ +import scala.annotation.tailrec import scala.collection.JavaConverters._ import scala.collection.mutable import scala.util.Random @@ -29,8 +30,40 @@ class LongRangeHyperParam(min: Long, max: Long, seed: Long = 0) extends RangeHyperParam[Long](min, max, seed) { def getNext(): Long = { + require(max >= min, s"LongRangeHyperParam requires max >= min, got min=$min, max=$max") val range = max - min - (random.nextLong() * range) + min + // nextLong() spans the full 64-bit range, so multiplying it by the range overflows and + // escapes [min, max) entirely. Reduce into the range first, as nextInt(range) does. + if (range == 0) { + min + } else if (range < 0) { + // The span is wider than Long.MaxValue so it cannot be represented as a Long. Draw from + // the whole 64-bit range instead; more than half of it lands in bounds, so this is cheap. + @tailrec def drawInBounds(): Long = { + val v = random.nextLong() + if (v >= min && v < max) v else drawInBounds() + } + drawInBounds() + } else { + boundedNextLong(range) + min + } + } + + /** Uniform draw from [0, bound), rejecting the partial final block so the result carries no + * modulo bias. Mirrors what java.util.Random.nextInt(bound) already does for Int. + */ + private def boundedNextLong(bound: Long): Long = { + val m = bound - 1 + if ((bound & m) == 0L) { + random.nextLong() & m // bound is a power of two, so the low bits are already uniform + } else { + @tailrec def draw(): Long = { + val u = random.nextLong() >>> 1 + val r = u % bound + if (u + m - r < 0L) draw() else r + } + draw() + } } } @@ -38,7 +71,7 @@ class LongRangeHyperParam(min: Long, max: Long, seed: Long = 0) class FloatRangeHyperParam(min: Float, max: Float, seed: Long = 0) extends RangeHyperParam[Float](min, max, seed) { - val doubleRange = new DoubleRangeHyperParam(min.toDouble, max.toDouble) + val doubleRange = new DoubleRangeHyperParam(min.toDouble, max.toDouble, seed) def getNext(): Float = { doubleRange.getNext().toFloat } diff --git a/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/PyCodegen.scala b/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/PyCodegen.scala index c8c3ab09e09..867060181c6 100644 --- a/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/PyCodegen.scala +++ b/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/PyCodegen.scala @@ -129,6 +129,7 @@ object PyCodegen { | long_description="SynapseML contains Microsoft's open source " | + "contributions to the Apache Spark ecosystem", | license="MIT", + | license_expression="MIT", | packages=find_namespace_packages(include=['synapse.ml', 'synapse.ml.*']) ${extraPackage}, | url="https://github.com/Microsoft/SynapseML", | author="Microsoft", @@ -138,8 +139,6 @@ object PyCodegen { | "Intended Audience :: Developers", | "Intended Audience :: Science/Research", | "Topic :: Software Development :: Libraries", - | "License :: OSI Approved :: MIT License", - | "Programming Language :: Python :: 2", | "Programming Language :: Python :: 3", | ], | zip_safe=True, diff --git a/core/src/main/scala/com/microsoft/azure/synapse/ml/core/metrics/MetricConstants.scala b/core/src/main/scala/com/microsoft/azure/synapse/ml/core/metrics/MetricConstants.scala index 8047f535572..278578ec38a 100644 --- a/core/src/main/scala/com/microsoft/azure/synapse/ml/core/metrics/MetricConstants.scala +++ b/core/src/main/scala/com/microsoft/azure/synapse/ml/core/metrics/MetricConstants.scala @@ -17,14 +17,15 @@ object MetricConstants { // Binary Classification metrics val AreaUnderROCMetric = "areaUnderROC" + val AreaUnderPRMetric = "areaUnderPR" val AucSparkMetric = "AUC" val AccuracySparkMetric = "accuracy" val PrecisionSparkMetric = "precision" val RecallSparkMetric = "recall" val ClassificationMetricsName = "classification" - val ClassificationMetrics: Set[String] = Set(AreaUnderROCMetric, AucSparkMetric, AccuracySparkMetric, - PrecisionSparkMetric, RecallSparkMetric, ClassificationMetricsName) + val ClassificationMetrics: Set[String] = Set(AreaUnderROCMetric, AreaUnderPRMetric, AucSparkMetric, + AccuracySparkMetric, PrecisionSparkMetric, RecallSparkMetric, ClassificationMetricsName) val AllSparkMetrics = "all" @@ -35,7 +36,8 @@ object MetricConstants { val MaeColumnName = "mean_absolute_error" // Binary Classification column names - val AucColumnName = "AUC" + val AucColumnName = "AUC" + val AreaUnderPRColumnName = "areaUnderPR" // Binary and Multiclass (micro-averaged) column names val PrecisionColumnName = "precision" @@ -50,7 +52,10 @@ object MetricConstants { val ConfusionMatrix = "confusion_matrix" // Metric to column name - val MetricToColumnName: Map[String, String] = Map(AccuracySparkMetric -> AccuracyColumnName, + val MetricToColumnName: Map[String, String] = Map(AreaUnderROCMetric -> AucColumnName, + AucSparkMetric -> AucColumnName, + AreaUnderPRMetric -> AreaUnderPRColumnName, + AccuracySparkMetric -> AccuracyColumnName, PrecisionSparkMetric -> PrecisionColumnName, RecallSparkMetric -> RecallColumnName, MseSparkMetric -> MseColumnName, @@ -58,7 +63,9 @@ object MetricConstants { R2SparkMetric -> R2ColumnName, MaeSparkMetric -> MaeColumnName) + // Shared by binary and multiclass metrics; preserved for the legacy all/classification schema contract. val ClassificationColumns = List(AccuracyColumnName, PrecisionColumnName, RecallColumnName) + val BinaryClassificationColumns = ClassificationColumns ++ List(AucColumnName, AreaUnderPRColumnName) val RegressionColumns = List(MseColumnName, RmseColumnName, R2ColumnName, MaeColumnName) @@ -92,6 +99,8 @@ object MetricConstants { MetricConstants.AccuracySparkMetric, MetricConstants.PrecisionSparkMetric, MetricConstants.RecallSparkMetric, - MetricConstants.AucSparkMetric) + MetricConstants.AreaUnderROCMetric, + MetricConstants.AucSparkMetric, + MetricConstants.AreaUnderPRMetric) } diff --git a/core/src/main/scala/com/microsoft/azure/synapse/ml/core/metrics/MetricUtils.scala b/core/src/main/scala/com/microsoft/azure/synapse/ml/core/metrics/MetricUtils.scala index dc2bb6184c7..bd211333f48 100644 --- a/core/src/main/scala/com/microsoft/azure/synapse/ml/core/metrics/MetricUtils.scala +++ b/core/src/main/scala/com/microsoft/azure/synapse/ml/core/metrics/MetricUtils.scala @@ -4,13 +4,87 @@ package com.microsoft.azure.synapse.ml.core.metrics import com.microsoft.azure.synapse.ml.core.schema.SchemaConstants.MMLTag -import com.microsoft.azure.synapse.ml.core.schema.{SchemaConstants, SparkSchema} +import com.microsoft.azure.synapse.ml.core.schema.SchemaConstants import org.apache.spark.sql.types.injections.MetadataUtilities -import org.apache.spark.sql.types.{Metadata, StructField, StructType} +import org.apache.spark.sql.types.{Metadata, StructType} + +import scala.util.control.NonFatal /** Utilities used by modules for metrics. */ object MetricUtils { + private case class ScoredModelCandidate(modelName: String, + labelColumnName: String, + scoreValueKind: String, + predictionColumnName: String, + otherColumns: Seq[ScoreColumnMetadata]) { + def schemaInfo: (String, String, String) = (modelName, labelColumnName, scoreValueKind) + + def description: String = + s"$modelName (label=$labelColumnName, kind=$scoreValueKind, prediction=$predictionColumnName)" + + def signature(includeOtherColumns: Boolean): (String, String, String, Seq[(String, String, String)]) = + (labelColumnName, scoreValueKind, predictionColumnName, + if (includeOtherColumns) { + otherColumns.map(column => (column.columnKind, column.columnName, column.scoreValueKind)) + } else { + Seq.empty + }) + } + + private case class ScoreColumnMetadata(modelName: String, + columnName: String, + columnKind: String, + scoreValueKind: String) + + private case class ScoredModelMetadata(modelName: String, columns: Seq[ScoreColumnMetadata]) { + private val labelColumns = columns.filter(_.columnKind == SchemaConstants.TrueLabelsColumn) + private val predictionColumns = columns.filter(_.columnKind == SchemaConstants.SparkPredictionColumn) + private val otherColumns = columns + .filter(column => + column.columnKind == SchemaConstants.SparkRawPredictionColumn || + column.columnKind == SchemaConstants.SparkProbabilityColumn) + .sortBy(column => (column.columnKind, column.columnName, column.scoreValueKind)) + + def candidates: Seq[ScoredModelCandidate] = { + for { + label <- labelColumns + prediction <- predictionColumns + if label.scoreValueKind == prediction.scoreValueKind + if ValidScoreValueKinds.contains(label.scoreValueKind) + } yield ScoredModelCandidate( + modelName, + label.columnName, + label.scoreValueKind, + prediction.columnName, + otherColumns) + } + + def conflictDescription(labelCol: Option[String], + requestedKind: Option[String]): Option[String] = { + val couldMatchLabel = labelCol.forall(label => labelColumns.exists(_.columnName == label)) + val availableKinds = (labelColumns ++ predictionColumns).map(_.scoreValueKind).toSet + val couldMatchKind = requestedKind.forall(availableKinds.contains) + if (labelColumns.nonEmpty && predictionColumns.nonEmpty && + candidates.isEmpty && couldMatchLabel && couldMatchKind) { + val labels = describeColumns(labelColumns) + val predictions = describeColumns(predictionColumns) + Some(s"$modelName has incompatible label metadata $labels and prediction metadata $predictions") + } else { + None + } + } + + private def describeColumns(scoreColumns: Seq[ScoreColumnMetadata]): String = + scoreColumns + .map(column => s"${column.columnName}:${column.scoreValueKind}") + .distinct + .sorted + .mkString("[", ", ", "]") + } + + private val ValidScoreValueKinds = Set(SchemaConstants.ClassificationKind, SchemaConstants.RegressionKind) + def isClassificationMetric(metric: String): Boolean = { if (MetricConstants.RegressionMetrics.contains(metric)) false else if (MetricConstants.ClassificationMetrics.contains(metric)) true @@ -19,46 +93,177 @@ object MetricUtils { def getSchemaInfo(schema: StructType, labelCol: Option[String], evaluationMetric: String): (String, String, String) = { - val schemaInfo = tryGetSchemaInfo(schema) - if (schemaInfo.isDefined) { - schemaInfo.get - } else { - if (labelCol.isEmpty) { - throw new Exception("Please score the model prior to evaluating") - } else if (evaluationMetric == MetricConstants.AllSparkMetrics) { - throw new Exception("Please specify whether you are using evaluation for " + - MetricConstants.ClassificationMetricsName + " or " + MetricConstants.RegressionMetricsName + - " instead of " + MetricConstants.AllSparkMetrics) + getSchemaInfo(schema, labelCol, evaluationMetric, caseSensitive = true) + } + + private[ml] def getSchemaInfo(schema: StructType, + labelCol: Option[String], + evaluationMetric: String, + caseSensitive: Boolean): (String, String, String) = { + val resolvedLabelCol = resolveLabelColumn(schema, labelCol, caseSensitive) + val requestedKind = getRequestedScoreValueKind(evaluationMetric) + tryGetSchemaInfo( + schema, + resolvedLabelCol, + requestedKind, + requiresAuxiliaryMetadata(evaluationMetric, requestedKind)).map(_.schemaInfo).getOrElse { + (resolvedLabelCol, requestedKind) match { + case (Some(labelColumnName), Some(scoreValueKind)) => + ("custom model", labelColumnName, scoreValueKind) + case _ => + val missingSettings = Seq( + if (labelCol.isEmpty) Some("labelCol") else None, + if (requestedKind.isEmpty) Some("evaluationMetric") else None).flatten + val availableColumns = schema.fieldNames.sorted.mkString("[", ", ", "]") + val metricHint = + if (requestedKind.isEmpty) s" (evaluationMetric must not be '${MetricConstants.AllSparkMetrics}')" + else "" + throw new IllegalArgumentException( + "Unable to determine a complete scored model from schema metadata. " + + s"Set ${missingSettings.mkString(" and ")}$metricHint, " + + "or score the dataset so one model has both label and prediction metadata. " + + s"Available columns: $availableColumns") } - ("custom model", labelCol.get, - if (isClassificationMetric(evaluationMetric)) - SchemaConstants.ClassificationKind - else SchemaConstants.RegressionKind) } } - private def tryGetSchemaInfo(schema: StructType): Option[(String, String, String)] = { - // TODO: evaluate all models; for now, get first model name found - val firstModelName = schema.collectFirst { - case StructField(_, _, _, m) if getFirstModelName(m) != null && getFirstModelName(m).isDefined => - getFirstModelName(m).get + private[ml] def getScoreColumnName(schema: StructType, + modelName: String, + columnKind: String, + scoreValueKind: String): Option[String] = { + val matchingColumns = getScoredModelMetadata(schema) + .find(_.modelName == modelName) + .toSeq + .flatMap(_.columns) + .filter(_.columnKind == columnKind) + val descriptions = matchingColumns + .map(column => s"${column.columnName}:${column.scoreValueKind}") + .distinct + .sorted + matchingColumns.map(_.scoreValueKind).distinct match { + case Seq(kind) if kind == scoreValueKind && descriptions.size == 1 => + Some(matchingColumns.head.columnName) + case Seq() => None + case _ => + throw new IllegalArgumentException( + s"Conflicting scored-model metadata. $modelName has $columnKind columns " + + descriptions.mkString("[", ", ", "].")) + } + } + + private def getRequestedScoreValueKind(evaluationMetric: String): Option[String] = { + if (evaluationMetric == MetricConstants.AllSparkMetrics) None + else if (isClassificationMetric(evaluationMetric)) Some(SchemaConstants.ClassificationKind) + else Some(SchemaConstants.RegressionKind) + } + + private def requiresAuxiliaryMetadata(evaluationMetric: String, + requestedKind: Option[String]): Boolean = { + requestedKind match { + case Some(SchemaConstants.RegressionKind) => false + case Some(SchemaConstants.ClassificationKind) => + evaluationMetric != MetricConstants.AccuracySparkMetric && + evaluationMetric != MetricConstants.PrecisionSparkMetric && + evaluationMetric != MetricConstants.RecallSparkMetric + case _ => true + } + } + + private def tryGetSchemaInfo(schema: StructType, + labelCol: Option[String], + requestedKind: Option[String], + includeOtherColumns: Boolean): Option[ScoredModelCandidate] = { + val scoredModels = getScoredModelMetadata(schema) + val conflicts = scoredModels + .flatMap(_.conflictDescription(labelCol, requestedKind)) + .sorted + if (conflicts.nonEmpty) { + throw new IllegalArgumentException( + "Conflicting scored-model metadata. " + conflicts.mkString("[", ", ", "].")) } - if (firstModelName.isEmpty) None - else { - val modelName = firstModelName.get - val labelColumnName = SparkSchema.getLabelColumnName(schema, modelName) - val scoreValueKind = SparkSchema.getScoreValueKind(schema, modelName, labelColumnName) - Option((modelName, labelColumnName, scoreValueKind)) + + val matchingCandidates = scoredModels + .flatMap(_.candidates) + .filter(candidate => labelCol.forall(_ == candidate.labelColumnName)) + .filter(candidate => requestedKind.forall(_ == candidate.scoreValueKind)) + val distinctCandidates = matchingCandidates + .groupBy(_.signature(includeOtherColumns)) + .values + .map(_.minBy(_.modelName)) + .toSeq + .sortBy(candidate => + (candidate.modelName, candidate.labelColumnName, + candidate.scoreValueKind, candidate.predictionColumnName)) + + distinctCandidates match { + case Seq(candidate) => Some(candidate) + case candidates if candidates.nonEmpty => + throw new IllegalArgumentException( + "Ambiguous scored-model metadata. Multiple complete candidates match: " + + candidates.map(_.description).mkString("[", ", ", "]. ") + + "Set labelCol and evaluationMetric to narrow candidates. If metadata still overlaps, " + + "set scoredLabelsCol/scoresCol explicitly or remove stale score metadata.") + case _ => None } } - private def getFirstModelName(colMetadata: Metadata): Option[String] = { - if (!colMetadata.contains(MMLTag)) null //scalastyle:ignore null - else { - val mlTagMetadata = colMetadata.getMetadata(MMLTag) - val metadataKeys = MetadataUtilities.getMetadataKeys(mlTagMetadata) - metadataKeys.find(key => key.startsWith(SchemaConstants.ScoreModelPrefix)) + private[ml] def resolveLabelColumn(schema: StructType, + labelCol: Option[String], + caseSensitive: Boolean): Option[String] = { + labelCol.map { requestedLabel => + if (caseSensitive) { + requestedLabel + } else { + schema.fieldNames.filter(_.equalsIgnoreCase(requestedLabel)) match { + case Array(resolvedLabel) => resolvedLabel + case _ => requestedLabel + } + } } } + private def getScoredModelMetadata(schema: StructType): Seq[ScoredModelMetadata] = { + schema.fields + .flatMap(field => getScoreColumnMetadata(field.name, field.metadata)) + .groupBy(_.modelName) + .toSeq + .sortBy(_._1) + .map { case (modelName, columns) => + ScoredModelMetadata( + modelName, + columns.sortBy(column => (column.columnKind, column.columnName, column.scoreValueKind))) + } + } + + private def getScoreColumnMetadata(columnName: String, + colMetadata: Metadata): Seq[ScoreColumnMetadata] = { + getMetadata(colMetadata, MMLTag).toSeq.flatMap { mlTagMetadata => + MetadataUtilities.getMetadataKeys(mlTagMetadata) + .filter(_.startsWith(SchemaConstants.ScoreModelPrefix)) + .toSeq + .sorted + .flatMap { modelName => + for { + modelMetadata <- getMetadata(mlTagMetadata, modelName) + columnKind <- getString(modelMetadata, SchemaConstants.ScoreColumnKind) + scoreValueKind <- getString(modelMetadata, SchemaConstants.ScoreValueKind) + } yield ScoreColumnMetadata(modelName, columnName, columnKind, scoreValueKind) + } + } + } + + private def getMetadata(metadata: Metadata, key: String): Option[Metadata] = + try { + if (metadata.contains(key)) Some(metadata.getMetadata(key)) else None + } catch { + case NonFatal(_) => None + } + + private def getString(metadata: Metadata, key: String): Option[String] = + try { + if (metadata.contains(key)) Option(metadata.getString(key)) else None + } catch { + case NonFatal(_) => None + } + } diff --git a/core/src/main/scala/com/microsoft/azure/synapse/ml/core/utils/ModelEquality.scala b/core/src/main/scala/com/microsoft/azure/synapse/ml/core/utils/ModelEquality.scala index 797e82d122d..66ad00917ec 100644 --- a/core/src/main/scala/com/microsoft/azure/synapse/ml/core/utils/ModelEquality.scala +++ b/core/src/main/scala/com/microsoft/azure/synapse/ml/core/utils/ModelEquality.scala @@ -14,10 +14,13 @@ trait ParamEquality[T] extends Param[T] { object ModelEquality { + /** Similarity of two strings as the Jaccard index over their character bigrams. */ def jaccardSimilarity(s1: String, s2: String): Double = { - val a = Set(s1) - val b = Set(s2) - a.intersect(b).size.toDouble / (a | b).size.toDouble + // Sets of the whole strings would only ever yield 1.0 or 0.0, making this a strict + // equality check rather than a similarity measure. + val a = s1.toLowerCase.sliding(2).toSet + val b = s2.toLowerCase.sliding(2).toSet + if (a.isEmpty && b.isEmpty) 1.0 else a.intersect(b).size.toDouble / (a | b).size.toDouble } def assertEqual(m1: Params, m2: Params): Unit = { diff --git a/core/src/main/scala/com/microsoft/azure/synapse/ml/exploratory/DistributionBalanceMeasure.scala b/core/src/main/scala/com/microsoft/azure/synapse/ml/exploratory/DistributionBalanceMeasure.scala index 1b923c47c1f..374ab5775d3 100644 --- a/core/src/main/scala/com/microsoft/azure/synapse/ml/exploratory/DistributionBalanceMeasure.scala +++ b/core/src/main/scala/com/microsoft/azure/synapse/ml/exploratory/DistributionBalanceMeasure.scala @@ -239,7 +239,11 @@ private[exploratory] case class DistributionMetrics(numFeatures: Int, val averageObsRef = (col(obsFeatureProbCol) + col(refFeatureProbCol)) / 2d val entropyObsAvg = entropy(col(obsFeatureProbCol), Some(averageObsRef)) val entropyRefAvg = entropy(col(refFeatureProbCol), Some(averageObsRef)) - sqrt((entropyRefAvg + entropyObsAvg) / 2d) + // Keep KL divergence in natural-log units while normalizing only JS divergence to base 2. + val jsDivergenceBase2 = (entropyRefAvg + entropyObsAvg) / (2d * math.log(2d)) + // Floating-point aggregation can make a theoretically non-negative divergence slightly negative. + val nonNegativeJsDivergence = when(jsDivergenceBase2 < 0d, lit(0d)).otherwise(jsDivergenceBase2) + sqrt(nonNegativeJsDivergence) } def infNormDistance: Column = max(absDiffObsRef) diff --git a/core/src/main/scala/com/microsoft/azure/synapse/ml/logging/SynapseMLLogging.scala b/core/src/main/scala/com/microsoft/azure/synapse/ml/logging/SynapseMLLogging.scala index 2269592be27..ee0d197c263 100644 --- a/core/src/main/scala/com/microsoft/azure/synapse/ml/logging/SynapseMLLogging.scala +++ b/core/src/main/scala/com/microsoft/azure/synapse/ml/logging/SynapseMLLogging.scala @@ -41,7 +41,10 @@ case class RequiredErrorFields(errorType: String, def toMap: Map[String, String] = { Map( "errorType" -> errorType, - "errorMessage" -> errorType + // Exception.getMessage is null for exceptions constructed without one + // (e.g. new NullPointerException). spray-json's JsString rejects null, + // so serializing the payload would throw and mask the original error. + "errorMessage" -> Option(errorMessage).getOrElse("") ) } } @@ -64,7 +67,12 @@ object SynapseMLLogging extends Logging { private[ml] def getHadoopConfEntries: Map[String, String] = { SparkSession.getActiveSession.map { spark => - val hc = spark.sparkContext.hadoopConfiguration + // Session-derived rather than spark.sparkContext, which Databricks Unity Catalog standard + // access mode rejects by name. Matches Spark's own MLlib ReadWrite (SPARK-48909) and picks + // up session-level conf overrides that sparkContext.hadoopConfiguration misses. + // Intentionally uncached: these are session identity fields that a notebook can rebind + // mid-session, and the ~0.2ms cost is per fit/transform/construct call, never per row. + val hc = spark.sessionState.newHadoopConf() //noinspection ScalaStyle HadoopKeysToLog.flatMap { case (field, name) => Option(hc.get(field)).map { v: String => (name, v) } diff --git a/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/EnsembleByKey.scala b/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/EnsembleByKey.scala index 72484947c4f..47ea2bb7724 100644 --- a/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/EnsembleByKey.scala +++ b/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/EnsembleByKey.scala @@ -11,17 +11,83 @@ import org.apache.spark.ml.linalg.SQLDataTypes._ import org.apache.spark.ml.param._ import org.apache.spark.ml.stat.Summarizer import org.apache.spark.ml.util.{DefaultParamsReadable, DefaultParamsWritable, Identifiable} +import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute +import org.apache.spark.sql.catalyst.expressions.{Attribute, Cast, ExprId, RowOrdering} import org.apache.spark.sql.functions._ import org.apache.spark.sql.types._ -import org.apache.spark.sql.{DataFrame, Dataset} +import org.apache.spark.sql.{Column, DataFrame, Dataset, SparkSession} import scala.collection.JavaConverters._ - -object EnsembleByKey extends DefaultParamsReadable[EnsembleByKey] +import scala.util.Try + +object EnsembleByKey extends DefaultParamsReadable[EnsembleByKey] { + + // Spark's union analysis re-aliases duplicated child outputs and tags them with this key so that + // AttributeSeq.resolve can prune them before reporting an ambiguous reference. + private val DuplicateMetadataKey = "__is_duplicate" + + private case class PathStep(name: String, mapKeyType: Option[DataType]) + + private case class ResolvedField( + reference: String, + qualifier: Array[String], + path: Array[PathStep], + ordinals: Array[Int], + field: StructField) + + private case class ResolvedColumns( + inputFields: Array[ResolvedField], + outputNames: Array[String], + keyFields: Array[ResolvedField], + aggregateFields: Array[StructField], + caseSensitive: Boolean) + + private case class ResolvedStep( + fieldName: String, + dataType: DataType, + nullable: Boolean, + metadata: Metadata, + ordinal: Int, + mapKeyType: Option[DataType]) + + private case class FieldRole( + consumesInputColumn: Boolean, + declaredOutput: StructField => Option[StructField]) + + private case class QualifiedMatch( + qualifier: Array[String], + requestedPath: Array[String], + ordinal: Int, + exprId: ExprId) + + private def columnNamesMatch(left: String, right: String, caseSensitive: Boolean): Boolean = + if (caseSensitive) left == right else left.equalsIgnoreCase(right) + + private def resolveFieldAtLevel( + schema: StructType, + fieldName: String, + reference: String, + caseSensitive: Boolean + ): (StructField, Int) = { + schema.fields.zipWithIndex.filter { case (field, _) => + columnNamesMatch(field.name, fieldName, caseSensitive) + } match { + case Array(result) => result + case Array() => throw new IllegalArgumentException( + s"$reference does not exist. Available: ${schema.fieldNames.mkString(", ")}") + case matches => throw new IllegalArgumentException( + s"$reference is ambiguous. Matches: ${matches.map(_._1.name).mkString(", ")}") + } + } +} class EnsembleByKey(val uid: String) extends Transformer with Wrappable with DefaultParamsWritable with SynapseMLLogging { + + import EnsembleByKey._ + logClass(FeatureNames.Core) + override protected lazy val pyInternalWrapper = true def this() = this(Identifiable.randomUID("EnsembleByKey")) @@ -47,7 +113,7 @@ class EnsembleByKey(val uid: String) extends Transformer val colNames = new StringArrayParam(this, "colNames", "Names of the result of each col") - def getColNames: Array[String] = $(colNames) + def getColNames: Array[String] = get(colNames).getOrElse(getCols.map(name => s"$getStrategy($name)")) def setColNames(arr: Array[String]): this.type = set(colNames, arr) @@ -83,73 +149,645 @@ class EnsembleByKey(val uid: String) extends Transformer setDefault(collapseGroup -> true) - override def transform(dataset: Dataset[_]): DataFrame = { - logTransform[DataFrame]({ + private val aggregateType: DataType => Option[DataType] = { + case _: DoubleType => Some(DoubleType) + case _: FloatType => Some(DoubleType) + case fdt if fdt == VectorType => Some(VectorType) + case _ => None + } - if (get(colNames).isEmpty) { - setDefault(colNames -> getCols.map(name => s"$getStrategy($name)")) - } + private val aggregateField = (outputName: String, dataType: DataType) => + StructField(outputName, dataType, nullable = dataType != VectorType) - transformSchema(dataset.schema) + private val keyRole = FieldRole( + consumesInputColumn = true, + field => Some(field.copy(name = ""))) + + private val aggregateRole = FieldRole( + consumesInputColumn = false, + field => aggregateType(field.dataType).map(aggregateField("", _))) + + private val topLevelMatches = (schema: StructType, fieldName: String, caseSensitive: Boolean) => + schema.fields.zipWithIndex.collect { + case (field, ordinal) if columnNamesMatch(field.name, fieldName, caseSensitive) => ordinal + } - val strategyToFloatFunction = Map( - "mean" -> { (x: String, y: String) => mean(x).alias(y) } - ) + private val analyzedAttributes = (dataset: Option[Dataset[_]]) => + dataset.toSeq.flatMap(_.queryExecution.analyzed.output) - val strategyToVectorFunction = Map( - "mean" -> { (x: String, y: String) => - Summarizer.mean(col(x)).alias(y) + private def pruneDuplicates[A](candidates: Seq[A])(metadataOf: A => Metadata): Seq[A] = { + if (candidates.length <= 1) { + candidates + } else { + val pruned = candidates.filterNot(metadataOf(_).contains(DuplicateMetadataKey)) + if (pruned.isEmpty) candidates else pruned + } + } + + private val withoutDuplicateMarker = (metadata: Metadata) => + if (metadata.contains(DuplicateMetadataKey)) { + new MetadataBuilder().withMetadata(metadata).remove(DuplicateMetadataKey).build() + } else { + metadata + } + + private val declaredField = (field: StructField, name: String) => + field.copy(name = name, metadata = withoutDuplicateMarker(field.metadata)) + + private val shareOneExpression = (attributes: Seq[Attribute], ordinals: Array[Int]) => + ordinals.length > 1 && ordinals.forall(_ < attributes.length) && + ordinals.map(attributes(_).exprId).distinct.length == 1 + + private def qualifiedPathMatches( + attributes: Seq[Attribute], + parsedPath: Array[String], + caseSensitive: Boolean + ): Seq[QualifiedMatch] = { + attributes.zipWithIndex + .flatMap { case (attribute, ordinal) => + (1 until parsedPath.length).collect { + case index + if columnNamesMatch(attribute.name, parsedPath(index), caseSensitive) && + qualifiersMatch(attribute.qualifier, parsedPath.take(index), caseSensitive) => + QualifiedMatch(parsedPath.take(index), parsedPath.drop(index), ordinal, attribute.exprId) } - ) - - val newCols = getCols.zip(getColNames).map { case (inColName, outColName) => - dataset.schema(inColName).dataType match { - case _: DoubleType => - strategyToFloatFunction(getStrategy)(inColName, outColName) - case _: FloatType => - strategyToFloatFunction(getStrategy)(inColName, outColName) - case v if v == VectorType => - strategyToVectorFunction(getStrategy)(inColName, outColName) - case t => - throw new IllegalArgumentException(s"Cannot operate on type $t with strategy $getStrategy") + } + } + + private def qualifiedMatch( + parsedPath: Array[String], + reference: String, + caseSensitive: Boolean, + dataset: Option[Dataset[_]] + ): Option[QualifiedMatch] = { + val attributes = analyzedAttributes(dataset) + val allMatches = qualifiedPathMatches(attributes, parsedPath, caseSensitive) + if (allMatches.isEmpty) { + None + } else { + // Spark selects the qualifier/name candidate set first and only then prunes duplicate-marked + // candidates, so pruning must never change which qualifier length wins. + val longestQualifier = allMatches.map(_.qualifier.length).max + val selected = allMatches.filter(_.qualifier.length == longestQualifier) + val matches = pruneDuplicates(selected)(candidate => attributes(candidate.ordinal).metadata) + require( + matches.map(_.exprId).distinct.length == 1, + s"$reference is ambiguous because it matches multiple dataset attributes") + Some(matches.head) + } + } + + private val schemaSplit = (schema: StructType, parsedPath: Array[String], caseSensitive: Boolean) => + parsedPath.indices.filter(index => + schema.fields.exists(field => + columnNamesMatch(field.name, parsedPath(index), caseSensitive))) + + private val qualifiersMatch = (actual: Seq[String], configured: Array[String], caseSensitive: Boolean) => + actual.length >= configured.length && + actual.takeRight(configured.length).zip(configured) + .forall { case (left, right) => columnNamesMatch(left, right, caseSensitive) } + + private def bindQualifier( + dataset: Dataset[_], + resolved: ResolvedField, + caseSensitive: Boolean + ): ResolvedField = { + if (resolved.qualifier.isEmpty) { + resolved + } else { + val candidates = dataset.queryExecution.analyzed.output.zipWithIndex.filter { case (attribute, _) => + columnNamesMatch(attribute.name, resolved.path.head.name, caseSensitive) && + qualifiersMatch(attribute.qualifier, resolved.qualifier, caseSensitive) + } + val matches = pruneDuplicates(candidates)(_._1.metadata) + matches match { + case Seq() => + throw new IllegalArgumentException(s"${resolved.reference} does not match a dataset qualifier") + case _ if matches.map(_._1.exprId).distinct.length == 1 => + resolved.copy(ordinals = resolved.ordinals.updated(0, matches.head._2)) + case _ => + throw new IllegalArgumentException(s"${resolved.reference} is ambiguous") + } + } + } + + // Spark's GetMapValue casts the requested literal to the map key type and additionally requires + // that key type to be orderable (TypeUtils.checkForOrderingExpr -> RowOrdering.isOrderable). + // RowOrdering.isOrderable(DataType) is identical in Spark 3.5 and Spark 4.1, so it is safe here. + private val mapKeyIsExtractable = (keyType: DataType) => + Cast.canCast(StringType, keyType) && RowOrdering.isOrderable(keyType) + + private val unsupportedMapKeyMessage = (reference: String, keyType: DataType) => + s"$reference cannot be extracted because map key type $keyType " + ( + if (Cast.canCast(StringType, keyType)) { + "is not orderable, so Spark cannot look up a map value by key. " + + "Use a map column whose key type is orderable, such as string." + } else "does not accept string keys") + + private def resolveStep( + currentType: DataType, + fieldName: String, + currentNullable: Boolean, + reference: String, + caseSensitive: Boolean + ): ResolvedStep = { + currentType match { + case currentSchema: StructType => + val (field, fieldOrdinal) = resolveFieldAtLevel( + currentSchema, + fieldName, + reference, + caseSensitive) + ResolvedStep( + field.name, + field.dataType, + currentNullable || field.nullable, + field.metadata, + fieldOrdinal, + None) + case ArrayType(elementSchema: StructType, containsNull) => + val (field, fieldOrdinal) = + resolveFieldAtLevel(elementSchema, fieldName, reference, caseSensitive) + ResolvedStep( + field.name, + ArrayType(field.dataType, containsNull || field.nullable), + currentNullable, + Metadata.empty, + fieldOrdinal, + None) + case MapType(keyType, valueType, _) if mapKeyIsExtractable(keyType) => + ResolvedStep(fieldName, valueType, nullable = true, Metadata.empty, -1, Some(keyType)) + case MapType(keyType, _, _) => + throw new IllegalArgumentException(unsupportedMapKeyMessage(reference, keyType)) + case _ => + throw new IllegalArgumentException( + s"$reference is not supported by Spark nested field extraction") + } + } + + private def resolvePath( + currentType: DataType, + remainingPath: List[String], + currentNullable: Boolean, + ordinals: List[Int], + reference: String, + caseSensitive: Boolean + ): (StructField, List[Int], List[PathStep]) = { + val step = resolveStep( + currentType, + remainingPath.head, + currentNullable, + reference, + caseSensitive) + val pathStep = PathStep(step.fieldName, step.mapKeyType) + + remainingPath.tail match { + case Nil => + (StructField(step.fieldName, step.dataType, step.nullable, step.metadata), + ordinals :+ step.ordinal, + List(pathStep)) + case nestedPath => + val (field, fieldOrdinals, fieldSteps) = resolvePath( + step.dataType, + nestedPath, + step.nullable, + ordinals :+ step.ordinal, + reference, + caseSensitive) + (field, fieldOrdinals, pathStep +: fieldSteps) + } + } + + private def candidateOutput( + schema: StructType, + requestedPath: Array[String], + ordinal: Int, + reference: String, + caseSensitive: Boolean, + role: FieldRole + ): Option[Option[StructField]] = { + Try(resolveAtOrdinal(schema, Array.empty[String], requestedPath, ordinal, reference, caseSensitive)) + .toOption + .map(resolved => role.declaredOutput(resolved.field)) + } + + private val candidateOutputsAgree = ( + schema: StructType, + matches: Array[Int], + requestedPath: Array[String], + reference: String, + caseSensitive: Boolean, + role: FieldRole) => + matches + .map(ordinal => candidateOutput(schema, requestedPath, ordinal, reference, caseSensitive, role)) + .distinct + .length <= 1 + + private def requireStableQualifiedField( + schema: StructType, + matches: Array[Int], + requestedPath: Array[String], + reference: String, + caseSensitive: Boolean, + role: FieldRole + ): Unit = { + require( + candidateOutputsAgree(schema, matches, requestedPath, reference, caseSensitive, role), + s"$reference matches columns with incompatible declared outputs") + require( + matches.length <= 1 || requestedPath.length > 1 || + getCollapseGroup || !role.consumesInputColumn, + s"$reference cannot be resolved from schema because multiple columns are named " + + s"${requestedPath.head} when collapseGroup is false") + } + + private def resolveFromSchema( + schema: StructType, + qualifier: Array[String], + requestedPath: Array[String], + reference: String, + caseSensitive: Boolean, + role: FieldRole, + dataset: Option[Dataset[_]] + ): ResolvedField = { + val candidates = topLevelMatches(schema, requestedPath.head, caseSensitive) + if (qualifier.isEmpty) { + resolveUnqualifiedFromSchema(schema, candidates, requestedPath, reference, caseSensitive, role, dataset) + } else if (candidates.isEmpty) { + resolveNestedPath(schema, qualifier, requestedPath, reference, caseSensitive) + } else { + // A schema carries no qualifier metadata, so every ordinal the dataset-aware path could select + // must derive the same output instead of pruning duplicate-marked fields out of the candidates. + requireStableQualifiedField(schema, candidates, requestedPath, reference, caseSensitive, role) + resolveAtOrdinal(schema, qualifier, requestedPath, candidates.head, reference, caseSensitive) + } + } + + private def resolveUnqualifiedFromSchema( + schema: StructType, + candidates: Array[Int], + requestedPath: Array[String], + reference: String, + caseSensitive: Boolean, + role: FieldRole, + dataset: Option[Dataset[_]] + ): ResolvedField = { + val matches = pruneDuplicates(candidates.toSeq)(schema(_).metadata).toArray + val resolvableDuplicates = candidates.length > 1 && + (matches.length == 1 || + ((dataset.isEmpty || shareOneExpression(analyzedAttributes(dataset), matches)) && + Try(requireStableQualifiedField( + schema, matches, requestedPath, reference, caseSensitive, role)).isSuccess)) + if (!resolvableDuplicates) { + resolveNestedPath(schema, Array.empty[String], requestedPath, reference, caseSensitive) + } else { + requireStableQualifiedField(schema, matches, requestedPath, reference, caseSensitive, role) + resolveAtOrdinal(schema, Array.empty[String], requestedPath, matches.head, reference, caseSensitive) + } + } + + private def resolveNestedPath( + schema: StructType, + qualifier: Array[String], + requestedPath: Array[String], + reference: String, + caseSensitive: Boolean + ): ResolvedField = { + val (field, ordinals, steps) = + resolvePath(schema, requestedPath.toList, false, Nil, reference, caseSensitive) + ResolvedField(reference, qualifier, steps.toArray, ordinals.toArray, declaredField(field, requestedPath.last)) + } + + private def resolveFromOrdinal( + schema: StructType, + qualifier: Array[String], + requestedPath: Array[String], + ordinal: Int, + reference: String, + caseSensitive: Boolean, + role: FieldRole + ): ResolvedField = { + val matches = topLevelMatches(schema, requestedPath.head, caseSensitive) + requireStableQualifiedField(schema, matches, requestedPath, reference, caseSensitive, role) + resolveAtOrdinal(schema, qualifier, requestedPath, ordinal, reference, caseSensitive) + } + + private def resolveAtOrdinal( + schema: StructType, + qualifier: Array[String], + requestedPath: Array[String], + ordinal: Int, + reference: String, + caseSensitive: Boolean + ): ResolvedField = { + val topField = schema(ordinal) + val topStep = PathStep(topField.name, None) + val (field, ordinals, steps) = requestedPath.tail.toList match { + case Nil => (topField, List(ordinal), List(topStep)) + case nestedPath => + val (nestedField, nestedOrdinals, nestedSteps) = resolvePath( + topField.dataType, + nestedPath, + topField.nullable, + List(ordinal), + reference, + caseSensitive) + (nestedField, nestedOrdinals, topStep +: nestedSteps) + } + ResolvedField(reference, qualifier, steps.toArray, ordinals.toArray, declaredField(field, requestedPath.last)) + } + + private def outputContribution( + role: FieldRole, + resolved: ResolvedField + ): (Option[StructField], Option[Int]) = { + val consumesOrdinal = + role.consumesInputColumn && !getCollapseGroup && resolved.path.length == 1 + val consumedOrdinal = if (consumesOrdinal) Some(resolved.ordinals.head) else None + (role.declaredOutput(resolved.field), consumedOrdinal) + } + + private def schemaInterpretations( + schema: StructType, + parsedPath: Array[String], + reference: String, + caseSensitive: Boolean, + role: FieldRole, + dataset: Option[Dataset[_]] + ): Seq[ResolvedField] = { + schemaSplit(schema, parsedPath, caseSensitive).flatMap(index => + Try(resolveFromSchema( + schema, + parsedPath.take(index), + parsedPath.drop(index), + reference, + caseSensitive, + role, + dataset)).toOption) + } + + private def resolveField( + schema: StructType, + reference: String, + caseSensitive: Boolean, + dataset: Option[Dataset[_]], + role: FieldRole + ): ResolvedField = { + val parsedPath = UnresolvedAttribute.parseAttributeName(reference).toArray + val interpretations = schemaInterpretations(schema, parsedPath, reference, caseSensitive, role, dataset) + require( + interpretations.map(outputContribution(role, _)).distinct.length <= 1, + s"$reference is ambiguous between a nested field and a dataset qualifier") + + qualifiedMatch(parsedPath, reference, caseSensitive, dataset) match { + case Some(matched) => + resolveFromOrdinal( + schema, + matched.qualifier, + matched.requestedPath, + matched.ordinal, + reference, + caseSensitive, + role) + case None => + interpretations.headOption.getOrElse { + val pathStart = schemaSplit(schema, parsedPath, caseSensitive).headOption.getOrElse(0) + resolveFromSchema( + schema, + parsedPath.take(pathStart), + parsedPath.drop(pathStart), + reference, + caseSensitive, + role, + dataset) } + } + } + + private def validateNonCollapsedKeys( + schema: StructType, + keyFields: Array[ResolvedField], + outputNames: Array[String], + caseSensitive: Boolean + ): Unit = { + val keyOutputCollisions = outputNames.filter(outputName => + keyFields.exists(resolved => + columnNamesMatch(resolved.field.name, outputName, caseSensitive))).distinct + require( + keyOutputCollisions.isEmpty, + s"Output columns ${keyOutputCollisions.mkString(", ")} cannot overwrite grouping keys " + + s"${keyFields.map(_.field.name).mkString(", ")} when collapseGroup is false") + + val nestedKeyCollisions = keyFields.filter(_.path.length > 1).filter(resolved => + schema.fields.exists(field => + columnNamesMatch(field.name, resolved.field.name, caseSensitive))) + require( + nestedKeyCollisions.isEmpty, + s"Nested grouping keys ${nestedKeyCollisions.map(_.reference).mkString(", ")} " + + "cannot overwrite top-level columns when collapseGroup is false") + + val duplicateNestedKeyNames = keyFields.indices.flatMap { leftIndex => + ((leftIndex + 1) until keyFields.length).collect { + case rightIndex + if columnNamesMatch( + keyFields(leftIndex).field.name, + keyFields(rightIndex).field.name, + caseSensitive) => + keyFields(leftIndex).field.name + } + }.distinct + require( + duplicateNestedKeyNames.isEmpty, + s"Grouping keys must resolve to distinct output columns when collapseGroup is false: " + + duplicateNestedKeyNames.mkString(", ")) + } + + private def getSchemaFields( + schema: StructType, + dataset: Option[Dataset[_]] = None + ): ResolvedColumns = { + val inputNames = get(cols).getOrElse( + throw new IllegalArgumentException("cols must be set and non-empty")) + val keyNames = get(keys).getOrElse( + throw new IllegalArgumentException("keys must be set and non-empty")) + require(inputNames.nonEmpty, "cols must be set and non-empty") + require(keyNames.nonEmpty, "keys must be set and non-empty") + val outputNames = get(colNames).getOrElse( + inputNames.map(name => s"$getStrategy($name)")) + require( + inputNames.length == outputNames.length, + s"cols (${inputNames.length}) and colNames (${outputNames.length}) must have the same length") + + val caseSensitive = dataset.map(_.sparkSession).orElse(SparkSession.getActiveSession) + .exists(_.conf.get("spark.sql.caseSensitive", "false").trim.toBoolean) + val inputFields = inputNames.map(resolveField(schema, _, caseSensitive, dataset, aggregateRole)) + val keyFields = keyNames.map(resolveField(schema, _, caseSensitive, dataset, keyRole)) + keyFields.foreach { key => + require(RowOrdering.isOrderable(key.field.dataType), + s"${key.reference} resolves to ${key.field.dataType}, which Spark cannot use as a grouping key") + } + if (!getCollapseGroup) { + validateNonCollapsedKeys(schema, keyFields, outputNames, caseSensitive) + } + + val aggregateFields = inputFields.zip(outputNames).map { case (resolvedInput, outputName) => + aggregateType(resolvedInput.field.dataType) + .map(aggregateField(outputName, _)) + .getOrElse(throw new IllegalArgumentException( + s"Cannot operate on type ${resolvedInput.field.dataType} with strategy $getStrategy")) + } + + ResolvedColumns(inputFields, outputNames, keyFields, aggregateFields, caseSensitive) + } + + private def bindQualifiers( + dataset: Dataset[_], + resolvedColumns: ResolvedColumns + ): ResolvedColumns = { + resolvedColumns.copy( + inputFields = resolvedColumns.inputFields.map(bindQualifier( + dataset, + _, + resolvedColumns.caseSensitive)), + keyFields = resolvedColumns.keyFields.map(bindQualifier( + dataset, + _, + resolvedColumns.caseSensitive))) + } + + private val quoteIdentifier = (name: String) => s"`${name.replace("`", "``")}`" + + private val inputName = (index: Int) => s"__ensemble_by_key_input_$index" + + private val keyName = (index: Int) => s"__ensemble_by_key_key_$index" + + private val aggregateName = (index: Int) => s"__ensemble_by_key_aggregate_$index" + + private val normalize = (dataset: Dataset[_]) => + dataset.toDF(dataset.schema.indices.map(inputName): _*) + + private def resolvedColumn(resolved: ResolvedField): Column = { + val root = col(quoteIdentifier(inputName(resolved.ordinals.head))) + resolved.path.tail.foldLeft(root) { (column, step) => + step.mapKeyType match { + case Some(keyType) => column(lit(step.name).cast(keyType)) + case None => column.getField(step.name) } + } + } - val aggregated = dataset.toDF() - .groupBy(getKeys.head, getKeys.tail: _*) - .agg(newCols.head, newCols.tail: _*) + // The identity cast prevents grouping analysis from propagating source metadata to the key. + private val keyColumn = (resolved: ResolvedField, index: Int) => + resolvedColumn(resolved).cast(resolved.field.dataType).as(keyName(index), resolved.field.metadata) + + private def aggregateColumn( + resolvedInput: ResolvedField, + outputName: String + ): Column = { + val inputColumn = resolvedColumn(resolvedInput) + aggregateType(resolvedInput.field.dataType) match { + case Some(fdt) if fdt == VectorType => Summarizer.mean(inputColumn).alias(outputName) + case Some(_) => mean(inputColumn).alias(outputName) + case None => throw new IllegalArgumentException( + s"Cannot operate on type ${resolvedInput.field.dataType} with strategy $getStrategy") + } + } + + private def aggregate( + dataset: Dataset[_], + normalized: DataFrame, + resolvedColumns: ResolvedColumns + ): DataFrame = { + val keyColumns = resolvedColumns.keyFields.zipWithIndex.map { case (r, i) => keyColumn(r, i) } + val newColumns = resolvedColumns.inputFields.zipWithIndex.map { case (resolvedInput, index) => + aggregateColumn(resolvedInput, aggregateName(index)) + } + val retainGroupColumns = dataset.sparkSession.conf + .get("spark.sql.retainGroupColumns", "true").trim.toBoolean + val aggregateColumns = if (retainGroupColumns) newColumns else keyColumns ++ newColumns + + normalized + .groupBy(keyColumns: _*) + .agg(aggregateColumns.head, aggregateColumns.tail: _*) + } + + private def outputKeyColumns(resolvedColumns: ResolvedColumns): Array[Column] = { + resolvedColumns.keyFields.zipWithIndex.map { case (resolved, index) => + col(quoteIdentifier(keyName(index))).as(resolved.field.name, resolved.field.metadata) + } + } + + private def outputAggregateColumns(resolvedColumns: ResolvedColumns): Array[Column] = { + resolvedColumns.outputNames.indices.map(index => + col(quoteIdentifier(aggregateName(index))).as(resolvedColumns.outputNames(index))).toArray + } + + private def passthroughColumns( + schema: StructType, + resolvedColumns: ResolvedColumns + ): Array[Column] = { + val topLevelKeyOrdinals = resolvedColumns.keyFields.filter(_.path.length == 1) + .map(_.ordinals.head).toSet + schema.fields.zipWithIndex.collect { + case (field, index) + if !topLevelKeyOrdinals(index) && + !resolvedColumns.outputNames.exists(outputName => + columnNamesMatch(field.name, outputName, resolvedColumns.caseSensitive)) => + col(quoteIdentifier(inputName(index))).as(field.name, field.metadata) + } + } + + private def mergeWithGroups( + normalized: DataFrame, + aggregated: DataFrame, + resolvedColumns: ResolvedColumns, + inputSchema: StructType + ): DataFrame = { + val leftKeys = resolvedColumns.keyFields.zipWithIndex.map { case (r, i) => keyColumn(r, i) } + val left = normalized.select((col("*") +: leftKeys.toSeq): _*) + val conditions = resolvedColumns.keyFields.indices.map(i => left(keyName(i)) <=> aggregated(keyName(i))) + val joined = left.join(aggregated, conditions.reduce(_ && _)).select( + (left.columns.map(left(_)) ++ resolvedColumns.outputNames.indices.map(i => + aggregated(aggregateName(i)))): _*) + val outputColumns = + outputKeyColumns(resolvedColumns) ++ + passthroughColumns(inputSchema, resolvedColumns) ++ + outputAggregateColumns(resolvedColumns) + joined.select(outputColumns: _*) + } + + override def transform(dataset: Dataset[_]): DataFrame = { + logTransform[DataFrame]({ + val resolvedColumns = bindQualifiers(dataset, getSchemaFields(dataset.schema, Some(dataset))) + val normalized = normalize(dataset) + val aggregated = aggregate(dataset, normalized, resolvedColumns) if (getCollapseGroup) { - aggregated + aggregated.select((outputKeyColumns(resolvedColumns) ++ + outputAggregateColumns(resolvedColumns)): _*) } else { - val needToDrop = getColNames.toSet & dataset.columns.toSet - dataset.drop(needToDrop.toList: _*).toDF().join(aggregated, getKeys) + mergeWithGroups(normalized, aggregated, resolvedColumns, dataset.schema) } }, dataset.columns.length) - } def transformSchema(schema: StructType): StructType = { - val colSet = getCols.toSet - val colToNewName = getCols.zip(getColNames).toMap - - val newFields = schema.fields.flatMap { f => - if (!colSet(f.name)) None - else { - val newField = StructField(colToNewName(f.name), f.dataType) - f.dataType match { - case _: DoubleType => Some(newField) - case _: FloatType => Some(newField) - case fdt if fdt == VectorType => Some(newField) - case t => throw new IllegalArgumentException(s"Cannot operate on type $t with strategy $getStrategy") - } + val resolvedColumns = getSchemaFields(schema) + val fields = if (getCollapseGroup) { + resolvedColumns.keyFields.map(_.field) ++ resolvedColumns.aggregateFields + } else { + val topLevelKeyOrdinals = resolvedColumns.keyFields.filter(_.path.length == 1) + .map(_.ordinals.head).toSet + val inputFields = schema.fields.zipWithIndex.collect { + case (field, index) + if !topLevelKeyOrdinals(index) && + !resolvedColumns.outputNames.exists(outputName => + columnNamesMatch(field.name, outputName, resolvedColumns.caseSensitive)) => + field } + resolvedColumns.keyFields.map(_.field) ++ inputFields ++ resolvedColumns.aggregateFields } - val keyFields = schema.fields.filter(f => colSet(f.name)) - val fields = - (if (getCollapseGroup) schema.fields else keyFields).++(newFields) - new StructType(fields) } diff --git a/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/EnsembleByKey.txt b/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/EnsembleByKey.txt index 52d490f4f86..1d5f14fa329 100644 --- a/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/EnsembleByKey.txt +++ b/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/EnsembleByKey.txt @@ -5,3 +5,21 @@ the first row of the column. To avoid materialization you can provide the vector through the ``setVectorDims`` function, which takes a mapping from columns (String) to dimension (Int). You can also choose to squash or keep the original dataset with the ``collapseGroup`` parameter. + +Column references support Spark field syntax, including dataset qualifiers, nested +struct paths, array-of-struct extraction, map extraction (the referenced segment is +cast from a string literal to the map key type, following Spark cast rules, and the +map key type must also be orderable because Spark looks map values up by key), and +backtick-quoted literal field names. Duplicate columns that Spark treats as one +expression resolve like a single column, union columns that Spark marks as duplicates +are pruned the same way Spark prunes them (only within the candidate set the requested +qualifier and name already selected, so a duplicate-marked ``u.group`` still wins over an +untagged ``v.group``), while references that match several distinct attributes are +rejected as ambiguous. Because a ``StructType`` does not retain dataset aliases, +``transformSchema`` cannot reject a qualifier that matches no dataset; ``transform`` +detects and reports that invalid qualifier when the analyzed dataset is available. +Schema-only case resolution similarly uses the active Spark session, while runtime +resolution uses the dataset session. If no matching active session exists and those +sessions use different ``spark.sql.caseSensitive`` values, pipeline schema validation +can differ from runtime resolution; keep the dataset session active while constructing +or validating a pipeline. diff --git a/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/LumpFeatures.scala b/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/LumpFeatures.scala new file mode 100644 index 00000000000..dc6d9042798 --- /dev/null +++ b/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/LumpFeatures.scala @@ -0,0 +1,601 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.stages + +import com.microsoft.azure.synapse.ml.codegen.Wrappable +import com.microsoft.azure.synapse.ml.logging.{FeatureNames, SynapseMLLogging} +import com.microsoft.azure.synapse.ml.param.{StringIntMapParam, StringStringMapParam} +import org.apache.spark.ml.param.{DoubleParam, IntParam, Param, ParamMap, ParamValidators} +import org.apache.spark.ml.util._ +import org.apache.spark.ml.{Estimator, Model} +import org.apache.spark.sql.expressions.Window +import org.apache.spark.sql.functions.{array, asc, coalesce, col, count, desc} +import org.apache.spark.sql.functions.{explode, lit, row_number, struct, sum, typedLit, when} +import org.apache.spark.sql.types._ +import org.apache.spark.sql.{Column, DataFrame, Dataset} +import spray.json.DefaultJsonProtocol._ +import spray.json._ + +import scala.collection.JavaConverters._ +import scala.util.control.NonFatal + +/** Parameters shared by [[LumpFeatures]] and [[LumpFeaturesModel]]. */ +trait LumpFeaturesParams extends Wrappable with DefaultParamsWritable { + + val lumpRules: StringIntMapParam = new StringIntMapParam( + this, "lumpRules", + "Map from column name to the maximum number of most-frequent values (top-K) to retain per column. " + + "This is a cap, not the only criterion: a value is retained only if it also clears minCount and " + + "minFreq. Every value that is not retained is replaced with otherValue.") + + def getLumpRules: Map[String, Int] = get(lumpRules).getOrElse(Map.empty) + + def setLumpRules(value: Map[String, Int]): this.type = set(lumpRules, value) + + def setLumpRules(value: java.util.HashMap[String, Int]): this.type = set(lumpRules, value.asScala.toMap) + + def setLumpRules(value: String): this.type = set(lumpRules, lumpRules.jsonDecode(value)) + + val outputCols: StringStringMapParam = new StringStringMapParam( + this, "outputCols", + "Optional map from a lumpRules column name to the column the lumped values are written to. " + + "Rule columns without an entry are replaced in place, which is the default and discards the " + + "original values. Naming a new destination keeps the raw column intact alongside the lumped one.") + + def getOutputCols: Map[String, String] = get(outputCols).getOrElse(Map.empty) + + def setOutputCols(value: Map[String, String]): this.type = set(outputCols, value) + + def setOutputCols(value: java.util.HashMap[String, String]): this.type = set(outputCols, value.asScala.toMap) + + def setOutputCols(value: String): this.type = set(outputCols, outputCols.jsonDecode(value)) + + val otherValue: Param[String] = new Param[String]( + this, "otherValue", + "The single explicit replacement value used for every non-retained value (and, when handleNull is " + + "'other', for null) across all rule columns. Must be non-null and must not occur in any rule column.") + + def getOtherValue: String = $(otherValue) + + def setOtherValue(value: String): this.type = set(otherValue, value) + + val handleNull: Param[String] = new Param[String]( + this, "handleNull", + "How to treat null inputs in rule columns: 'keep' preserves null, 'other' maps null to otherValue. " + + "Unseen non-null values always map to otherValue.", + ParamValidators.inArray(Array("keep", "other"))) + + def getHandleNull: String = $(handleNull) + + def setHandleNull(value: String): this.type = set(handleNull, value) + + setDefault(otherValue -> "__other__", handleNull -> "keep") + + protected def validateRules(): Unit = { + val rules = getLumpRules + require(rules.nonEmpty, + "lumpRules must be a non-empty map from column name to a positive top-K.") + rules.foreach { case (name, k) => + require(Option(name).exists(_.nonEmpty), + "lumpRules contains an empty or null column name; every rule column name must be non-empty.") + require(k > 0, + s"lumpRules top-K for column '$name' must be a positive integer but was $k.") + } + getOutputCols.foreach { case (name, out) => + require(rules.contains(name), + s"outputCols references column '$name' which has no lumpRules entry " + + s"[${rules.keySet.toSeq.sorted.mkString(", ")}].") + require(Option(out).exists(_.nonEmpty), + s"outputCols destination for column '$name' must be a non-empty column name.") + } + val duplicated = orderedOutputs.map(_._2).groupBy(identity).collect { case (d, ds) if ds.size > 1 => d } + require(duplicated.isEmpty, + s"outputCols maps more than one rule column to destination(s) " + + s"[${duplicated.toSeq.sorted.mkString(", ")}]; every destination must be distinct.") + require(Option(getOtherValue).isDefined, + "otherValue must be non-null.") + } + + /** Every rule column paired with the column its lumped values are written to, in a deterministic + * order so transformSchema and transform always agree on where appended columns land. + */ + protected def orderedOutputs: Seq[(String, String)] = { + val explicit = getOutputCols + getLumpRules.keys.toSeq.sorted.map(name => (name, explicit.getOrElse(name, name))) + } + + /** A lumped column is always a plain string, and its nullability follows handleNull alone so the + * declared contract never depends on how the input schema happened to declare nullability. Input + * metadata is dropped because it describes the pre-lumping value set. + */ + private def lumpedField(name: String, keepNulls: Boolean): StructField = + StructField(name, StringType, nullable = keepNulls, Metadata.empty) + + protected def validateAndTransformSchema(schema: StructType): StructType = { + validateRules() + val outputs = orderedOutputs + outputs.foreach { case (name, out) => + require(schema.fieldNames.contains(name), + s"lumpRules references column '$name' which is not present in the input schema " + + s"[${schema.fieldNames.mkString(", ")}].") + val dt = schema(name).dataType + require(dt == StringType, + s"lumpRules column '$name' must be StringType for LumpFeatures v1 but was ${dt.simpleString}.") + require(out == name || !schema.fieldNames.contains(out), + s"outputCols destination '$out' for rule column '$name' already exists in the input schema; " + + "choose a new name or drop the entry to replace the rule column in place.") + } + val keepNulls = getHandleNull == "keep" + val replacedInPlace = outputs.collect { case (name, out) if name == out => name }.toSet + val retained = schema.fields.map { f => + if (replacedInPlace.contains(f.name)) lumpedField(f.name, keepNulls) else f + } + val appended = outputs.collect { case (name, out) if name != out => lumpedField(out, keepNulls) } + StructType(retained ++ appended) + } + + /** Reference a top-level column by its literal name, backtick-quoting it and doubling any embedded + * backticks so names containing dots, backticks, or reserved words like `count` are never parsed + * as nested paths or interpreted as aggregate/reserved identifiers. + */ + protected def litCol(name: String): Column = col("`" + name.replace("`", "``") + "`") +} + +object LumpFeatures extends DefaultParamsReadable[LumpFeatures] { + + /** One distinct value of one rule column, with its rank inside that column and whether it survived + * the eligibility filters. Collected once per fit. + */ + private[stages] case class RankedValue(column: String, value: String, rank: Int, retained: Boolean) +} + +/** Learns, per configured column, which string values to retain and produces a [[LumpFeaturesModel]] + * that replaces every other value with a single global otherValue. + * + * A value is retained when it clears both eligibility filters (minCount and minFreq) and then falls + * inside its column's top-K from lumpRules. Filtering before capping is the order scikit-learn's + * OneHotEncoder uses for min_frequency and max_categories: the frequency thresholds decide which + * values are worth keeping at all, and top-K only bounds how many survive. Relying on top-K alone + * is unreliable for training, because it is blind to the shape of the distribution - it will lump + * healthy levels in a low-cardinality column, and in a very long-tailed column it can retain values + * that together cover almost none of the rows. + */ +class LumpFeatures(override val uid: String) + extends Estimator[LumpFeaturesModel] with LumpFeaturesParams with SynapseMLLogging { + logClass(FeatureNames.Core) + + def this() = this(Identifiable.randomUID("LumpFeatures")) + + val minCount: IntParam = new IntParam( + this, "minCount", + "Minimum number of times a value must occur in the fitting data to be eligible for retention. " + + "Rarer values are lumped into otherValue. Applied before the lumpRules top-K cap. " + + "The default of 1 disables count-based lumping.", + ParamValidators.gtEq(1)) + + def getMinCount: Int = $(minCount) + + def setMinCount(value: Int): this.type = set(minCount, value) + + val minFreq: DoubleParam = new DoubleParam( + this, "minFreq", + "Minimum share of a column's non-null fitting rows a value must account for to be eligible for " + + "retention, in [0, 1]. Rarer values are lumped into otherValue. Applied before the lumpRules " + + "top-K cap. The default of 0.0 disables frequency-based lumping.", + ParamValidators.inRange(0.0, 1.0)) + + def getMinFreq: Double = $(minFreq) + + def setMinFreq(value: Double): this.type = set(minFreq, value) + + setDefault(minCount -> 1, minFreq -> 0.0) + + override def fit(dataset: Dataset[_]): LumpFeaturesModel = { + logFit({ + transformSchema(dataset.schema) + val rules = getLumpRules + val other = getOtherValue + val ranked = rankValues(dataset.toDF(), rules, other) + + val collided = ranked.filter(_.value == other).map(_.column).distinct.sorted + require(collided.isEmpty, + s"otherValue '$other' collides with an existing value in rule column(s) " + + s"[${collided.mkString(", ")}]. Choose an otherValue that does not occur in the data.") + + val byColumn = ranked.filter(_.retained).groupBy(_.column) + val keptJson = rules.map { case (name, k) => + val top = byColumn.getOrElse(name, Seq.empty).sortBy(_.rank).map(_.value).toList + (name, LumpFeaturesModel.encodeKept(k, top, other)) + } + + val model = new LumpFeaturesModel(uid) + .setLumpRules(rules) + .setOtherValue(other) + .setHandleNull(getHandleNull) + get(outputCols).foreach(model.setOutputCols) + model.setKeptValuesJson(keptJson).setParent(this) + }, dataset.columns.length) + } + + /** Rank every distinct non-null value of every rule column in a single pass. + * + * The rule columns are melted into (column, value) pairs so one aggregation serves all of them. + * That keeps the cost at one shuffle instead of a separate full scan per column and, more + * importantly, guarantees every column is learned from the same materialization of the input. + * Per-column jobs cannot promise that: against a non-deterministic upstream plan (a sample, rand, + * or an unordered limit) each column would be learned from different rows. Values equal to + * otherValue are always returned so the collision check reuses this same aggregation. + */ + private def rankValues(df: DataFrame, rules: Map[String, Int], other: String): Seq[LumpFeatures.RankedValue] = { + val columnAlias = "__lump_column__" + val valueAlias = "__lump_value__" + val countAlias = "__lump_count__" + val totalAlias = "__lump_total__" + val rankAlias = "__lump_rank__" + val retainedAlias = "__lump_retained__" + val pairAlias = "__lump_pair__" + val cols = rules.keys.toSeq.sorted + + val melted = df + .select(explode(array(cols.map(c => struct(lit(c).as(columnAlias), litCol(c).as(valueAlias))): _*)) + .as(pairAlias)) + .select(col(s"$pairAlias.$columnAlias").as(columnAlias), col(s"$pairAlias.$valueAlias").as(valueAlias)) + .where(col(valueAlias).isNotNull) + + val perColumn = Window.partitionBy(col(columnAlias)) + val counted = melted + .groupBy(col(columnAlias), col(valueAlias)) + .agg(count(lit(1)).as(countAlias)) + .withColumn(totalAlias, sum(col(countAlias)).over(perColumn)) + .withColumn(rankAlias, row_number().over(perColumn.orderBy(desc(countAlias), asc(valueAlias)))) + + // Eligibility can only reject a suffix of the count-descending order, so intersecting it with the + // rank cap is exactly "filter by frequency first, then keep at most K". + val topK = cols.foldLeft(lit(0)) { (fallback, c) => + when(col(columnAlias) === lit(c), lit(rules(c))).otherwise(fallback) + } + val eligible = col(countAlias) >= lit(getMinCount) && col(countAlias) >= col(totalAlias) * lit(getMinFreq) + + counted + .withColumn(retainedAlias, col(rankAlias) <= topK && eligible) + .where(col(retainedAlias) || col(valueAlias) === lit(other)) + .select(col(columnAlias), col(valueAlias), col(rankAlias), col(retainedAlias)) + .collect() + .map(r => LumpFeatures.RankedValue(r.getString(0), r.getString(1), r.getInt(2), r.getBoolean(3))) + .toSeq + } + + override def copy(extra: ParamMap): LumpFeatures = defaultCopy(extra) + + override def transformSchema(schema: StructType): StructType = validateAndTransformSchema(schema) +} + +object LumpFeaturesModel extends DefaultParamsReadable[LumpFeaturesModel] { + + private[stages] case class KeptState(topK: Int, values: Seq[String], otherValue: Option[String]) + + override def read: MLReader[LumpFeaturesModel] = { + val delegate = super.read + new MLReader[LumpFeaturesModel] { + override def load(path: String): LumpFeaturesModel = { + delegate.session(sparkSession) + delegate.load(path).prepareLoadedModel() + } + } + } + + /** Encode a column's fitted state, including the reserved fallback that was collision-checked + * against the fitting data. Keeping it inside learned state avoids a separately clearable Param. + */ + private[stages] def encodeKept(topK: Int, values: Seq[String], otherValue: String): String = + encodeKept(KeptState(topK, values, Some(otherValue))) + + private[stages] def encodeKept(state: KeptState): String = { + val base = Map[String, JsValue]( + "topK" -> JsNumber(state.topK), + "values" -> JsArray(state.values.map(JsString(_)).toVector)) + val fields = state.otherValue.fold(base)(value => base + ("otherValue" -> JsString(value))) + JsObject(fields).compactPrint + } + + /** Decode one column's state. otherValue is optional only for loading artifacts written before + * that fit-time invariant was persisted; the reader upgrades those entries before returning. + */ + private[stages] def decodeKept(name: String, json: String): KeptState = { + parseKeptJson(name, json) match { + case JsObject(fields) => + KeptState( + decodeTopK(name, json, fields), + decodeValueList(name, json, fields), + decodeOtherValue(name, json, fields)) + case _ => + throw new IllegalArgumentException( + s"keptValuesJson for column '$name' must be a JSON object with fitted state fields: '$json'.") + } + } + + private def parseKeptJson(name: String, json: String): JsValue = + try json.parseJson + catch { + case NonFatal(e) => + throw new IllegalArgumentException(s"keptValuesJson for column '$name' is not valid JSON: '$json'.", e) + } + + private def decodeTopK(name: String, json: String, fields: Map[String, JsValue]): Int = + fields.get("topK") match { + case Some(JsNumber(n)) if n.isValidInt => n.toInt + case _ => + throw new IllegalArgumentException( + s"keptValuesJson for column '$name' must contain an integer topK field: '$json'.") + } + + private def decodeValueList(name: String, json: String, fields: Map[String, JsValue]): Seq[String] = + fields.get("values") match { + case Some(JsArray(elems)) => elems.map { + case JsString(v) => v + case other => + throw new IllegalArgumentException( + s"keptValuesJson values for column '$name' must be strings but found: ${other.compactPrint}.") + }.toList + case _ => + throw new IllegalArgumentException( + s"keptValuesJson for column '$name' must contain a string-array values field: '$json'.") + } + + private def decodeOtherValue(name: String, json: String, fields: Map[String, JsValue]): Option[String] = + fields.get("otherValue") match { + case None => None + case Some(JsString(value)) => Some(value) + case _ => + throw new IllegalArgumentException( + s"keptValuesJson for column '$name' must contain a string otherValue field when present: '$json'.") + } +} + +/** Model produced by [[LumpFeatures]]. Replaces every non-retained value in each rule column with + * otherValue, leaving the learned values unchanged. By default each rule column is rewritten in + * place; set outputCols to write the lumped values to new columns and keep the raw values. + */ +class LumpFeaturesModel(override val uid: String) + extends Model[LumpFeaturesModel] with LumpFeaturesParams with SynapseMLLogging { + logClass(FeatureNames.Core) + + override protected lazy val pyInternalWrapper: Boolean = true + + def this() = this(Identifiable.randomUID("LumpFeaturesModel")) + + val keptValuesJson: StringStringMapParam = new StringStringMapParam( + this, "keptValuesJson", + "Learned model-only state per column, encoded as a map from column name to a JSON object holding the " + + "fitted top-K, retained values, and fit-time otherValue. Populated by LumpFeatures during fit and " + + "immutable thereafter.") + + private var learnedStateSnapshot: Option[Map[String, String]] = None + private var otherValueWasSetWithLearnedState: Boolean = false + + def getKeptValuesJson: Map[String, String] = get(keptValuesJson).getOrElse(Map.empty) + + private def learnedStateChangeMessage: String = + "Cannot change or clear keptValuesJson on a fitted LumpFeaturesModel; re-fit LumpFeatures to change learned state." + + private def guardLearnedStateChange(value: Map[String, String]): Unit = + learnedStateSnapshot.foreach(expected => require(value == expected, learnedStateChangeMessage)) + + def setKeptValuesJson(value: Map[String, String]): this.type = { + guardLearnedStateChange(value) + set(keptValuesJson, value) + } + + def setKeptValuesJson(value: java.util.HashMap[String, String]): this.type = + setKeptValuesJson(value.asScala.toMap) + + def getKeptValues: Map[String, Seq[String]] = + decodedKept.map { case (name, state) => (name, state.values) } + + /** The learned values per column as one JSON object mapping column name to its retained values. + * Language wrappers cannot reliably read the keptValuesJson param map off the JVM object, so this + * gives Python and R callers a single well-defined way to inspect and audit what the model learned. + */ + def getKeptValuesAsJson: String = + JsObject(getKeptValues.map { case (name, values) => + (name, JsArray(values.map(JsString(_)).toVector): JsValue) + }).compactPrint + + private def decodedKept: Map[String, LumpFeaturesModel.KeptState] = + getKeptValuesJson.map { case (name, json) => (name, LumpFeaturesModel.decodeKept(name, json)) } + + /** The fitted top-K per column recorded at fit time; a fitted model keeps lumpRules equal to this. */ + private def getFittedTopK(kept: Map[String, LumpFeaturesModel.KeptState]): Map[String, Int] = + kept.map { case (name, state) => (name, state.topK) } + + private def guardLumpRulesChange(value: Map[String, Int]): Unit = { + if (isDefined(keptValuesJson)) { + val fitted = getFittedTopK(decodedKept) + require(value == fitted, + s"Cannot change lumpRules on a fitted LumpFeaturesModel. The model was fitted with top-K $fitted " + + s"but $value was requested; re-fit LumpFeatures to change the rules.") + } + } + + override def setLumpRules(value: Map[String, Int]): this.type = { + guardLumpRulesChange(value) + set(lumpRules, value) + } + + override def setLumpRules(value: java.util.HashMap[String, Int]): this.type = + setLumpRules(value.asScala.toMap) + + override def setLumpRules(value: String): this.type = + setLumpRules(lumpRules.jsonDecode(value)) + + private def incompatibleOtherValue(value: String, fitted: String): String = + s"Cannot change otherValue on a fitted LumpFeaturesModel. The model reserved '$fitted' during fit " + + s"but '$value' was requested; re-fit LumpFeatures to change the fallback value." + + private def fittedOtherValue(kept: Map[String, LumpFeaturesModel.KeptState]): Option[String] = { + val present = kept.values.flatMap(_.otherValue).toSet + val missing = kept.collect { case (name, state) if state.otherValue.isEmpty => name }.toSeq.sorted + require(present.isEmpty || missing.isEmpty, + s"Model state is inconsistent: fit-time otherValue is missing for column(s) [${missing.mkString(", ")}].") + require(present.size <= 1, + s"Model state contains inconsistent fit-time otherValue entries [${present.toSeq.sorted.mkString(", ")}].") + present.headOption + } + + private def decodedLearnedStateSnapshot: Option[Map[String, LumpFeaturesModel.KeptState]] = + learnedStateSnapshot.map(_.map { case (name, json) => (name, LumpFeaturesModel.decodeKept(name, json)) }) + + private def guardOtherValueChange(value: String): Unit = + decodedLearnedStateSnapshot.flatMap(fittedOtherValue).foreach { fitted => + require(value == fitted, incompatibleOtherValue(value, fitted)) + } + + override def setOtherValue(value: String): this.type = { + guardOtherValueChange(value) + set(otherValue, value) + } + + /** Spark mutates the ParamMap before this hook. Retain an internal snapshot so learned state can + * be restored after generic set/clear calls, and use its embedded fallback to protect otherValue. + */ + override def onParamChange(param: Param[_]): Unit = { + if (keptValuesJson != null && learnedStateSnapshot != null && param == keptValuesJson) { + handleLearnedStateChange() + } + if (otherValue != null && learnedStateSnapshot != null && param == otherValue) { + handleOtherValueChange() + } + } + + private def handleLearnedStateChange(): Unit = { + val current = get(keptValuesJson) + learnedStateSnapshot match { + case None => + current.foreach { value => + learnedStateSnapshot = Some(value) + otherValueWasSetWithLearnedState = isSet(otherValue) + } + case Some(expected) if current.contains(expected) => + case Some(expected) => + set(keptValuesJson, expected) + throw new IllegalArgumentException(learnedStateChangeMessage) + } + } + + private def handleOtherValueChange(): Unit = { + decodedLearnedStateSnapshot.flatMap(fittedOtherValue).foreach { fitted => + val current = getOtherValue + if (current != fitted && (isSet(otherValue) || otherValueWasSetWithLearnedState)) { + set(otherValue, fitted) + throw new IllegalArgumentException(incompatibleOtherValue(current, fitted)) + } + if (current == fitted && isSet(otherValue)) { + otherValueWasSetWithLearnedState = true + } + } + } + + private def validateModelState(kept: Map[String, LumpFeaturesModel.KeptState]): Unit = { + validateRules() + val rules = getLumpRules + require(rules.keySet == kept.keySet, + s"Model state is inconsistent with lumpRules: rule columns " + + s"[${rules.keySet.toSeq.sorted.mkString(", ")}] do not match learned columns " + + s"[${kept.keySet.toSeq.sorted.mkString(", ")}].") + val other = getOtherValue + fittedOtherValue(kept).foreach { fitted => + require(other == fitted, incompatibleOtherValue(other, fitted)) + } + kept.foreach { case (name, state) => + require(rules(name) == state.topK, + s"lumpRules top-K for column '$name' is ${rules(name)} but the model was fitted with top-K " + + s"${state.topK}. " + + "A fitted LumpFeaturesModel does not allow lumpRules to change after fit; re-fit to change rules.") + require(state.values.size <= state.topK, + s"Model retains ${state.values.size} values for column '$name' which exceeds its fitted top-K " + + s"of ${state.topK}.") + require(!state.values.contains(other), + s"otherValue '$other' collides with a retained value in column '$name'. " + + "Choose an otherValue that is not among the learned values.") + } + } + + private def validatedModelState(): Map[String, LumpFeaturesModel.KeptState] = { + val kept = decodedKept + validateModelState(kept) + kept + } + + protected def validateModelState(): Unit = { + validateModelState(decodedKept) + } + + private def replaceLearnedState(value: Map[String, String]): Unit = { + learnedStateSnapshot = None + otherValueWasSetWithLearnedState = false + set(keptValuesJson, value) + } + + private def upgradeLegacyState( + kept: Map[String, LumpFeaturesModel.KeptState]): Map[String, LumpFeaturesModel.KeptState] = { + if (kept.values.exists(_.otherValue.isEmpty)) { + val upgraded = kept.map { case (name, state) => + (name, if (state.otherValue.isDefined) state else state.copy(otherValue = Some(getOtherValue))) + } + replaceLearnedState(upgraded.map { case (name, state) => (name, LumpFeaturesModel.encodeKept(state)) }) + upgraded + } else { + kept + } + } + + private[stages] def prepareLoadedModel(): LumpFeaturesModel = { + val kept = upgradeLegacyState(decodedKept) + validateModelState(kept) + this + } + + override def transformSchema(schema: StructType): StructType = { + validateModelState() + validateAndTransformSchema(schema) + } + + override def transform(dataset: Dataset[_]): DataFrame = { + logTransform[DataFrame]({ + val state = validatedModelState() + validateAndTransformSchema(dataset.schema) + val other = getOtherValue + val keepNulls = getHandleNull == "keep" + orderedOutputs.foldLeft(dataset.toDF()) { case (acc, (name, out)) => + val values = state.get(name).map(_.values).getOrElse(Seq.empty) + acc.withColumn(out, lumpExpr(name, values, other, keepNulls)) + } + }, dataset.columns.length) + } + + private def lumpExpr(name: String, values: Seq[String], other: String, keepNulls: Boolean): Column = { + val retain = + if (values.isEmpty) lit(other) + else when(litCol(name).isin(values: _*), litCol(name)).otherwise(lit(other)) + // The declared schema takes nullability from handleNull alone, so force the expression to agree + // regardless of how the input column declared itself: a typed null literal stays nullable even + // over a non-nullable input, and coalesce makes the non-null contract hold over a nullable one. + if (keepNulls) when(litCol(name).isNull, typedLit(Option.empty[String])).otherwise(retain) + else coalesce(retain, lit(other)) + } + + private def guardLearnedStateCopy(extra: ParamMap): Unit = { + extra.get(keptValuesJson).foreach { requested => + val fitted = learnedStateSnapshot.getOrElse(getKeptValuesJson) + require(requested == fitted, learnedStateChangeMessage) + } + } + + override def copy(extra: ParamMap): LumpFeaturesModel = { + guardLearnedStateCopy(extra) + val copied = copyValues(new LumpFeaturesModel(uid), extra).setParent(parent) + if (copied.isDefined(copied.keptValuesJson)) copied.prepareLoadedModel() + copied + } +} diff --git a/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/UnicodeNormalize.scala b/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/UnicodeNormalize.scala index 5ef14b37642..6ef6df50c4d 100644 --- a/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/UnicodeNormalize.scala +++ b/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/UnicodeNormalize.scala @@ -32,8 +32,9 @@ class UnicodeNormalize(val uid: String) extends Transformer /** @group setParam */ def setForm(value: String): this.type = { - // check input value - Normalizer.Form.valueOf(getForm) + // Validate the incoming value, not the value already set. Validating getForm let an invalid + // form be stored and only surface later, inside the UDF on the executors. + Normalizer.Form.valueOf(value) set("form", value) } diff --git a/core/src/main/scala/com/microsoft/azure/synapse/ml/train/ComputeModelStatistics.scala b/core/src/main/scala/com/microsoft/azure/synapse/ml/train/ComputeModelStatistics.scala index fa609a6e51d..fbf0a87c0e3 100644 --- a/core/src/main/scala/com/microsoft/azure/synapse/ml/train/ComputeModelStatistics.scala +++ b/core/src/main/scala/com/microsoft/azure/synapse/ml/train/ComputeModelStatistics.scala @@ -6,7 +6,7 @@ package com.microsoft.azure.synapse.ml.train import com.microsoft.azure.synapse.ml.codegen.Wrappable import com.microsoft.azure.synapse.ml.core.contracts._ import com.microsoft.azure.synapse.ml.core.metrics.{MetricConstants, MetricUtils} -import com.microsoft.azure.synapse.ml.core.schema.{CategoricalUtilities, SchemaConstants, SparkSchema} +import com.microsoft.azure.synapse.ml.core.schema.{CategoricalUtilities, SchemaConstants} import com.microsoft.azure.synapse.ml.logging.{FeatureNames, SynapseMLLogging} import org.apache.log4j.Logger import org.apache.spark.ml.Transformer @@ -30,8 +30,9 @@ trait ComputeModelStatisticsParams extends Wrappable with DefaultParamsWritable * The metrics that can be chosen are: * * For binary classification: - * - areaUnderROC + * - areaUnderROC (reported in the AUC column) * - AUC + * - areaUnderPR * - accuracy * - precision * - recall @@ -74,24 +75,29 @@ class ComputeModelStatistics(override val uid: String) extends Transformer //scalastyle:off cyclomatic.complexity override def transform(dataset: Dataset[_]): DataFrame = { logTransform[DataFrame]({ - val (modelName, labelColumnName, scoreValueKind) = - MetricUtils.getSchemaInfo( - dataset.schema, - if (isDefined(labelCol)) Some(getLabelCol) else None, - getEvaluationMetric) + val (modelName, resolvedLabelColumnName, scoreValueKind) = + resolveSchemaInfo(dataset.schema, isCaseSensitive(dataset.sparkSession)) + val labelColumnName = validateColumn( + dataset, resolvedLabelColumnName, "label", "setLabelCol") // For creating the result dataframe in classification or regression case val spark = dataset.sparkSession import spark.implicits._ if (scoreValueKind == SchemaConstants.ClassificationKind) { - + val scoredLabelsColumnName = validateColumn( + dataset, + if (isDefined(scoredLabelsCol)) getScoredLabelsCol + else MetricUtils.getScoreColumnName( + dataset.schema, + modelName, + SchemaConstants.SparkPredictionColumn, + scoreValueKind).orNull, + "classification prediction", + "setScoredLabelsCol") var resultDF: DataFrame = Seq(MetricConstants.ClassificationEvaluationType) .toDF(MetricConstants.EvaluationType) - val scoredLabelsColumnName = - if (isDefined(scoredLabelsCol)) getScoredLabelsCol - else SparkSchema.getSparkPredictionColumnName(dataset.schema, modelName) // Get levels for label column if categorical val levels = CategoricalUtilities.getLevels(dataset.schema, labelColumnName) @@ -108,11 +114,22 @@ class ComputeModelStatistics(override val uid: String) extends Transformer lazy val scoresAndLabels = { val scoresColumnName = - if (isDefined(scoresCol)) getScoresCol - else SparkSchema.getSparkRawPredictionColumnName(dataset.schema, modelName) - if (scoresColumnName == null) predictionAndLabels - else if (levelsExist) getScoresAndLabels(dataset, labelColumnName, scoresColumnName, levelsToIndexMap) - else getScalarScoresAndLabels(dataset, labelColumnName, scoresColumnName) + if (isDefined(scoresCol)) { + Some(validateColumn(dataset, getScoresCol, "classification score", "setScoresCol")) + } else { + MetricUtils.getScoreColumnName( + dataset.schema, + modelName, + SchemaConstants.SparkRawPredictionColumn, + scoreValueKind) + .map(columnName => validateColumn(dataset, columnName, "classification score", "setScoresCol")) + } + scoresColumnName match { + case Some(columnName) if levelsExist => + getScoresAndLabels(dataset, labelColumnName, columnName, levelsToIndexMap) + case Some(columnName) => getScalarScoresAndLabels(dataset, labelColumnName, columnName) + case None => predictionAndLabels + } } lazy val (labels: Array[Double], confusionMatrix: Matrix) = createConfusionMatrix(predictionAndLabels) @@ -129,24 +146,39 @@ class ComputeModelStatistics(override val uid: String) extends Transformer simpleMetric == MetricConstants.PrecisionSparkMetric || simpleMetric == MetricConstants.RecallSparkMetric => resultDF = addSimpleMetric(simpleMetric, predictionAndLabels, resultDF) - case MetricConstants.AucSparkMetric => + case MetricConstants.AucSparkMetric | MetricConstants.AreaUnderROCMetric => val numLevels = if (levelsExist) levels.get.length else confusionMatrix.numRows if (numLevels <= 2) { - // Add the AUC - val auc: Double = getAUC(modelName, dataset, labelColumnName, scoresAndLabels) + val auc = getAUC(modelName, dataset, labelColumnName, scoresAndLabels) resultDF = resultDF.withColumn(MetricConstants.AucColumnName, lit(auc)) } else { throw new Exception("Error: AUC is not available for multiclass case") } + case MetricConstants.AreaUnderPRMetric => + val numLevels = if (levelsExist) levels.get.length + else confusionMatrix.numRows + if (numLevels <= 2) { + val areaUnderPR = getAreaUnderPR(scoresAndLabels) + resultDF = resultDF.withColumn(MetricConstants.AreaUnderPRColumnName, lit(areaUnderPR)) + } else { + throw new Exception("Error: areaUnderPR is not available for multiclass case") + } case default => throw new Exception(s"Error: $default is not a classification metric") } resultDF } else if (scoreValueKind == SchemaConstants.RegressionKind) { - val scoresColumnName = + val scoresColumnName = validateColumn( + dataset, if (isDefined(scoresCol)) getScoresCol - else SparkSchema.getSparkPredictionColumnName(dataset.schema, modelName) + else MetricUtils.getScoreColumnName( + dataset.schema, + modelName, + SchemaConstants.SparkPredictionColumn, + scoreValueKind).orNull, + "regression prediction/score", + "setScoresCol") val scoresAndLabels = selectAndCastToRDD(dataset, scoresColumnName, labelColumnName) @@ -172,6 +204,70 @@ class ComputeModelStatistics(override val uid: String) extends Transformer //scalastyle:on method.length //scalastyle:on cyclomatic.complexity + private def validateColumn(dataset: Dataset[_], + columnName: String, + columnRole: String, + setterName: String): String = { + val requestedColumn = Option(columnName).filter(_.nonEmpty) + val columns = dataset.columns + val caseSensitive = dataset.sparkSession.conf.get("spark.sql.caseSensitive", "false").toBoolean + val matchingColumns = requestedColumn.toSeq.flatMap { column => + if (caseSensitive) columns.filter(_ == column) + else columns.filter(_.equalsIgnoreCase(column)) + } + if (matchingColumns.size > 1) { + throw new IllegalArgumentException( + s"Unable to resolve $columnRole column '${requestedColumn.get}' unambiguously. " + + s"Matching columns: ${matchingColumns.sorted.mkString("[", ", ", "]")}") + } else if (matchingColumns.isEmpty) { + val requestedDescription = requestedColumn.map(name => s"'$name'").getOrElse("") + val availableColumns = columns.sorted.mkString("[", ", ", "]") + throw new IllegalArgumentException( + s"Unable to resolve $columnRole column $requestedDescription. " + + s"Call $setterName(...) with an existing column. Available columns: $availableColumns") + } + matchingColumns.head + } + + private def resolveSchemaInfo(schema: StructType, + caseSensitive: Boolean): (String, String, String) = { + if (canEvaluateWithoutMetadata) { + val scoreValueKind = + if (MetricUtils.isClassificationMetric(getEvaluationMetric)) SchemaConstants.ClassificationKind + else SchemaConstants.RegressionKind + val resolvedLabelCol = + MetricUtils.resolveLabelColumn(schema, Some(getLabelCol), caseSensitive).get + ("custom model", resolvedLabelCol, scoreValueKind) + } else { + MetricUtils.getSchemaInfo( + schema, + if (isDefined(labelCol)) Some(getLabelCol) else None, + getEvaluationMetric, + caseSensitive) + } + } + + private def isCaseSensitive(sparkSession: SparkSession): Boolean = + sparkSession.conf.get("spark.sql.caseSensitive", "false").toBoolean + + private def canEvaluateWithoutMetadata: Boolean = { + if (!isDefined(labelCol) || getEvaluationMetric == MetricConstants.AllSparkMetrics) { + false + } else if (MetricUtils.isClassificationMetric(getEvaluationMetric)) { + isDefined(scoredLabelsCol) && + (!classificationMetricRequiresScores || isDefined(scoresCol)) + } else { + isDefined(scoresCol) + } + } + + private def classificationMetricRequiresScores: Boolean = { + getEvaluationMetric == MetricConstants.ClassificationMetricsName || + getEvaluationMetric == MetricConstants.AucSparkMetric || + getEvaluationMetric == MetricConstants.AreaUnderROCMetric || + getEvaluationMetric == MetricConstants.AreaUnderPRMetric + } + private def addSimpleMetric(simpleMetric: String, predictionAndLabels: RDD[(Double, Double)], resultDF: DataFrame): DataFrame = { @@ -220,15 +316,15 @@ class ComputeModelStatistics(override val uid: String) extends Transformer val (accuracy: Double, precision: Double, recall: Double) = getBinaryAccuracyPrecisionRecall(confusionMatrix) metricsLogger.logClassificationMetrics(accuracy, precision, recall) - // Add the AUC - val auc: Double = getAUC(modelName, dataset, labelColumnName, scoresAndLabels) - metricsLogger.logAUC(auc) + val (auc, areaUnderPR) = + getBinaryMetrics(modelName, dataset, labelColumnName, scoresAndLabels) // Add the metrics to the DF resultDF .withColumn(MetricConstants.AccuracyColumnName, lit(accuracy)) .withColumn(MetricConstants.PrecisionColumnName, lit(precision)) .withColumn(MetricConstants.RecallColumnName, lit(recall)) .withColumn(MetricConstants.AucColumnName, lit(auc)) + .withColumn(MetricConstants.AreaUnderPRColumnName, lit(areaUnderPR)) } else { val (microAvgAccuracy: Double, microAvgPrecision: Double, @@ -381,13 +477,28 @@ class ComputeModelStatistics(override val uid: String) extends Transformer (microAvgAccuracy, microAvgPrecision, microAvgRecall, averageAccuracy, macroAveragedPrecision, macroAveragedRecall) } + private def getBinaryMetrics(modelName: String, + dataset: Dataset[_], + labelColumnName: String, + scoresAndLabels: RDD[(Double, Double)]): (Double, Double) = { + withBinaryClassificationMetrics(scoresAndLabels) { binaryMetrics => + (getAUC(modelName, dataset, labelColumnName, binaryMetrics), getAreaUnderPR(binaryMetrics)) + } + } + private def getAUC(modelName: String, - dataset: Dataset[_], - labelColumnName: String, - scoresAndLabels: RDD[(Double, Double)]): Double = { - val binaryMetrics = new BinaryClassificationMetrics(scoresAndLabels, - MetricConstants.BinningThreshold) + dataset: Dataset[_], + labelColumnName: String, + scoresAndLabels: RDD[(Double, Double)]): Double = { + withBinaryClassificationMetrics(scoresAndLabels) { binaryMetrics => + getAUC(modelName, dataset, labelColumnName, binaryMetrics) + } + } + private def getAUC(modelName: String, + dataset: Dataset[_], + labelColumnName: String, + binaryMetrics: BinaryClassificationMetrics): Double = { val spark = dataset.sparkSession import spark.implicits._ @@ -399,6 +510,28 @@ class ComputeModelStatistics(override val uid: String) extends Transformer auc } + private def getAreaUnderPR(scoresAndLabels: RDD[(Double, Double)]): Double = { + withBinaryClassificationMetrics(scoresAndLabels) { binaryMetrics => + getAreaUnderPR(binaryMetrics) + } + } + + private def getAreaUnderPR(binaryMetrics: BinaryClassificationMetrics): Double = { + val areaUnderPR = binaryMetrics.areaUnderPR() + metricsLogger.logAreaUnderPR(areaUnderPR) + areaUnderPR + } + + private def withBinaryClassificationMetrics[T](scoresAndLabels: RDD[(Double, Double)]) + (f: BinaryClassificationMetrics => T): T = { + val binaryMetrics = new BinaryClassificationMetrics(scoresAndLabels, MetricConstants.BinningThreshold) + try { + f(binaryMetrics) + } finally { + binaryMetrics.unpersist() + } + } + private def getBinaryAccuracyPrecisionRecall(confusionMatrix: Matrix): (Double, Double, Double) = { val tp: Double = confusionMatrix(1, 1) val fp: Double = confusionMatrix(0, 1) @@ -436,19 +569,24 @@ class ComputeModelStatistics(override val uid: String) extends Transformer (labels, confusionMatrix) } - override def copy(extra: ParamMap): Transformer = new ComputeModelStatistics() + override def copy(extra: ParamMap): Transformer = defaultCopy(extra) override def transformSchema(schema: StructType): StructType = { - val (_, _, scoreValueKind) = - MetricUtils.getSchemaInfo( + val (_, labelColumnName, scoreValueKind) = + resolveSchemaInfo( schema, - if (isDefined(labelCol)) Some(getLabelCol) else None, - getEvaluationMetric) - val columns = - if (scoreValueKind == SchemaConstants.ClassificationKind) MetricConstants.ClassificationColumns - else if (scoreValueKind == SchemaConstants.RegressionKind) MetricConstants.RegressionColumns + SparkSession.getActiveSession + .orElse(SparkSession.getDefaultSession) + .exists(isCaseSensitive)) + val labelLevels = getLabelLevels(schema, labelColumnName) + val (columns, validMetrics) = + if (scoreValueKind == SchemaConstants.ClassificationKind) + (getClassificationColumns(labelLevels), MetricConstants.ClassificationMetrics) + else if (scoreValueKind == SchemaConstants.RegressionKind) + (MetricConstants.RegressionColumns, MetricConstants.RegressionMetrics) else throwOnInvalidScoringKind(scoreValueKind) - getTransformedSchema(columns, scoreValueKind) + validateBinaryOnlyMetricSchema(labelLevels, scoreValueKind) + getTransformedSchema(columns, scoreValueKind, validMetrics) } @@ -456,14 +594,50 @@ class ComputeModelStatistics(override val uid: String) extends Transformer throw new Exception(s"Error: unknown scoring kind $scoreValueKind") } - private def getTransformedSchema(columns: List[String], metricType: String) = { + private def getLabelLevels(schema: StructType, labelColumnName: String): Option[Array[_]] = { + if (schema.fieldNames.contains(labelColumnName)) CategoricalUtilities.getLevels(schema, labelColumnName) + else None + } + + private def getClassificationColumns(labelLevels: Option[Array[_]]): List[String] = { + // Keep the legacy common-metrics schema when cardinality is unknown or multiclass. + if (labelLevels.exists(_.length <= 2)) MetricConstants.BinaryClassificationColumns + else MetricConstants.ClassificationColumns + } + + private def validateBinaryOnlyMetricSchema(labelLevels: Option[Array[_]], + scoreValueKind: String): Unit = { + val isBinaryOnlyClassificationMetric = + getEvaluationMetric == MetricConstants.AucSparkMetric || + getEvaluationMetric == MetricConstants.AreaUnderROCMetric || + getEvaluationMetric == MetricConstants.AreaUnderPRMetric + // Match runtime semantics: only SynapseML categorical metadata establishes label cardinality. + if (scoreValueKind == SchemaConstants.ClassificationKind && + isBinaryOnlyClassificationMetric && + labelLevels.exists(_.length > 2)) { + throw new IllegalArgumentException(getBinaryOnlyMetricMulticlassError(getEvaluationMetric)) + } + } + + private def getBinaryOnlyMetricMulticlassError(metric: String): String = metric match { + case MetricConstants.AucSparkMetric | MetricConstants.AreaUnderROCMetric => + "Error: AUC is not available for multiclass case" + case MetricConstants.AreaUnderPRMetric => + "Error: areaUnderPR is not available for multiclass case" + case _ => + throw new IllegalArgumentException(s"Error: $metric is not a classification metric") + } + + private def getTransformedSchema(columns: List[String], + metricType: String, + validMetrics: Set[String]): StructType = { getEvaluationMetric match { case allMetrics if allMetrics == MetricConstants.AllSparkMetrics || allMetrics == MetricConstants.ClassificationMetricsName || allMetrics == MetricConstants.RegressionMetricsName => StructType(columns.map(StructField(_, DoubleType))) - case metric: String if MetricConstants.MetricToColumnName.contains(metric) && - columns.contains(MetricConstants.MetricToColumnName(metric)) => + case metric: String if validMetrics.contains(metric) && + MetricConstants.MetricToColumnName.contains(metric) => StructType(Array(StructField(MetricConstants.MetricToColumnName(metric), DoubleType))) case default => throw new Exception(s"Error: $default is not a $metricType metric") @@ -502,6 +676,12 @@ class MetricsLogger(uid: String) { logger.info(metrics) } + def logAreaUnderPR(areaUnderPR: Double): Unit = { + val metrics = MetricData.create( + Map(MetricConstants.AreaUnderPRColumnName -> areaUnderPR), "AreaUnderPR Metric", uid) + logger.info(metrics) + } + def logROC(roc: DataFrame): Unit = { val metrics = MetricData.createTable( Map(MetricConstants.TpRateROCLog -> diff --git a/core/src/main/scala/com/microsoft/azure/synapse/ml/train/ComputeModelStatistics.txt b/core/src/main/scala/com/microsoft/azure/synapse/ml/train/ComputeModelStatistics.txt index 4085e92aaef..2c4379022bf 100644 --- a/core/src/main/scala/com/microsoft/azure/synapse/ml/train/ComputeModelStatistics.txt +++ b/core/src/main/scala/com/microsoft/azure/synapse/ml/train/ComputeModelStatistics.txt @@ -5,15 +5,20 @@ The possible metrics are: Binary Classifiers: -- "AreaUnderROC" +- "areaUnderROC" (reported in the "AUC" column) - "AUC" +- "areaUnderPR" - "accuracy" +- "precision" - "recall" +- "classification" - "all" -Regression Classifiers: +Regression Models: - "mse" - "rmse" - "r2" +- "mae" +- "regression" - "all" diff --git a/core/src/main/scala/com/microsoft/azure/synapse/ml/train/ComputePerInstanceStatistics.scala b/core/src/main/scala/com/microsoft/azure/synapse/ml/train/ComputePerInstanceStatistics.scala index cceecf77ea4..0d55a1b80c6 100644 --- a/core/src/main/scala/com/microsoft/azure/synapse/ml/train/ComputePerInstanceStatistics.scala +++ b/core/src/main/scala/com/microsoft/azure/synapse/ml/train/ComputePerInstanceStatistics.scala @@ -54,7 +54,8 @@ class ComputePerInstanceStatistics(override val uid: String) extends Transformer MetricUtils.getSchemaInfo( dataset.schema, if (isDefined(labelCol)) Some(getLabelCol) else None, - getEvaluationMetric) + getEvaluationMetric, + dataset.sparkSession.conf.get("spark.sql.caseSensitive", "false").toBoolean) val dataframe = dataset.toDF() @@ -71,7 +72,7 @@ class ComputePerInstanceStatistics(override val uid: String) extends Transformer if (levels.get.length > 2) levels.get.length else 2 } else { // Otherwise compute unique levels - dataset.select(col(labelColumnName).cast(DoubleType)).rdd.distinct().count().toInt + dataset.select(col(labelColumnName).cast(DoubleType)).distinct().count().toInt } val logLossFunc = udf((scoredLabel: Double, scores: org.apache.spark.ml.linalg.Vector) => diff --git a/core/src/main/scala/org/apache/spark/ml/ComplexParamsSerializer.scala b/core/src/main/scala/org/apache/spark/ml/ComplexParamsSerializer.scala index a0e7f41d0ca..72a8c82ce94 100644 --- a/core/src/main/scala/org/apache/spark/ml/ComplexParamsSerializer.scala +++ b/core/src/main/scala/org/apache/spark/ml/ComplexParamsSerializer.scala @@ -5,7 +5,6 @@ package org.apache.spark.ml import com.microsoft.azure.synapse.ml.core.serialize.ComplexParam import org.apache.hadoop.fs.Path -import org.apache.spark.SparkContext import org.apache.spark.ml.param.{ParamPair, Params} import org.apache.spark.ml.util.DefaultParamsReader.Metadata import org.apache.spark.ml.util._ @@ -37,7 +36,7 @@ private[ml] class ComplexParamsWriter(instance: Params) extends MLWriter { override protected def saveImpl(path: String): Unit = { val complexParamLocs = ComplexParamsWriter.getComplexParamLocations(instance, path) val complexParamJson = ComplexParamsWriter.getComplexMetadata(complexParamLocs) - ComplexParamsWriter.saveMetadata(instance, path, sc, complexParamJson) + ComplexParamsWriter.saveMetadata(instance, path, sparkSession, complexParamJson) ComplexParamsWriter.saveComplexParams(path, complexParamLocs, shouldOverwrite) } } @@ -90,12 +89,12 @@ private[ml] object ComplexParamsWriter { */ def saveMetadata(instance: Params, path: String, - sc: SparkContext, + spark: SparkSession, extraMetadata: Option[JObject] = None, paramMap: Option[JValue] = None): Unit = { val metadataPath = new Path(path, "metadata").toString - val metadataJson = getMetadataToSave(instance, sc, extraMetadata, paramMap) - sc.parallelize(Seq(metadataJson), 1).saveAsTextFile(metadataPath) + val metadataJson = getMetadataToSave(instance, spark, extraMetadata, paramMap) + spark.createDataFrame(Seq(Tuple1(metadataJson))).toDF("value").write.text(metadataPath) } /** Helper for [[saveMetadata()]] which extracts the JSON to save. @@ -104,7 +103,7 @@ private[ml] object ComplexParamsWriter { * @see [[saveMetadata()]] for details on what this includes. */ def getMetadataToSave(instance: Params, - sc: SparkContext, + spark: SparkSession, extraMetadata: Option[JObject] = None, paramMap: Option[JValue] = None): String = { val uid = instance.uid @@ -121,7 +120,7 @@ private[ml] object ComplexParamsWriter { }.toList) val basicMetadata = ("class" -> cls) ~ ("timestamp" -> System.currentTimeMillis()) ~ - ("sparkVersion" -> sc.version) ~ + ("sparkVersion" -> spark.version) ~ ("uid" -> uid) ~ ("paramMap" -> jsonParams) ~ ("defaultParamMap" -> jsonDefaultParams) @@ -147,7 +146,15 @@ private[ml] object ComplexParamsWriter { private[ml] class ComplexParamsReader[T] extends MLReader[T] { override def load(path: String): T = { - val metadata = DefaultParamsReader.loadMetadata(path, sc) + // Mirrors Spark 4.0's DefaultParamsReader.loadMetadata(path, spark, expectedClassName), + // which reads the metadata through the session rather than SparkContext.textFile. Spark 3.5 + // only offers the SparkContext overload, so inline the same three lines here: otherwise + // loading a model still needs a SparkContext and remains unusable on Databricks Unity + // Catalog shared access mode and Spark Connect, which is the whole point of the writer + // change above. + val metadataPath = new Path(path, "metadata").toString + val metadataStr = sparkSession.read.text(metadataPath).first().getString(0) + val metadata = DefaultParamsReader.parseMetadata(metadataStr) val cls = Utils.classForName(metadata.className) val instance = cls.getConstructor(classOf[String]).newInstance(metadata.uid).asInstanceOf[Params] diff --git a/core/src/main/scala/org/apache/spark/ml/Serializer.scala b/core/src/main/scala/org/apache/spark/ml/Serializer.scala index 478b6089dc7..c9296087ffa 100644 --- a/core/src/main/scala/org/apache/spark/ml/Serializer.scala +++ b/core/src/main/scala/org/apache/spark/ml/Serializer.scala @@ -5,8 +5,8 @@ package org.apache.spark.ml import com.microsoft.azure.synapse.ml.core.env.StreamUtilities._ import com.microsoft.azure.synapse.ml.core.utils.ContextObjectInputStream +import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path -import org.apache.spark.SparkContext import org.apache.spark.ml.util.MLWritable import org.apache.spark.sql._ @@ -42,7 +42,7 @@ object Serializer { (if (tpe <:< typeOf[PipelineStage]) new PipelineSerializer() else if (tpe <:< typeOf[Array[PipelineStage]]) new PipelineArraySerializer() else if (tpe <:< typeOf[Dataset[_]]) new DFSerializer(sparkSession) - else new ObjectSerializer(sparkSession.sparkContext)(typeToTypeTag(tpe))) + else new ObjectSerializer(sparkSession)(typeToTypeTag(tpe))) .asInstanceOf[Serializer[T]] } @@ -64,15 +64,25 @@ object Serializer { }.get } + /** Hadoop configuration derived from the session instead of the SparkContext. + * + * `SparkSession.sparkContext` is unavailable under Spark Connect and is explicitly unsupported + * on Databricks Unity Catalog standard access mode, so deriving the configuration from the + * session is what keeps model persistence working there. This mirrors Spark MLlib's own + * `session.sessionState.newHadoopConf()`. It also layers in session-level conf, which + * `sparkContext.hadoopConfiguration` alone does not. + */ + private[ml] def sessionHadoopConf(spark: SparkSession): Configuration = + spark.sessionState.newHadoopConf() + /** Writes the object to the given path. * * @param obj The object to write. * @param outputPath Where to write the object */ - def writeToHDFS[O](sc: SparkContext, obj: O, outputPath: Path, overwrite: Boolean) + def writeToHDFS[O](spark: SparkSession, obj: O, outputPath: Path, overwrite: Boolean) (implicit ttag: TypeTag[O]): Unit = { - val hadoopConf = sc.hadoopConfiguration - using(outputPath.getFileSystem(hadoopConf).create(outputPath, overwrite)) { os => + using(outputPath.getFileSystem(sessionHadoopConf(spark)).create(outputPath, overwrite)) { os => write[O](obj, os)(ttag) }.get } @@ -82,16 +92,18 @@ object Serializer { * @param path The main path for model to load the object from. * @return The loaded object. */ - def readFromHDFS[O](sc: SparkContext, path: Path)(implicit ttag: TypeTag[O]): O = { - val hadoopConf = sc.hadoopConfiguration - using(path.getFileSystem(hadoopConf).open(path)) { in => + def readFromHDFS[O](spark: SparkSession, path: Path)(implicit ttag: TypeTag[O]): O = { + using(path.getFileSystem(sessionHadoopConf(spark)).open(path)) { in => read[O](in)(ttag) }.get } - def makeQualifiedPath(sc: SparkContext, path: String): Path = { + def makeQualifiedPath(spark: SparkSession, path: String): Path = { + makeQualifiedPath(sessionHadoopConf(spark), path) + } + + private def makeQualifiedPath(hadoopConf: Configuration, path: String): Path = { val modelPath = new Path(path) - val hadoopConf = sc.hadoopConfiguration // Note: to get correct working dir, must use root path instead of root + part val fs = modelPath.getFileSystem(hadoopConf) modelPath.makeQualified(fs.getUri, fs.getWorkingDirectory) @@ -99,10 +111,11 @@ object Serializer { } -class ObjectSerializer[O](sc: SparkContext)(implicit ttag: TypeTag[O]) extends Serializer[O] { - def write(obj: O, path: Path, overwrite: Boolean): Unit = Serializer.writeToHDFS(sc, obj, path, overwrite) +class ObjectSerializer[O](spark: SparkSession)(implicit ttag: TypeTag[O]) extends Serializer[O] { + + def write(obj: O, path: Path, overwrite: Boolean): Unit = Serializer.writeToHDFS(spark, obj, path, overwrite) - def read(path: Path): O = Serializer.readFromHDFS(sc, path) + def read(path: Path): O = Serializer.readFromHDFS(spark, path) } class DFSerializer(spark: SparkSession) extends Serializer[DataFrame] { diff --git a/core/src/test/python/synapsemltest/stages/__init__.py b/core/src/test/python/synapsemltest/stages/__init__.py new file mode 100644 index 00000000000..f780f4fea7e --- /dev/null +++ b/core/src/test/python/synapsemltest/stages/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE in project root for information. diff --git a/core/src/test/python/synapsemltest/stages/test_ensemble_by_key.py b/core/src/test/python/synapsemltest/stages/test_ensemble_by_key.py new file mode 100644 index 00000000000..ba470bf9cd7 --- /dev/null +++ b/core/src/test/python/synapsemltest/stages/test_ensemble_by_key.py @@ -0,0 +1,57 @@ +# Copyright (C) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE in project root for information. + +import tempfile +import unittest +from pathlib import Path + +from synapse.ml.core.init_spark import init_spark +from synapse.ml.stages import EnsembleByKey + +spark = init_spark() + + +class EnsembleByKeySpec(unittest.TestCase): + def test_col_names_follow_params_after_transform_and_load(self): + frame = spark.createDataFrame( + [("group", 1.0, 2.0), ("group", 3.0, 4.0)], + ["key", "score", "other"], + ) + with self.assertRaisesRegex(Exception, "keys must be set and non-empty"): + EnsembleByKey(keys=[], cols=["score"]).transform(frame) + with self.assertRaisesRegex(Exception, "cols must be set and non-empty"): + EnsembleByKey(keys=["key"], cols=[]).transform(frame) + + transformer = EnsembleByKey(keys=["key"], cols=["score"]) + + self.assertEqual(transformer.getColNames(), ["mean(score)"]) + self.assertFalse(transformer.isSet(transformer.colNames)) + self.assertFalse(transformer.hasDefault(transformer.colNames)) + transformer.transform(frame).collect() + self.assertFalse(transformer.isSet(transformer.colNames)) + self.assertFalse(transformer.hasDefault(transformer.colNames)) + + transformer.setCols(["score", "other"]) + self.assertEqual(transformer.getColNames(), ["mean(score)", "mean(other)"]) + + with tempfile.TemporaryDirectory() as directory: + model_path = str(Path(directory) / "ensemble-by-key") + transformer.write().save(model_path) + loaded = EnsembleByKey.load(model_path) + + self.assertEqual(loaded.getColNames(), ["mean(score)", "mean(other)"]) + self.assertFalse(loaded.isSet(loaded.colNames)) + self.assertFalse(loaded.hasDefault(loaded.colNames)) + + transformer.setColNames(["average-score", "average-other"]) + with tempfile.TemporaryDirectory() as directory: + model_path = str(Path(directory) / "ensemble-by-key-explicit") + transformer.write().save(model_path) + loaded = EnsembleByKey.load(model_path) + + self.assertEqual(loaded.getColNames(), ["average-score", "average-other"]) + self.assertTrue(loaded.isSet(loaded.colNames)) + + +if __name__ == "__main__": + unittest.main() diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/automl/HyperparamRangeSuite.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/automl/HyperparamRangeSuite.scala new file mode 100644 index 00000000000..3e150dfaf74 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/automl/HyperparamRangeSuite.scala @@ -0,0 +1,99 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.automl + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase + +// scalastyle:off magic.number +/** Covers the bounds and seeding contract shared by every RangeHyperParam. */ +class HyperparamRangeSuite extends TestBase { + + private val draws = 500 + + test("LongRangeHyperParam stays within its range") { + val hp = new LongRangeHyperParam(0L, 100L, seed = 42) + val values = (1 to draws).map(_ => hp.getNext()) + assert(values.forall(v => v >= 0L && v < 100L), s"out of range: ${values.filter(v => v < 0L || v >= 100L)}") + assert(values.toSet.size > 1) + } + + test("LongRangeHyperParam stays within a range too large to fit in an Int") { + val min = -4000000000L + val max = 4000000000L + val hp = new LongRangeHyperParam(min, max, seed = 7) + val values = (1 to draws).map(_ => hp.getNext()) + assert(values.forall(v => v >= min && v < max)) + assert(values.exists(_ < 0L) && values.exists(_ > 0L)) + } + + test("LongRangeHyperParam rejects an inverted range instead of silently returning min") { + val hp = new LongRangeHyperParam(10L, 5L, seed = 1) + assertThrows[IllegalArgumentException](hp.getNext()) + } + + test("LongRangeHyperParam stays in bounds when the span overflows Long") { + // max - min wraps negative here, so the ordinary reduction path cannot be used. + val hp = new LongRangeHyperParam(Long.MinValue + 1, Long.MaxValue, seed = 7) + val draws = (1 to 200).map(_ => hp.getNext()) + assert(draws.forall(v => v >= Long.MinValue + 1 && v < Long.MaxValue)) + assert(draws.distinct.length > 1) + } + + test("LongRangeHyperParam with an empty range returns min") { + val hp = new LongRangeHyperParam(5L, 5L, seed = 42) + assert((1 to 10).forall(_ => hp.getNext() === 5L)) + } + + test("LongRangeHyperParam covers a power-of-two range without bias") { + // Power-of-two bounds take the mask branch; every bucket must still be reachable. + val hp = new LongRangeHyperParam(0L, 8L, seed = 3) + val counts = (1 to 4000).map(_ => hp.getNext()).groupBy(identity).map { case (k, v) => k -> v.size } + assert(counts.keySet === (0L until 8L).toSet) + assert(counts.values.forall(c => c > 300 && c < 700), s"uneven distribution: $counts") + } + + test("LongRangeHyperParam covers a non-power-of-two range without bias") { + // Non-power-of-two bounds take the rejection branch, which must terminate and stay uniform. + val hp = new LongRangeHyperParam(0L, 7L, seed = 3) + val counts = (1 to 4000).map(_ => hp.getNext()).groupBy(identity).map { case (k, v) => k -> v.size } + assert(counts.keySet === (0L until 7L).toSet) + assert(counts.values.forall(c => c > 350 && c < 800), s"uneven distribution: $counts") + } + + test("IntRangeHyperParam stays within its range") { + val hp = new IntRangeHyperParam(5, 15, seed = 42) + val values = (1 to draws).map(_ => hp.getNext()) + assert(values.forall(v => v >= 5 && v < 15)) + } + + test("DoubleRangeHyperParam stays within its range") { + val hp = new DoubleRangeHyperParam(0.0, 1.0, seed = 42) + assert((1 to draws).map(_ => hp.getNext()).forall(v => v >= 0.0 && v < 1.0)) + } + + test("FloatRangeHyperParam stays within its range") { + val hp = new FloatRangeHyperParam(0.0f, 1.0f, seed = 42) + assert((1 to draws).map(_ => hp.getNext()).forall(v => v >= 0.0f && v < 1.0f)) + } + + test("equal seeds reproduce equal sequences and differing seeds diverge") { + def draw(hp: Dist[_]): Seq[Any] = (1 to 20).map(_ => hp.getNext) + + assert(draw(new IntRangeHyperParam(0, 1000, 11)) === draw(new IntRangeHyperParam(0, 1000, 11))) + assert(draw(new LongRangeHyperParam(0L, 1000L, 11)) === draw(new LongRangeHyperParam(0L, 1000L, 11))) + assert(draw(new DoubleRangeHyperParam(0.0, 1.0, 11)) === draw(new DoubleRangeHyperParam(0.0, 1.0, 11))) + // FloatRangeHyperParam delegates to an inner DoubleRangeHyperParam; it must forward its seed. + assert(draw(new FloatRangeHyperParam(0.0f, 1.0f, 11)) === draw(new FloatRangeHyperParam(0.0f, 1.0f, 11))) + assert(draw(new FloatRangeHyperParam(0.0f, 1.0f, 11)) !== draw(new FloatRangeHyperParam(0.0f, 1.0f, 12))) + } + + test("HyperParamUtils.getRangeHyperParam honors the seed for every numeric type") { + Seq[(Any, Any)]((0, 1000), (0L, 1000L), (0.0, 1.0), (0.0f, 1.0f)).foreach { case (min, max) => + val a = HyperParamUtils.getRangeHyperParam(min, max, 99) + val b = HyperParamUtils.getRangeHyperParam(min, max, 99) + assert((1 to 20).map(_ => a.getNext) === (1 to 20).map(_ => b.getNext), s"seed ignored for $min/$max") + } + } +} +// scalastyle:on magic.number diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/automl/VerifyDefaultHyperparams.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/automl/VerifyDefaultHyperparams.scala new file mode 100644 index 00000000000..a60443a09c6 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/automl/VerifyDefaultHyperparams.scala @@ -0,0 +1,205 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.automl + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.spark.ml.classification._ + +// scalastyle:off magic.number +class VerifyDefaultHyperparams extends TestBase { + + test("defaultRange for LogisticRegression returns non-empty array") { + val lr = new LogisticRegression() + val ranges = DefaultHyperparams.defaultRange(lr) + assert(ranges.nonEmpty) + assert(ranges.length === 3) // regParam, elasticNetParam, maxIter + } + + test("defaultRange for LogisticRegression includes expected params") { + val lr = new LogisticRegression() + val ranges = DefaultHyperparams.defaultRange(lr) + val paramNames = ranges.map(_._1.name).toSet + assert(paramNames.contains("regParam")) + assert(paramNames.contains("elasticNetParam")) + assert(paramNames.contains("maxIter")) + } + + test("LogisticRegression default range is non-empty") { + val lr = new LogisticRegression() + val params = DefaultHyperparams.defaultRange(lr) + assert(params.nonEmpty) + val paramNames = params.map(_._1.name).toSet + assert(paramNames.contains("regParam")) + assert(paramNames.contains("elasticNetParam")) + assert(paramNames.contains("maxIter")) + } + + test("defaultRange for DecisionTreeClassifier returns non-empty array") { + val dt = new DecisionTreeClassifier() + val ranges = DefaultHyperparams.defaultRange(dt) + assert(ranges.nonEmpty) + assert(ranges.length === 4) // maxBins, maxDepth, minInfoGain, minInstancesPerNode + } + + test("defaultRange for DecisionTreeClassifier includes expected params") { + val dt = new DecisionTreeClassifier() + val ranges = DefaultHyperparams.defaultRange(dt) + val paramNames = ranges.map(_._1.name).toSet + assert(paramNames.contains("maxBins")) + assert(paramNames.contains("maxDepth")) + assert(paramNames.contains("minInfoGain")) + assert(paramNames.contains("minInstancesPerNode")) + } + + test("DecisionTreeClassifier default range is non-empty") { + val dt = new DecisionTreeClassifier() + val params = DefaultHyperparams.defaultRange(dt) + assert(params.nonEmpty) + val paramNames = params.map(_._1.name).toSet + assert(paramNames.contains("maxBins")) + assert(paramNames.contains("maxDepth")) + } + + test("defaultRange for GBTClassifier returns non-empty array") { + val gbt = new GBTClassifier() + val ranges = DefaultHyperparams.defaultRange(gbt) + assert(ranges.nonEmpty) + assert(ranges.length === 7) + } + + test("defaultRange for GBTClassifier includes expected params") { + val gbt = new GBTClassifier() + val ranges = DefaultHyperparams.defaultRange(gbt) + val paramNames = ranges.map(_._1.name).toSet + assert(paramNames.contains("maxBins")) + assert(paramNames.contains("maxDepth")) + assert(paramNames.contains("minInfoGain")) + assert(paramNames.contains("minInstancesPerNode")) + assert(paramNames.contains("maxIter")) + assert(paramNames.contains("stepSize")) + assert(paramNames.contains("subsamplingRate")) + } + + test("GBTClassifier default range is non-empty") { + val gbt = new GBTClassifier() + val params = DefaultHyperparams.defaultRange(gbt) + assert(params.nonEmpty) + assert(params.length >= 5) + } + + test("defaultRange for RandomForestClassifier returns non-empty array") { + val rf = new RandomForestClassifier() + val ranges = DefaultHyperparams.defaultRange(rf) + assert(ranges.nonEmpty) + assert(ranges.length === 6) + } + + test("defaultRange for RandomForestClassifier includes expected params") { + val rf = new RandomForestClassifier() + val ranges = DefaultHyperparams.defaultRange(rf) + val paramNames = ranges.map(_._1.name).toSet + assert(paramNames.contains("maxBins")) + assert(paramNames.contains("maxDepth")) + assert(paramNames.contains("minInfoGain")) + assert(paramNames.contains("minInstancesPerNode")) + assert(paramNames.contains("numTrees")) + assert(paramNames.contains("subsamplingRate")) + } + + test("RandomForestClassifier default range is non-empty") { + val rf = new RandomForestClassifier() + val params = DefaultHyperparams.defaultRange(rf) + assert(params.nonEmpty) + val paramNames = params.map(_._1.name).toSet + assert(paramNames.contains("numTrees")) + } + + test("defaultRange for MultilayerPerceptronClassifier returns non-empty array") { + val mlp = new MultilayerPerceptronClassifier() + val ranges = DefaultHyperparams.defaultRange(mlp) + assert(ranges.nonEmpty) + assert(ranges.length === 4) // blockSize, maxIter, tol, layers + } + + test("defaultRange for MultilayerPerceptronClassifier includes expected params") { + val mlp = new MultilayerPerceptronClassifier() + val ranges = DefaultHyperparams.defaultRange(mlp) + val paramNames = ranges.map(_._1.name).toSet + assert(paramNames.contains("blockSize")) + assert(paramNames.contains("maxIter")) + assert(paramNames.contains("tol")) + assert(paramNames.contains("layers")) + } + + test("MultilayerPerceptronClassifier default range is non-empty") { + val mlp = new MultilayerPerceptronClassifier() + val params = DefaultHyperparams.defaultRange(mlp) + assert(params.nonEmpty) + val paramNames = params.map(_._1.name).toSet + assert(paramNames.contains("blockSize")) + assert(paramNames.contains("layers")) + } + + test("defaultRange for NaiveBayes returns non-empty array") { + val nb = new NaiveBayes() + val ranges = DefaultHyperparams.defaultRange(nb) + assert(ranges.nonEmpty) + assert(ranges.length === 1) // smoothing + } + + test("defaultRange for NaiveBayes includes smoothing param") { + val nb = new NaiveBayes() + val ranges = DefaultHyperparams.defaultRange(nb) + val paramNames = ranges.map(_._1.name).toSet + assert(paramNames.contains("smoothing")) + } + + test("NaiveBayes default range is non-empty") { + val nb = new NaiveBayes() + val params = DefaultHyperparams.defaultRange(nb) + assert(params.nonEmpty) + val paramNames = params.map(_._1.name).toSet + assert(paramNames.contains("smoothing")) + } + + test("all defaultRange entries are concrete distributions that sample in range") { + val lr = new LogisticRegression() + val ranges = DefaultHyperparams.defaultRange(lr) + assert(ranges.nonEmpty) + ranges.foreach { case (param, dist) => + assert(param != null) + dist match { + case d: IntRangeHyperParam => + val v = d.getNext(); assert(v >= d.min && v < d.max, s"${param.name} sampled $v") + case d: LongRangeHyperParam => + val v = d.getNext(); assert(v >= d.min && v < d.max, s"${param.name} sampled $v") + case d: DoubleRangeHyperParam => + val v = d.getNext(); assert(v >= d.min && v < d.max, s"${param.name} sampled $v") + case d: FloatRangeHyperParam => + val v = d.getNext(); assert(v >= d.min && v < d.max, s"${param.name} sampled $v") + case d: DiscreteHyperParam[_] => + assert(d.getValues.size > 0, s"${param.name} has no discrete values") + case other => + fail(s"${param.name} mapped to an unsupported Dist: ${other.getClass.getName}") + } + } + } + + test("default ranges produce values inside the declared bounds") { + val lr = new LogisticRegression() + val params = DefaultHyperparams.defaultRange(lr) + def dist(name: String): Dist[_] = params.find(_._1.name == name).get._2 + // Bounds come from DefaultHyperparams.defaultRange(LogisticRegression). Asserting only + // non-null would pass even if every distribution returned a constant. + (1 to 50).foreach { _ => + val reg = dist("regParam").getNext.asInstanceOf[Double] + assert(reg >= 0.001 && reg < 1.0) + val elastic = dist("elasticNetParam").getNext.asInstanceOf[Double] + assert(elastic >= 0.001 && elastic < 1.0) + val iters = dist("maxIter").getNext.asInstanceOf[Int] + assert(iters >= 5 && iters < 10) + } + } +} +// scalastyle:on magic.number diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/automl/VerifyEvaluationUtils.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/automl/VerifyEvaluationUtils.scala new file mode 100644 index 00000000000..b3b76847842 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/automl/VerifyEvaluationUtils.scala @@ -0,0 +1,222 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.automl + +import com.microsoft.azure.synapse.ml.core.metrics.MetricConstants +import com.microsoft.azure.synapse.ml.core.schema.SchemaConstants +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.spark.ml.classification.LogisticRegression +import org.apache.spark.ml.feature.Tokenizer +import org.apache.spark.ml.regression.{DecisionTreeRegressor, GBTRegressor, LinearRegression, RandomForestRegressor} + +class VerifyEvaluationUtils extends TestBase { + + test("ModelTypeUnsupportedErr constant has expected value") { + assert(EvaluationUtils.ModelTypeUnsupportedErr === "Model type not supported for evaluation") + } + + test("getMetricWithOperator returns correct metric for regression MSE") { + val (metricName, ordering) = EvaluationUtils.getMetricWithOperator( + SchemaConstants.RegressionKind, + MetricConstants.MseSparkMetric + ) + assert(metricName === MetricConstants.MseColumnName) + // MSE should use lowest (reverse ordering) + assert(ordering.compare(1.0, 2.0) > 0) // 1.0 is "better" than 2.0 for MSE + } + + test("getMetricWithOperator returns correct metric for regression RMSE") { + val (metricName, ordering) = EvaluationUtils.getMetricWithOperator( + SchemaConstants.RegressionKind, + MetricConstants.RmseSparkMetric + ) + assert(metricName === MetricConstants.RmseColumnName) + // RMSE should use lowest + assert(ordering.compare(1.0, 2.0) > 0) + } + + test("getMetricWithOperator returns correct metric for regression R2") { + val (metricName, ordering) = EvaluationUtils.getMetricWithOperator( + SchemaConstants.RegressionKind, + MetricConstants.R2SparkMetric + ) + assert(metricName === MetricConstants.R2ColumnName) + // R2 should use highest + assert(ordering.compare(2.0, 1.0) > 0) // 2.0 is "better" than 1.0 for R2 + } + + test("getMetricWithOperator returns correct metric for regression MAE") { + val (metricName, ordering) = EvaluationUtils.getMetricWithOperator( + SchemaConstants.RegressionKind, + MetricConstants.MaeSparkMetric + ) + assert(metricName === MetricConstants.MaeColumnName) + // MAE should use lowest + assert(ordering.compare(1.0, 2.0) > 0) + } + + test("regression metrics use chooseLowest ordering (except R2)") { + val (_, mseOrd) = EvaluationUtils.getMetricWithOperator( + SchemaConstants.RegressionKind, MetricConstants.MseSparkMetric) + // MSE should prefer lower values + assert(mseOrd.compare(1.0, 2.0) > 0) + + val (_, r2Ord) = EvaluationUtils.getMetricWithOperator( + SchemaConstants.RegressionKind, MetricConstants.R2SparkMetric) + // R2 should prefer higher values + assert(r2Ord.compare(1.0, 2.0) < 0) + } + + test("getMetricWithOperator returns correct metric for classification AUC") { + val (metricName, ordering) = EvaluationUtils.getMetricWithOperator( + SchemaConstants.ClassificationKind, + MetricConstants.AucSparkMetric + ) + assert(metricName === MetricConstants.AucColumnName) + // AUC should use highest + assert(ordering.compare(2.0, 1.0) > 0) + } + + test("getMetricWithOperator treats areaUnderROC as an AUC alias") { + val (metricName, ordering) = EvaluationUtils.getMetricWithOperator( + SchemaConstants.ClassificationKind, + MetricConstants.AreaUnderROCMetric + ) + assert(metricName === MetricConstants.AucColumnName) + assert(ordering.compare(2.0, 1.0) > 0) + } + + test("getMetricWithOperator returns correct metric for classification areaUnderPR") { + val (metricName, ordering) = EvaluationUtils.getMetricWithOperator( + SchemaConstants.ClassificationKind, + MetricConstants.AreaUnderPRMetric + ) + assert(metricName === MetricConstants.AreaUnderPRColumnName) + assert(ordering.compare(2.0, 1.0) > 0) + } + + test("getMetricWithOperator returns correct metric for classification Precision") { + val (metricName, ordering) = EvaluationUtils.getMetricWithOperator( + SchemaConstants.ClassificationKind, + MetricConstants.PrecisionSparkMetric + ) + assert(metricName === MetricConstants.PrecisionColumnName) + // Precision should use highest + assert(ordering.compare(2.0, 1.0) > 0) + } + + test("getMetricWithOperator returns correct metric for classification Recall") { + val (metricName, ordering) = EvaluationUtils.getMetricWithOperator( + SchemaConstants.ClassificationKind, + MetricConstants.RecallSparkMetric + ) + assert(metricName === MetricConstants.RecallColumnName) + // Recall should use highest + assert(ordering.compare(2.0, 1.0) > 0) + } + + test("getMetricWithOperator returns correct metric for classification Accuracy") { + val (metricName, ordering) = EvaluationUtils.getMetricWithOperator( + SchemaConstants.ClassificationKind, + MetricConstants.AccuracySparkMetric + ) + assert(metricName === MetricConstants.AccuracyColumnName) + // Accuracy should use highest + assert(ordering.compare(2.0, 1.0) > 0) + } + + test("getMetricWithOperator returns correct metric for classification accuracy") { + val (name, _) = EvaluationUtils.getMetricWithOperator( + SchemaConstants.ClassificationKind, MetricConstants.AccuracySparkMetric) + assert(name === MetricConstants.AccuracyColumnName) + } + + test("classification metrics use chooseHighest ordering") { + val (_, aucOrd) = EvaluationUtils.getMetricWithOperator( + SchemaConstants.ClassificationKind, MetricConstants.AucSparkMetric) + // AUC should prefer higher values + assert(aucOrd.compare(1.0, 2.0) < 0) + } + + test("getMetricWithOperator throws for unsupported regression metric") { + assertThrows[Exception] { + EvaluationUtils.getMetricWithOperator( + SchemaConstants.RegressionKind, + "unsupported_metric" + ) + } + } + + test("unsupported regression metric throws") { + assertThrows[Exception] { + EvaluationUtils.getMetricWithOperator(SchemaConstants.RegressionKind, "bogus_metric") + } + } + + test("getMetricWithOperator rejects areaUnderPR for regressors") { + assertThrows[Exception] { + EvaluationUtils.getMetricWithOperator( + SchemaConstants.RegressionKind, + MetricConstants.AreaUnderPRMetric + ) + } + } + + test("getMetricWithOperator throws for unsupported classification metric") { + assertThrows[Exception] { + EvaluationUtils.getMetricWithOperator( + SchemaConstants.ClassificationKind, + "unsupported_metric" + ) + } + } + + test("unsupported classification metric throws") { + assertThrows[Exception] { + EvaluationUtils.getMetricWithOperator(SchemaConstants.ClassificationKind, "bogus_metric") + } + } + + test("getMetricWithOperator throws for unsupported model type") { + assertThrows[Exception] { + EvaluationUtils.getMetricWithOperator( + "unsupported_model_type", + MetricConstants.MseSparkMetric + ) + } + } + + test("unsupported model type throws") { + assertThrows[Exception] { + EvaluationUtils.getMetricWithOperator("unsupported_type", MetricConstants.MseSparkMetric) + } + } + + test("getModelType returns ClassificationKind for a Classifier") { + assert(EvaluationUtils.getModelType(new LogisticRegression()) === SchemaConstants.ClassificationKind) + } + + test("getModelType returns RegressionKind for LinearRegression") { + assert(EvaluationUtils.getModelType(new LinearRegression()) === SchemaConstants.RegressionKind) + } + + test("getModelType returns RegressionKind for DecisionTreeRegressor") { + assert(EvaluationUtils.getModelType(new DecisionTreeRegressor()) === SchemaConstants.RegressionKind) + } + + test("getModelType returns RegressionKind for GBTRegressor") { + assert(EvaluationUtils.getModelType(new GBTRegressor()) === SchemaConstants.RegressionKind) + } + + test("getModelType returns RegressionKind for RandomForestRegressor") { + assert(EvaluationUtils.getModelType(new RandomForestRegressor()) === SchemaConstants.RegressionKind) + } + + test("getModelType throws ModelTypeUnsupportedErr for an unsupported stage") { + val caught = intercept[Exception] { + EvaluationUtils.getModelType(new Tokenizer()) + } + assert(caught.getMessage === EvaluationUtils.ModelTypeUnsupportedErr) + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/automl/VerifyFindBestModel.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/automl/VerifyFindBestModel.scala index e080537bde0..784f39e6b8e 100644 --- a/core/src/test/scala/com/microsoft/azure/synapse/ml/automl/VerifyFindBestModel.scala +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/automl/VerifyFindBestModel.scala @@ -46,7 +46,7 @@ class VerifyFindBestModel extends EstimatorFuzzing[FindBestModel]{ bestModel.transform(dataset) } - test("Verify the best model can be saved") { + test("Verify the best model can be saved with AUC") { val dataset: DataFrame = createMockDataset val logisticRegressor = createLR.setLabelCol(mockLabelColumn) val model = logisticRegressor.fit(dataset) @@ -61,6 +61,32 @@ class VerifyFindBestModel extends EstimatorFuzzing[FindBestModel]{ assert(myModelFile.exists()) } + test("Verify the best model can be saved with areaUnderPR") { + val dataset: DataFrame = createMockDataset + val model = createLR.setLabelCol(mockLabelColumn).fit(dataset) + val bestModel = new FindBestModel() + .setModels(Array(model.asInstanceOf[Transformer], model.asInstanceOf[Transformer])) + .setEvaluationMetric(MetricConstants.AreaUnderPRMetric) + .fit(dataset) + + val myModelFile = new File(tmpDir.toFile, "testEvalModelAreaUnderPR") + bestModel.save(myModelFile.toString) + assert(myModelFile.exists()) + } + + test("FindBestModel rejects invalid evaluation metrics") { + val dataset: DataFrame = createMockDataset + val model = createLR.setLabelCol(mockLabelColumn).fit(dataset) + val error = intercept[Exception] { + new FindBestModel() + .setModels(Array(model.asInstanceOf[Transformer])) + .setEvaluationMetric("averagePrecision") + .fit(dataset) + } + + assert(error.getMessage === "Invalid evaluation metric") + } + test("Verify the best model metrics can be retrieved and are valid") { val dataset: DataFrame = createMockDataset val logisticRegressor = createLR.setLabelCol(mockLabelColumn) diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/automl/VerifyHyperparamBuilder.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/automl/VerifyHyperparamBuilder.scala new file mode 100644 index 00000000000..740f8f65d1b --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/automl/VerifyHyperparamBuilder.scala @@ -0,0 +1,254 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.automl + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.spark.ml.param.{DoubleParam, FloatParam, IntParam, LongParam, ParamMap, Params} + +import scala.collection.JavaConverters._ + +class VerifyHyperparamBuilder extends TestBase { + + // Helper class for creating test params + private class TestParams extends Params { + override val uid: String = "test" + val intParam = new IntParam(this, "intParam", "test int param") + val doubleParam = new DoubleParam(this, "doubleParam", "test double param") + val longParam = new LongParam(this, "longParam", "test long param") + val floatParam = new FloatParam(this, "floatParam", "test float param") + override def copy(extra: ParamMap): Params = this + } + + private val testParamsInstance = new TestParams + + test("IntRangeHyperParam generates values within range") { + val hp = new IntRangeHyperParam(5, 15, seed = 42) + val values = (1 to 100).map(_ => hp.getNext()) + assert(values.forall(v => v >= 5 && v < 15)) + assert(values.toSet.size > 1) // not all the same + } + + test("IntRangeHyperParam respects seed for reproducibility") { + val param1 = new IntRangeHyperParam(0, 100, seed = 42) + val param2 = new IntRangeHyperParam(0, 100, seed = 42) + val values1 = (1 to 10).map(_ => param1.getNext()) + val values2 = (1 to 10).map(_ => param2.getNext()) + assert(values1 === values2) + } + + test("DoubleRangeHyperParam generates values within range") { + val param = new DoubleRangeHyperParam(0.0, 1.0, seed = 42) + for (_ <- 1 to 100) { + val value = param.getNext() + assert(value >= 0.0 && value < 1.0) + } + } + + test("DoubleRangeHyperParam respects seed for reproducibility") { + val param1 = new DoubleRangeHyperParam(0.0, 10.0, seed = 42) + val param2 = new DoubleRangeHyperParam(0.0, 10.0, seed = 42) + val values1 = (1 to 10).map(_ => param1.getNext()) + val values2 = (1 to 10).map(_ => param2.getNext()) + assert(values1 === values2) + } + + test("LongRangeHyperParam generates values within range") { + val param = new LongRangeHyperParam(0L, 100L, seed = 42) + for (_ <- 1 to 100) { + val value = param.getNext() + assert(value >= 0L && value < 100L, s"$value escaped [0, 100)") + } + } + + test("LongRangeHyperParam stays in range for a span wider than Long.MaxValue") { + val param = new LongRangeHyperParam(Long.MinValue, Long.MaxValue, seed = 42) + for (_ <- 1 to 100) { + val value = param.getNext() + assert(value >= Long.MinValue && value < Long.MaxValue) + } + } + + test("LongRangeHyperParam respects seed for reproducibility") { + val param1 = new LongRangeHyperParam(0L, 100L, seed = 42) + val param2 = new LongRangeHyperParam(0L, 100L, seed = 42) + val values1 = (1 to 10).map(_ => param1.getNext()) + val values2 = (1 to 10).map(_ => param2.getNext()) + assert(values1 === values2) + } + + test("FloatRangeHyperParam generates values within range") { + val param = new FloatRangeHyperParam(0.0f, 1.0f, seed = 42) + for (_ <- 1 to 100) { + val value = param.getNext() + assert(value >= 0.0f && value < 1.0f) + } + } + + test("DiscreteHyperParam selects from provided values") { + val hp = new DiscreteHyperParam(List("a", "b", "c"), seed = 42) + val values = (1 to 100).map(_ => hp.getNext()) + assert(values.forall(Set("a", "b", "c").contains)) + assert(values.toSet.size > 1) + } + + test("DiscreteHyperParam.getValues returns Java list") { + val values = List(1, 2, 3) + val param = new DiscreteHyperParam(values) + val javaList = param.getValues + assert(javaList.size() === 3) + assert(javaList.get(0) === 1) + assert(javaList.get(1) === 2) + assert(javaList.get(2) === 3) + } + + test("DiscreteHyperParam getValues returns Java list") { + val hp = new DiscreteHyperParam(List(1, 2, 3)) + val javaList = hp.getValues + assert(javaList.asScala.toList === List(1, 2, 3)) + } + + test("HyperparamBuilder builds empty array when no params added") { + val builder = new HyperparamBuilder() + val result = builder.build() + assert(result.isEmpty) + } + + test("HyperparamBuilder empty build returns empty array") { + val hp = new HyperparamBuilder().build() + assert(hp.isEmpty) + } + + test("HyperparamBuilder adds single hyperparam") { + val builder = new HyperparamBuilder() + builder.addHyperparam(testParamsInstance.intParam, new IntRangeHyperParam(1, 10)) + val result = builder.build() + assert(result.length === 1) + assert(result.head._1 === testParamsInstance.intParam) + } + + test("HyperparamBuilder adds multiple hyperparams") { + val builder = new HyperparamBuilder() + .addHyperparam(testParamsInstance.intParam, new IntRangeHyperParam(1, 10)) + .addHyperparam(testParamsInstance.doubleParam, new DoubleRangeHyperParam(0.0, 1.0)) + val result = builder.build() + assert(result.length === 2) + } + + test("HyperparamBuilder supports method chaining") { + val builder = new HyperparamBuilder() + val result = builder + .addHyperparam(testParamsInstance.intParam, new IntRangeHyperParam(1, 10)) + .addHyperparam(testParamsInstance.doubleParam, new DoubleRangeHyperParam(0.0, 1.0)) + .build() + assert(result.length === 2) + } + + test("HyperparamBuilder builds array of param-dist pairs") { + val hp = new HyperparamBuilder() + .addHyperparam(testParamsInstance.intParam, new IntRangeHyperParam(1, 10)) + .addHyperparam(testParamsInstance.doubleParam, new DoubleRangeHyperParam(0.0, 1.0)) + .build() + assert(hp.length === 2) + assert(hp.map(_._1.name).toSet === Set("intParam", "doubleParam")) + } + + test("HyperParamUtils.getRangeHyperParam returns IntRangeHyperParam for Int") { + val result = HyperParamUtils.getRangeHyperParam(1, 10) + assert(result.isInstanceOf[IntRangeHyperParam]) + val intResult = result.asInstanceOf[IntRangeHyperParam] + assert(intResult.min === 1) + assert(intResult.max === 10) + } + + test("HyperParamUtils.getRangeHyperParam matches Int type") { + val hp = HyperParamUtils.getRangeHyperParam(1, 10) + assert(hp.isInstanceOf[IntRangeHyperParam]) + } + + test("HyperParamUtils.getRangeHyperParam returns DoubleRangeHyperParam for Double") { + val result = HyperParamUtils.getRangeHyperParam(0.0, 1.0) + assert(result.isInstanceOf[DoubleRangeHyperParam]) + val doubleResult = result.asInstanceOf[DoubleRangeHyperParam] + assert(doubleResult.min === 0.0) + assert(doubleResult.max === 1.0) + } + + test("HyperParamUtils.getRangeHyperParam matches Double type") { + val hp = HyperParamUtils.getRangeHyperParam(0.0, 1.0) + assert(hp.isInstanceOf[DoubleRangeHyperParam]) + } + + test("HyperParamUtils.getRangeHyperParam returns LongRangeHyperParam for Long") { + val result = HyperParamUtils.getRangeHyperParam(0L, 100L) + assert(result.isInstanceOf[LongRangeHyperParam]) + val longResult = result.asInstanceOf[LongRangeHyperParam] + assert(longResult.min === 0L) + assert(longResult.max === 100L) + } + + test("HyperParamUtils.getRangeHyperParam matches Long type") { + val hp = HyperParamUtils.getRangeHyperParam(0L, 100L) + assert(hp.isInstanceOf[LongRangeHyperParam]) + } + + test("HyperParamUtils.getRangeHyperParam returns FloatRangeHyperParam for Float") { + val result = HyperParamUtils.getRangeHyperParam(0.0f, 1.0f) + assert(result.isInstanceOf[FloatRangeHyperParam]) + val floatResult = result.asInstanceOf[FloatRangeHyperParam] + assert(floatResult.min === 0.0f) + assert(floatResult.max === 1.0f) + } + + test("HyperParamUtils.getRangeHyperParam matches Float type") { + val hp = HyperParamUtils.getRangeHyperParam(0.0f, 1.0f) + assert(hp.isInstanceOf[FloatRangeHyperParam]) + } + + test("HyperParamUtils.getRangeHyperParam throws for unsupported types") { + assertThrows[Exception] { + HyperParamUtils.getRangeHyperParam("a", "b") + } + } + + test("HyperParamUtils.getRangeHyperParam throws on unsupported type") { + assertThrows[Exception] { + HyperParamUtils.getRangeHyperParam("a", "z") + } + } + + test("HyperParamUtils.getDiscreteHyperParam creates DiscreteHyperParam from Java ArrayList") { + val javaList = new java.util.ArrayList[Int]() + javaList.add(1) + javaList.add(2) + javaList.add(3) + val result = HyperParamUtils.getDiscreteHyperParam(javaList) + assert(result.isInstanceOf[DiscreteHyperParam[_]]) + val value = result.getNext() + assert(Seq(1, 2, 3).contains(value)) + } + + test("HyperParamUtils.getDiscreteHyperParam creates from Java ArrayList") { + val javaList = new java.util.ArrayList[String]() + javaList.add("x") + javaList.add("y") + val hp = HyperParamUtils.getDiscreteHyperParam(javaList) + val values = (1 to 50).map(_ => hp.getNext().toString) + assert(values.forall(v => v == "x" || v == "y")) + } + + test("RangeHyperParam stores min, max, and seed") { + val param = new IntRangeHyperParam(5, 15, seed = 123) + assert(param.min === 5) + assert(param.max === 15) + assert(param.seed === 123) + } + + test("seeded RangeHyperParam produces deterministic sequences") { + val hp1 = new IntRangeHyperParam(0, 100, seed = 123) + val hp2 = new IntRangeHyperParam(0, 100, seed = 123) + val seq1 = (1 to 10).map(_ => hp1.getNext()) + val seq2 = (1 to 10).map(_ => hp2.getNext()) + assert(seq1 === seq2) + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/automl/VerifyParamSpace.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/automl/VerifyParamSpace.scala new file mode 100644 index 00000000000..ef29cc3e923 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/automl/VerifyParamSpace.scala @@ -0,0 +1,65 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.automl + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.spark.ml.param.{IntParam, ParamMap, Params} +import org.apache.spark.ml.util.Identifiable + +// scalastyle:off magic.number +class VerifyParamSpace extends TestBase { + + private object TestParams extends Params { + override val uid: String = Identifiable.randomUID("TestParams") // scalastyle:ignore field.name + override def copy(extra: ParamMap): Params = this + val intParam = new IntParam(this, "intParam", "test int param") // scalastyle:ignore field.name + } + + test("GridSpace iterates over all ParamMaps in order") { + val pm1 = ParamMap(TestParams.intParam -> 1) + val pm2 = ParamMap(TestParams.intParam -> 2) + val pm3 = ParamMap(TestParams.intParam -> 3) + val grid = new GridSpace(Array(pm1, pm2, pm3)) + // Asserting only the length would pass even if GridSpace dropped or reordered entries, + // which matters because these values are positional in a tuning sweep. + assert(grid.paramMaps.map(_.get(TestParams.intParam).get).toList === List(1, 2, 3)) + } + + test("GridSpace with empty array produces empty iterator") { + val grid = new GridSpace(Array.empty[ParamMap]) + assert(!grid.paramMaps.hasNext) + } + + test("RandomSpace produces infinite iterator") { + val builder = new HyperparamBuilder() + .addHyperparam(TestParams.intParam, new IntRangeHyperParam(1, 100)) + val space = new RandomSpace(builder.build()) + val values = space.paramMaps.take(50).toList + assert(values.length === 50) + values.foreach { pm => + val v = pm.get(TestParams.intParam) + assert(v.isDefined) + assert(v.get >= 1 && v.get < 100) + } + } + + test("RandomSpace iterator always hasNext") { + val builder = new HyperparamBuilder() + .addHyperparam(TestParams.intParam, new IntRangeHyperParam(0, 10)) + val space = new RandomSpace(builder.build()) + assert(space.paramMaps.hasNext) + space.paramMaps.next() + assert(space.paramMaps.hasNext) + } + + test("Dist.getParamPair creates correct ParamPair") { + val dist = new IntRangeHyperParam(5, 15, seed = 42) + val pp = dist.getParamPair(TestParams.intParam) + assert(pp.param.name === TestParams.intParam.name) + val value = pp.value.asInstanceOf[Int] + assert(value >= 5) + assert(value < 15) + } +} +// scalastyle:on magic.number diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/causal/VerifyCacheOps.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/causal/VerifyCacheOps.scala new file mode 100644 index 00000000000..cb132c26ab5 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/causal/VerifyCacheOps.scala @@ -0,0 +1,61 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.causal + +import breeze.linalg.{DenseVector => BDV} +import com.microsoft.azure.synapse.ml.core.test.base.TestBase + +class VerifyCacheOps extends TestBase { + + test("CacheOps trait has default implementations that return input unchanged") { + val ops = new CacheOps[String] {} + val data = "test" + assert(ops.cache(data) === data) + assert(ops.checkpoint(data) === data) + } + + test("BDVCacheOps.cache returns the same vector") { + val vector = BDV(1.0, 2.0, 3.0) + val result = BDVCacheOps.cache(vector) + assert(result eq vector) + } + + test("BDVCacheOps.checkpoint returns the same vector") { + val vector = BDV(1.0, 2.0, 3.0) + val result = BDVCacheOps.checkpoint(vector) + assert(result eq vector) + } + + test("BDVCacheOps is a no-op for dense vectors") { + val vector = BDV(1.0, 2.0, 3.0, 4.0, 5.0) + + // Both operations should return the exact same instance + val cached = BDVCacheOps.cache(vector) + val checkpointed = BDVCacheOps.checkpoint(vector) + + assert(cached eq vector) + assert(checkpointed eq vector) + assert(cached.toArray === Array(1.0, 2.0, 3.0, 4.0, 5.0)) + } + + test("BDVCacheOps preserves vector data") { + val vector = BDV(10.0, 20.0, 30.0) + val cached = BDVCacheOps.cache(vector) + + assert(cached(0) === 10.0) + assert(cached(1) === 20.0) + assert(cached(2) === 30.0) + assert(cached.length === 3) + } + + test("CacheOps works with generic types") { + case class TestData(value: Int) + + val ops = new CacheOps[TestData] {} + val data = TestData(42) + + assert(ops.cache(data) === data) + assert(ops.checkpoint(data) === data) + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/causal/VerifySharedParams.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/causal/VerifySharedParams.scala new file mode 100644 index 00000000000..f100be4c4d8 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/causal/VerifySharedParams.scala @@ -0,0 +1,104 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.causal + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.spark.ml.param.Params +import org.apache.spark.ml.util.Identifiable + +class VerifySharedParams extends TestBase { + + // Test implementation that mixes in all the traits + private class TestParamsImpl(override val uid: String) + extends Params + with HasTreatmentCol + with HasOutcomeCol + with HasPostTreatmentCol + with HasUnitCol + with HasTimeCol { + override def copy(extra: org.apache.spark.ml.param.ParamMap): Params = this + } + + test("HasTreatmentCol sets and gets treatment column") { + val params = new TestParamsImpl(Identifiable.randomUID("test")) + params.setTreatmentCol("treatment") + assert(params.getTreatmentCol === "treatment") + } + + test("HasTreatmentCol param has correct name and doc") { + val params = new TestParamsImpl(Identifiable.randomUID("test")) + assert(params.treatmentCol.name === "treatmentCol") + assert(params.treatmentCol.doc === "treatment column") + } + + test("HasOutcomeCol sets and gets outcome column") { + val params = new TestParamsImpl(Identifiable.randomUID("test")) + params.setOutcomeCol("outcome") + assert(params.getOutcomeCol === "outcome") + } + + test("HasOutcomeCol param has correct name and doc") { + val params = new TestParamsImpl(Identifiable.randomUID("test")) + assert(params.outcomeCol.name === "outcomeCol") + assert(params.outcomeCol.doc === "outcome column") + } + + test("HasPostTreatmentCol sets and gets post treatment column") { + val params = new TestParamsImpl(Identifiable.randomUID("test")) + params.setPostTreatmentCol("postTreatment") + assert(params.getPostTreatmentCol === "postTreatment") + } + + test("HasPostTreatmentCol param has correct name and doc") { + val params = new TestParamsImpl(Identifiable.randomUID("test")) + assert(params.postTreatmentCol.name === "postTreatmentCol") + assert(params.postTreatmentCol.doc === "post treatment indicator column") + } + + test("HasUnitCol sets and gets unit column") { + val params = new TestParamsImpl(Identifiable.randomUID("test")) + params.setUnitCol("userId") + assert(params.getUnitCol === "userId") + } + + test("HasUnitCol param has correct name") { + val params = new TestParamsImpl(Identifiable.randomUID("test")) + assert(params.unitCol.name === "unitCol") + assert(params.unitCol.doc.contains("identifier for each observed unit")) + } + + test("HasTimeCol sets and gets time column") { + val params = new TestParamsImpl(Identifiable.randomUID("test")) + params.setTimeCol("date") + assert(params.getTimeCol === "date") + } + + test("HasTimeCol param has correct name") { + val params = new TestParamsImpl(Identifiable.randomUID("test")) + assert(params.timeCol.name === "timeCol") + assert(params.timeCol.doc.contains("time when outcome is measured")) + } + + test("All params can be set together") { + val params = new TestParamsImpl(Identifiable.randomUID("test")) + params + .setTreatmentCol("treatment") + .setOutcomeCol("outcome") + .setPostTreatmentCol("post") + .setUnitCol("user") + .setTimeCol("time") + + assert(params.getTreatmentCol === "treatment") + assert(params.getOutcomeCol === "outcome") + assert(params.getPostTreatmentCol === "post") + assert(params.getUnitCol === "user") + assert(params.getTimeCol === "time") + } + + test("Setters return this.type for chaining") { + val params = new TestParamsImpl(Identifiable.randomUID("test")) + val result = params.setTreatmentCol("treatment") + assert(result eq params) + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/codegen/PyCodegenSuite.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/codegen/PyCodegenSuite.scala index 68af4d0f7ac..5c86fef031b 100644 --- a/core/src/test/scala/com/microsoft/azure/synapse/ml/codegen/PyCodegenSuite.scala +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/codegen/PyCodegenSuite.scala @@ -3,10 +3,8 @@ package com.microsoft.azure.synapse.ml.codegen -import com.microsoft.azure.synapse.ml.codegen.CodegenConfigProtocol._ import org.apache.commons.io.{FileUtils, IOUtils} import org.scalatest.funsuite.AnyFunSuite -import spray.json._ import java.io.File import java.nio.charset.StandardCharsets @@ -126,16 +124,6 @@ class PyCodegenSuite extends AnyFunSuite { lines.head.trim.stripPrefix("| packages=").stripSuffix(",") } - test("config argument supports UTF-8 response files") { - withTempDir { root => - val conf = codegenConfig(root) - val configFile = new File(root, "codegen-config.json") - writeUtf8(configFile, conf.toJson.compactPrint) - - assert(PyCodegen.parseConfigArg("@" + configFile.getAbsolutePath) === conf) - } - } - test("nested init keeps UTF-8 manual content after deterministic generated imports") { withTempDir { root => val conf = codegenConfig(root) @@ -346,25 +334,20 @@ class PyCodegenSuite extends AnyFunSuite { } } - test("manual initializer packages remain entirely hand written") { + test("cognitive compatibility init remains entirely hand written") { withTempDir { root => val conf = codegenConfig(root) - val folders = Seq("/cognitive", "/dl", "/hf") - def manual(folder: String): String = "compatibility_value = \"" + folder + "\"\n" - folders.foreach { folder => - addManualInit(conf, folder, manual(folder)) - addModule(conf, folder, "Generated.py") - } + val folder = "/cognitive" + val manual = "compatibility_value = \"manual 雪\"\n" + addManualInit(conf, folder, manual) + addModule(conf, folder, "Generated.py") PyCodegen.generateInitFiles(conf) - folders.foreach { folder => - assert(readUtf8(initFile(conf.pySrcDir, folder)) === manual(folder)) - } + val output = initFile(conf.pySrcDir, folder) + assert(readUtf8(output) === manual) PyCodegen.generateInitFiles(conf) - folders.foreach { folder => - assert(readUtf8(initFile(conf.pySrcDir, folder)) === manual(folder)) - } + assert(readUtf8(output) === manual) } } diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/codegen/VerifyCodegenConfig.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/codegen/VerifyCodegenConfig.scala new file mode 100644 index 00000000000..dc270f51b6d --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/codegen/VerifyCodegenConfig.scala @@ -0,0 +1,63 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.codegen + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import java.io.File + +class VerifyCodegenConfig extends TestBase { + + private val config = CodegenConfig( + name = "testmod", + jarName = Some("testmod.jar"), + topDir = "/top", + targetDir = "/target", + // Deliberately distinct so a test asserting one cannot pass by reading the other. + version = "1.0.0", + pythonizedVersion = "1.0.0.dev1", + rVersion = "1.0.0.1", + packageName = "com.test" + ) + + test("generatedDir returns correct path") { + assert(config.generatedDir === new File("/target", "generated")) + } + + test("pySrcDir derives from srcDir") { + assert(config.pySrcDir === new File(config.srcDir, "python")) + } + + test("rSrcDir derives from rSrcRoot") { + assert(config.rSrcDir === new File(config.rSrcRoot, "synapseml/R")) + } + + test("srcDir derives from generatedDir") { + assert(config.srcDir === new File(config.generatedDir, "src")) + } + + test("testDir derives from generatedDir") { + assert(config.testDir === new File(config.generatedDir, "test")) + } + + test("copyrightLines is non-empty") { + assert(config.copyrightLines.nonEmpty) + assert(config.copyrightLines.contains("Copyright")) + } + + test("scopeDepth is 4 spaces") { + assert(config.scopeDepth === " ") + assert(config.scopeDepth.length === 4) // scalastyle:ignore magic.number + } + + test("internalPrefix is underscore") { + assert(config.internalPrefix === "_") + } + + test("packageHelp produces valid content") { + val help = config.packageHelp("import foo") + assert(help.contains("SynapseML")) + assert(help.contains("import foo")) + assert(help.contains(config.pythonizedVersion)) + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/codegen/VerifyDefaultParamInfo.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/codegen/VerifyDefaultParamInfo.scala new file mode 100644 index 00000000000..fef8a9a02d8 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/codegen/VerifyDefaultParamInfo.scala @@ -0,0 +1,51 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.codegen + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.spark.ml.param._ +import org.apache.spark.ml.util.Identifiable + +class VerifyDefaultParamInfo extends TestBase { + + private object TestParams extends Params { + override val uid: String = Identifiable.randomUID("TestParams") // scalastyle:ignore field.name + override def copy(extra: ParamMap): Params = this + } + + test("getGeneralParamInfo returns BooleanInfo for BooleanParam") { + val p = new BooleanParam(TestParams, "b", "desc") + assert(DefaultParamInfo.getGeneralParamInfo(p) === DefaultParamInfo.BooleanInfo) + } + + test("getGeneralParamInfo returns IntInfo for IntParam") { + val p = new IntParam(TestParams, "i", "desc") + assert(DefaultParamInfo.getGeneralParamInfo(p) === DefaultParamInfo.IntInfo) + } + + test("getGeneralParamInfo returns DoubleInfo for DoubleParam") { + val p = new DoubleParam(TestParams, "d", "desc") + assert(DefaultParamInfo.getGeneralParamInfo(p) === DefaultParamInfo.DoubleInfo) + } + + test("getGeneralParamInfo returns StringArrayInfo for StringArrayParam") { + val p = new StringArrayParam(TestParams, "sa", "desc") + assert(DefaultParamInfo.getGeneralParamInfo(p) === DefaultParamInfo.StringArrayInfo) + } + + test("getGeneralParamInfo returns UnknownInfo for unrecognized param") { + val p = new Param[Any](TestParams, "unknown", "desc") + assert(DefaultParamInfo.getGeneralParamInfo(p) === DefaultParamInfo.UnknownInfo) + } + + test("ParamInfo instances have correct pyType values") { + assert(DefaultParamInfo.BooleanInfo.pyType === "bool") + assert(DefaultParamInfo.IntInfo.pyType === "int") + assert(DefaultParamInfo.DoubleInfo.pyType === "float") + assert(DefaultParamInfo.StringArrayInfo.pyType === "list") + assert(DefaultParamInfo.StringStringMapInfo.pyType === "dict") + assert(DefaultParamInfo.StringInfo.pyType === "str") + assert(DefaultParamInfo.UnknownInfo.pyType === "object") + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/codegen/VerifyGenerationUtils.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/codegen/VerifyGenerationUtils.scala new file mode 100644 index 00000000000..456dc892286 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/codegen/VerifyGenerationUtils.scala @@ -0,0 +1,51 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.codegen + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase + +class VerifyGenerationUtils extends TestBase { + + test("indent adds correct number of spaces") { + val input = "line1\nline2\nline3" + val result = GenerationUtils.indent(input, 1) + assert(result === " line1\n line2\n line3") + } + + test("indent with multiple tabs") { + val input = "hello" + val result = GenerationUtils.indent(input, 3) + assert(result === " hello") + } + + test("indent with zero tabs") { + val input = "hello\nworld" + val result = GenerationUtils.indent(input, 0) + assert(result === "hello\nworld") + } + + test("camelToSnake converts simple camelCase") { + assert(GenerationUtils.camelToSnake("maxIter") === "max_iter") + } + + test("camelToSnake converts single word") { + assert(GenerationUtils.camelToSnake("hello") === "hello") + } + + test("camelToSnake handles leading uppercase") { + assert(GenerationUtils.camelToSnake("GBTClassifier") === "gbt_classifier") + } + + test("camelToSnake handles multiple uppercase transitions") { + assert(GenerationUtils.camelToSnake("minInstancesPerNode") === "min_instances_per_node") + } + + test("camelToSnake handles all uppercase") { + assert(GenerationUtils.camelToSnake("ABC") === "abc") + } + + test("camelToSnake handles digits as word boundaries") { + assert(GenerationUtils.camelToSnake("spark3Version") === "spark_3_version") + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/core/contracts/VerifyMetrics.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/core/contracts/VerifyMetrics.scala new file mode 100644 index 00000000000..740e1dd8924 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/core/contracts/VerifyMetrics.scala @@ -0,0 +1,105 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.core.contracts + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase + +class VerifyMetrics extends TestBase { + + test("TypedMetric stores name and value correctly") { + val metric = TypedMetric("accuracy", 0.95) + assert(metric.name === "accuracy") + assert(metric.value === 0.95) + } + + test("TypedMetric works with different types") { + val doubleMetric = TypedMetric[Double]("score", 1.5) + val stringMetric = TypedMetric[String]("label", "positive") + val intMetric = TypedMetric[Int]("count", 42) + + assert(doubleMetric.value === 1.5) + assert(stringMetric.value === "positive") + assert(intMetric.value === 42) + } + + test("DoubleMetric stores name and double value") { + val metric = DoubleMetric("precision", 0.85) + assert(metric.name === "precision") + assert(metric.value === 0.85) + } + + test("StringMetric stores name and string value") { + val metric = StringMetric("category", "classification") + assert(metric.name === "category") + assert(metric.value === "classification") + } + + test("IntegralMetric stores name and long value") { + val metric = IntegralMetric("count", 1000L) + assert(metric.name === "count") + assert(metric.value === 1000L) + } + + test("TypenameMetricGroup stores name and values map") { + val metrics = Map( + "group1" -> Seq(DoubleMetric("m1", 1.0), DoubleMetric("m2", 2.0)), + "group2" -> Seq(StringMetric("s1", "test")) + ) + val group = TypenameMetricGroup("myGroup", metrics) + assert(group.name === "myGroup") + assert(group.values.size === 2) + assert(group.values("group1").length === 2) + } + + test("MetricData stores data, metricType, and modelName") { + val data = Map("accuracy" -> Seq(0.9, 0.91, 0.92)) + val metricData = MetricData(data, "classification", "logisticRegression") + + assert(metricData.data === data) + assert(metricData.metricType === "classification") + assert(metricData.modelName === "logisticRegression") + } + + test("MetricData.create converts single values to sequences") { + val singleValues = Map("accuracy" -> 0.95, "precision" -> 0.90) + val metricData = MetricData.create(singleValues, "classification", "svm") + + assert(metricData.data("accuracy") === List(0.95)) + assert(metricData.data("precision") === List(0.90)) + assert(metricData.metricType === "classification") + assert(metricData.modelName === "svm") + } + + test("MetricData.createTable preserves sequences") { + val tableData = Map( + "mse" -> Seq(0.1, 0.2, 0.3), + "rmse" -> Seq(0.316, 0.447, 0.548) + ) + val metricData = MetricData.createTable(tableData, "regression", "linearRegression") + + assert(metricData.data("mse") === Seq(0.1, 0.2, 0.3)) + assert(metricData.data("rmse").length === 3) + assert(metricData.metricType === "regression") + assert(metricData.modelName === "linearRegression") + } + + test("MetricData.create handles empty map") { + val metricData = MetricData.create(Map.empty[String, Double], "test", "model") + assert(metricData.data.isEmpty) + } + + test("MetricData.createTable handles empty map") { + val metricData = MetricData.createTable(Map.empty[String, Seq[Double]], "test", "model") + assert(metricData.data.isEmpty) + } + + test("ConvenienceTypes type aliases work correctly") { + import ConvenienceTypes._ + val name: UniqueName = "testMetric" + val table: MetricTable = Map(name -> Seq(TypedMetric("m1", 1.0))) + + assert(name === "testMetric") + assert(table.contains("testMetric")) + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/core/contracts/VerifyParams.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/core/contracts/VerifyParams.scala new file mode 100644 index 00000000000..8088b3cebae --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/core/contracts/VerifyParams.scala @@ -0,0 +1,189 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.core.contracts + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.spark.ml.param.ParamMap +import org.apache.spark.ml.util.Identifiable + +// Test implementations of the param traits +class TestHasInputCol(override val uid: String) + extends HasInputCol { + def this() = this(Identifiable.randomUID("TestHasInputCol")) + override def copy(extra: ParamMap): TestHasInputCol = defaultCopy(extra) +} + +class TestHasOutputCol(override val uid: String) + extends HasOutputCol { + def this() = this(Identifiable.randomUID("TestHasOutputCol")) + override def copy(extra: ParamMap): TestHasOutputCol = defaultCopy(extra) +} + +class TestHasInputCols(override val uid: String) + extends HasInputCols { + def this() = this(Identifiable.randomUID("TestHasInputCols")) + override def copy(extra: ParamMap): TestHasInputCols = defaultCopy(extra) +} + +class TestHasOutputCols(override val uid: String) + extends HasOutputCols { + def this() = this(Identifiable.randomUID("TestHasOutputCols")) + override def copy(extra: ParamMap): TestHasOutputCols = defaultCopy(extra) +} + +class TestHasLabelCol(override val uid: String) + extends HasLabelCol { + def this() = this(Identifiable.randomUID("TestHasLabelCol")) + override def copy(extra: ParamMap): TestHasLabelCol = defaultCopy(extra) +} + +class TestHasFeaturesCol(override val uid: String) + extends HasFeaturesCol { + def this() = this(Identifiable.randomUID("TestHasFeaturesCol")) + override def copy(extra: ParamMap): TestHasFeaturesCol = defaultCopy(extra) +} + +class TestHasWeightCol(override val uid: String) + extends HasWeightCol { + def this() = this(Identifiable.randomUID("TestHasWeightCol")) + override def copy(extra: ParamMap): TestHasWeightCol = defaultCopy(extra) +} + +class TestHasScoredLabelsCol(override val uid: String) + extends HasScoredLabelsCol { + def this() = this(Identifiable.randomUID("TestHasScoredLabelsCol")) + override def copy(extra: ParamMap): TestHasScoredLabelsCol = defaultCopy(extra) +} + +class TestHasScoresCol(override val uid: String) + extends HasScoresCol { + def this() = this(Identifiable.randomUID("TestHasScoresCol")) + override def copy(extra: ParamMap): TestHasScoresCol = defaultCopy(extra) +} + +class TestHasScoredProbabilitiesCol(override val uid: String) + extends HasScoredProbabilitiesCol { + def this() = this(Identifiable.randomUID("TestHasScoredProbabilitiesCol")) + override def copy(extra: ParamMap): TestHasScoredProbabilitiesCol = defaultCopy(extra) +} + +class TestHasEvaluationMetric(override val uid: String) + extends HasEvaluationMetric { + def this() = this(Identifiable.randomUID("TestHasEvaluationMetric")) + override def copy(extra: ParamMap): TestHasEvaluationMetric = defaultCopy(extra) +} + +class TestHasValidationIndicatorCol(override val uid: String) + extends HasValidationIndicatorCol { + def this() = this(Identifiable.randomUID("TestHasValidationIndicatorCol")) + override def copy(extra: ParamMap): TestHasValidationIndicatorCol = defaultCopy(extra) +} + +class TestHasInitScoreCol(override val uid: String) + extends HasInitScoreCol { + def this() = this(Identifiable.randomUID("TestHasInitScoreCol")) + override def copy(extra: ParamMap): TestHasInitScoreCol = defaultCopy(extra) +} + +class TestHasGroupCol(override val uid: String) + extends HasGroupCol { + def this() = this(Identifiable.randomUID("TestHasGroupCol")) + override def copy(extra: ParamMap): TestHasGroupCol = defaultCopy(extra) +} + +class VerifyParams extends TestBase { + + test("HasInputCol set and get work correctly") { + val obj = new TestHasInputCol() + obj.setInputCol("myInput") + assert(obj.getInputCol === "myInput") + } + + test("HasOutputCol set and get work correctly") { + val obj = new TestHasOutputCol() + obj.setOutputCol("myOutput") + assert(obj.getOutputCol === "myOutput") + } + + test("HasInputCols set and get work correctly") { + val obj = new TestHasInputCols() + val cols = Array("col1", "col2", "col3") + obj.setInputCols(cols) + assert(obj.getInputCols.sameElements(cols)) + } + + test("HasOutputCols set and get work correctly") { + val obj = new TestHasOutputCols() + val cols = Array("out1", "out2") + obj.setOutputCols(cols) + assert(obj.getOutputCols.sameElements(cols)) + } + + test("HasLabelCol set and get work correctly") { + val obj = new TestHasLabelCol() + obj.setLabelCol("target") + assert(obj.getLabelCol === "target") + } + + test("HasFeaturesCol set and get work correctly") { + val obj = new TestHasFeaturesCol() + obj.setFeaturesCol("features") + assert(obj.getFeaturesCol === "features") + } + + test("HasWeightCol set and get work correctly") { + val obj = new TestHasWeightCol() + obj.setWeightCol("weight") + assert(obj.getWeightCol === "weight") + } + + test("HasScoredLabelsCol set and get work correctly") { + val obj = new TestHasScoredLabelsCol() + obj.setScoredLabelsCol("scoredLabels") + assert(obj.getScoredLabelsCol === "scoredLabels") + } + + test("HasScoresCol set and get work correctly") { + val obj = new TestHasScoresCol() + obj.setScoresCol("scores") + assert(obj.getScoresCol === "scores") + } + + test("HasScoredProbabilitiesCol set and get work correctly") { + val obj = new TestHasScoredProbabilitiesCol() + obj.setScoredProbabilitiesCol("probs") + assert(obj.getScoredProbabilitiesCol === "probs") + } + + test("HasEvaluationMetric set and get work correctly") { + val obj = new TestHasEvaluationMetric() + obj.setEvaluationMetric("accuracy") + assert(obj.getEvaluationMetric === "accuracy") + } + + test("HasValidationIndicatorCol set and get work correctly") { + val obj = new TestHasValidationIndicatorCol() + obj.setValidationIndicatorCol("isValidation") + assert(obj.getValidationIndicatorCol === "isValidation") + } + + test("HasInitScoreCol set and get work correctly") { + val obj = new TestHasInitScoreCol() + obj.setInitScoreCol("initScore") + assert(obj.getInitScoreCol === "initScore") + } + + test("HasGroupCol set and get work correctly") { + val obj = new TestHasGroupCol() + obj.setGroupCol("group") + assert(obj.getGroupCol === "group") + } + + // Test chaining + test("param setters return this for chaining") { + val obj = new TestHasInputCol() + val result = obj.setInputCol("test") + assert(result eq obj) + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/core/env/VerifyPackageUtils.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/core/env/VerifyPackageUtils.scala new file mode 100644 index 00000000000..6a50afcbbaf --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/core/env/VerifyPackageUtils.scala @@ -0,0 +1,58 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.core.env + +import com.microsoft.azure.synapse.ml.build.BuildInfo +import com.microsoft.azure.synapse.ml.core.test.base.TestBase + +class VerifyPackageUtils extends TestBase { + + test("ScalaVersionSuffix is extracted from BuildInfo.scalaVersion") { + // Scala version is typically like "2.12.15" or "2.13.x" + // ScalaVersionSuffix should be "2.12" or "2.13" + val suffix = PackageUtils.ScalaVersionSuffix + assert(suffix.split("\\.").length === 2) + assert(suffix.startsWith("2.")) + } + + test("PackageGroup has expected value") { + assert(PackageUtils.PackageGroup === "com.microsoft.azure") + } + + test("PackageName contains scala version suffix") { + assert(PackageUtils.PackageName.startsWith("synapseml_")) + assert(PackageUtils.PackageName.contains(PackageUtils.ScalaVersionSuffix)) + } + + test("PackageMavenCoordinate has correct format") { + val coord = PackageUtils.PackageMavenCoordinate + // Format should be: group:artifact:version + val parts = coord.split(":") + assert(parts.length === 3) + assert(parts(0) === PackageUtils.PackageGroup) + assert(parts(1) === PackageUtils.PackageName) + assert(parts(2) === BuildInfo.version) + } + + test("PackageRepository is a valid URL") { + val repo = PackageUtils.PackageRepository + assert(repo.startsWith("https://")) + } + + test("SparkMavenPackageList contains package coordinate") { + val packages = PackageUtils.SparkMavenPackageList + assert(packages.contains(PackageUtils.PackageMavenCoordinate)) + } + + test("SparkMavenPackageList contains spark-avro") { + val packages = PackageUtils.SparkMavenPackageList + assert(packages.contains("spark-avro")) + } + + test("SparkMavenRepositoryList points at the SynapseML maven feed") { + val repos = PackageUtils.SparkMavenRepositoryList + assert(repos === "https://mmlspark.blob.core.windows.net/maven") + assert(repos === PackageUtils.PackageRepository) + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/core/metrics/VerifyMetricConstants.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/core/metrics/VerifyMetricConstants.scala new file mode 100644 index 00000000000..e2541c95344 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/core/metrics/VerifyMetricConstants.scala @@ -0,0 +1,172 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.core.metrics + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase + +class VerifyMetricConstants extends TestBase { + + // Regression metrics tests + test("regression metric constants have expected values") { + assert(MetricConstants.MseSparkMetric === "mse") + assert(MetricConstants.RmseSparkMetric === "rmse") + assert(MetricConstants.R2SparkMetric === "r2") + assert(MetricConstants.MaeSparkMetric === "mae") + assert(MetricConstants.RegressionMetricsName === "regression") + } + + test("RegressionMetrics set contains all regression metrics") { + assert(MetricConstants.RegressionMetrics.contains(MetricConstants.MseSparkMetric)) + assert(MetricConstants.RegressionMetrics.contains(MetricConstants.RmseSparkMetric)) + assert(MetricConstants.RegressionMetrics.contains(MetricConstants.R2SparkMetric)) + assert(MetricConstants.RegressionMetrics.contains(MetricConstants.MaeSparkMetric)) + assert(MetricConstants.RegressionMetrics.contains(MetricConstants.RegressionMetricsName)) + assert(MetricConstants.RegressionMetrics.size === 5) + } + + // Classification metrics tests + test("classification metric constants have expected values") { + assert(MetricConstants.AreaUnderROCMetric === "areaUnderROC") + assert(MetricConstants.AreaUnderPRMetric === "areaUnderPR") + assert(MetricConstants.AucSparkMetric === "AUC") + assert(MetricConstants.AccuracySparkMetric === "accuracy") + assert(MetricConstants.PrecisionSparkMetric === "precision") + assert(MetricConstants.RecallSparkMetric === "recall") + assert(MetricConstants.ClassificationMetricsName === "classification") + } + + test("ClassificationMetrics set contains all classification metrics") { + assert(MetricConstants.ClassificationMetrics.contains(MetricConstants.AreaUnderROCMetric)) + assert(MetricConstants.ClassificationMetrics.contains(MetricConstants.AreaUnderPRMetric)) + assert(MetricConstants.ClassificationMetrics.contains(MetricConstants.AucSparkMetric)) + assert(MetricConstants.ClassificationMetrics.contains(MetricConstants.AccuracySparkMetric)) + assert(MetricConstants.ClassificationMetrics.contains(MetricConstants.PrecisionSparkMetric)) + assert(MetricConstants.ClassificationMetrics.contains(MetricConstants.RecallSparkMetric)) + assert(MetricConstants.ClassificationMetrics.contains(MetricConstants.ClassificationMetricsName)) + assert(MetricConstants.ClassificationMetrics.size === 7) + } + + test("AllSparkMetrics constant") { + assert(MetricConstants.AllSparkMetrics === "all") + } + + // Column name tests + test("regression column names have expected values") { + assert(MetricConstants.MseColumnName === "mean_squared_error") + assert(MetricConstants.RmseColumnName === "root_mean_squared_error") + assert(MetricConstants.R2ColumnName === "R^2") + assert(MetricConstants.MaeColumnName === "mean_absolute_error") + } + + test("classification column names have expected values") { + assert(MetricConstants.AucColumnName === "AUC") + assert(MetricConstants.AreaUnderPRColumnName === "areaUnderPR") + assert(MetricConstants.PrecisionColumnName === "precision") + assert(MetricConstants.RecallColumnName === "recall") + assert(MetricConstants.AccuracyColumnName === "accuracy") + } + + test("multiclass column names have expected values") { + assert(MetricConstants.AverageAccuracy === "average_accuracy") + assert(MetricConstants.MacroAveragedRecall === "macro_averaged_recall") + assert(MetricConstants.MacroAveragedPrecision === "macro_averaged_precision") + assert(MetricConstants.ConfusionMatrix === "confusion_matrix") + } + + // MetricToColumnName mapping tests + test("MetricToColumnName contains correct mappings") { + assert(MetricConstants.MetricToColumnName(MetricConstants.AreaUnderROCMetric) === + MetricConstants.AucColumnName) + assert(MetricConstants.MetricToColumnName(MetricConstants.AucSparkMetric) === + MetricConstants.AucColumnName) + assert(MetricConstants.MetricToColumnName(MetricConstants.AreaUnderPRMetric) === + MetricConstants.AreaUnderPRColumnName) + assert(MetricConstants.MetricToColumnName(MetricConstants.AccuracySparkMetric) === + MetricConstants.AccuracyColumnName) + assert(MetricConstants.MetricToColumnName(MetricConstants.PrecisionSparkMetric) === + MetricConstants.PrecisionColumnName) + assert(MetricConstants.MetricToColumnName(MetricConstants.RecallSparkMetric) === + MetricConstants.RecallColumnName) + assert(MetricConstants.MetricToColumnName(MetricConstants.MseSparkMetric) === + MetricConstants.MseColumnName) + assert(MetricConstants.MetricToColumnName(MetricConstants.RmseSparkMetric) === + MetricConstants.RmseColumnName) + assert(MetricConstants.MetricToColumnName(MetricConstants.R2SparkMetric) === + MetricConstants.R2ColumnName) + assert(MetricConstants.MetricToColumnName(MetricConstants.MaeSparkMetric) === + MetricConstants.MaeColumnName) + } + + // Column lists tests + test("ClassificationColumns preserves the legacy common schema") { + assert(MetricConstants.ClassificationColumns === List( + MetricConstants.AccuracyColumnName, + MetricConstants.PrecisionColumnName, + MetricConstants.RecallColumnName)) + } + + test("BinaryClassificationColumns contains binary-only metrics after common metrics") { + assert(MetricConstants.BinaryClassificationColumns === MetricConstants.ClassificationColumns ++ List( + MetricConstants.AucColumnName, + MetricConstants.AreaUnderPRColumnName)) + } + + test("RegressionColumns contains expected columns") { + assert(MetricConstants.RegressionColumns === List( + MetricConstants.MseColumnName, + MetricConstants.RmseColumnName, + MetricConstants.R2ColumnName, + MetricConstants.MaeColumnName)) + } + + // Evaluation type tests + test("evaluation type constants have expected values") { + assert(MetricConstants.ClassificationEvaluationType === "Classification") + assert(MetricConstants.EvaluationType === "evaluation_type") + } + + // ROC column names tests + test("ROC column names have expected values") { + assert(MetricConstants.FpRateROCColumnName === "false_positive_rate") + assert(MetricConstants.TpRateROCColumnName === "true_positive_rate") + assert(MetricConstants.FpRateROCLog === "fpr") + assert(MetricConstants.TpRateROCLog === "tpr") + } + + test("BinningThreshold has expected value") { + assert(MetricConstants.BinningThreshold === 1000) + } + + // Per instance metrics tests + test("per instance metric constants have expected values") { + assert(MetricConstants.L1LossMetric === "L1_loss") + assert(MetricConstants.L2LossMetric === "L2_loss") + assert(MetricConstants.LogLossMetric === "log_loss") + } + + test("RegressionPerInstanceMetrics contains expected metrics") { + assert(MetricConstants.RegressionPerInstanceMetrics.contains(MetricConstants.RegressionMetricsName)) + assert(MetricConstants.RegressionPerInstanceMetrics.size === 1) + } + + test("ClassificationPerInstanceMetrics contains expected metrics") { + assert(MetricConstants.ClassificationPerInstanceMetrics.contains(MetricConstants.ClassificationMetricsName)) + assert(MetricConstants.ClassificationPerInstanceMetrics.size === 1) + } + + // FindBestModelMetrics tests + test("FindBestModelMetrics contains all expected metrics") { + assert(MetricConstants.FindBestModelMetrics.contains(MetricConstants.MseSparkMetric)) + assert(MetricConstants.FindBestModelMetrics.contains(MetricConstants.RmseSparkMetric)) + assert(MetricConstants.FindBestModelMetrics.contains(MetricConstants.R2SparkMetric)) + assert(MetricConstants.FindBestModelMetrics.contains(MetricConstants.MaeSparkMetric)) + assert(MetricConstants.FindBestModelMetrics.contains(MetricConstants.AccuracySparkMetric)) + assert(MetricConstants.FindBestModelMetrics.contains(MetricConstants.PrecisionSparkMetric)) + assert(MetricConstants.FindBestModelMetrics.contains(MetricConstants.RecallSparkMetric)) + assert(MetricConstants.FindBestModelMetrics.contains(MetricConstants.AreaUnderROCMetric)) + assert(MetricConstants.FindBestModelMetrics.contains(MetricConstants.AucSparkMetric)) + assert(MetricConstants.FindBestModelMetrics.contains(MetricConstants.AreaUnderPRMetric)) + assert(MetricConstants.FindBestModelMetrics.size === 10) + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/core/schema/VerifyBinaryFileSchema.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/core/schema/VerifyBinaryFileSchema.scala new file mode 100644 index 00000000000..86b81ad7a6f --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/core/schema/VerifyBinaryFileSchema.scala @@ -0,0 +1,92 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.core.schema + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.spark.sql.Row +import org.apache.spark.sql.types._ + +class VerifyBinaryFileSchema extends TestBase { + + test("ColumnSchema has correct structure") { + val schema = BinaryFileSchema.ColumnSchema + assert(schema.fields.length === 2) + + val pathField = schema.fields(0) + assert(pathField.name === "path") + assert(pathField.dataType === StringType) + assert(pathField.nullable === true) + + val bytesField = schema.fields(1) + assert(bytesField.name === "bytes") + assert(bytesField.dataType === BinaryType) + assert(bytesField.nullable === true) + } + + test("Schema wraps ColumnSchema in value field") { + val schema = BinaryFileSchema.Schema + assert(schema.fields.length === 1) + + val valueField = schema.fields(0) + assert(valueField.name === "value") + assert(valueField.dataType === BinaryFileSchema.ColumnSchema) + assert(valueField.nullable === true) + } + + test("getPath extracts path from Row") { + val testPath = "/path/to/file.bin" + val testBytes = Array[Byte](1, 2, 3) + val row = Row(testPath, testBytes) + + assert(BinaryFileSchema.getPath(row) === testPath) + } + + test("getBytes extracts bytes from Row") { + val testPath = "/path/to/file.bin" + val testBytes = Array[Byte](1, 2, 3, 4, 5) + val row = Row(testPath, testBytes) + + val result = BinaryFileSchema.getBytes(row) + assert(result.sameElements(testBytes)) + } + + test("getPath handles empty path") { + val row = Row("", Array[Byte]()) + assert(BinaryFileSchema.getPath(row) === "") + } + + test("getBytes handles empty byte array") { + val row = Row("test", Array[Byte]()) + assert(BinaryFileSchema.getBytes(row).length === 0) + } + + test("isBinaryFile with DataType returns true for matching schema") { + assert(BinaryFileSchema.isBinaryFile(BinaryFileSchema.ColumnSchema) === true) + } + + test("isBinaryFile with DataType returns false for non-matching schema") { + assert(BinaryFileSchema.isBinaryFile(StringType) === false) + assert(BinaryFileSchema.isBinaryFile(BinaryType) === false) + assert(BinaryFileSchema.isBinaryFile(IntegerType) === false) + + // Different StructType + val differentSchema = StructType(Seq( + StructField("path", StringType, true) + )) + assert(BinaryFileSchema.isBinaryFile(differentSchema) === false) + } + + test("isBinaryFile with StructField returns true for matching schema") { + val field = StructField("data", BinaryFileSchema.ColumnSchema, true) + assert(BinaryFileSchema.isBinaryFile(field) === true) + } + + test("isBinaryFile with StructField returns false for non-matching schema") { + val stringField = StructField("data", StringType, true) + assert(BinaryFileSchema.isBinaryFile(stringField) === false) + + val binaryField = StructField("data", BinaryType, true) + assert(BinaryFileSchema.isBinaryFile(binaryField) === false) + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/core/schema/VerifyImageSchemaUtils.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/core/schema/VerifyImageSchemaUtils.scala new file mode 100644 index 00000000000..3aebc578557 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/core/schema/VerifyImageSchemaUtils.scala @@ -0,0 +1,86 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.core.schema + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.spark.ml.image.ImageSchema +import org.apache.spark.sql.types._ + +class VerifyImageSchemaUtils extends TestBase { + + test("ColumnSchemaNullable has correct structure") { + val schema = ImageSchemaUtils.ColumnSchemaNullable + assert(schema.fields.length === 6) + + assert(schema.fields(0).name === "origin") + assert(schema.fields(0).dataType === StringType) + + assert(schema.fields(1).name === "height") + assert(schema.fields(1).dataType === IntegerType) + + assert(schema.fields(2).name === "width") + assert(schema.fields(2).dataType === IntegerType) + + assert(schema.fields(3).name === "nChannels") + assert(schema.fields(3).dataType === IntegerType) + + assert(schema.fields(4).name === "mode") + assert(schema.fields(4).dataType === IntegerType) + + assert(schema.fields(5).name === "data") + assert(schema.fields(5).dataType === BinaryType) + } + + test("ColumnSchemaNullable fields are all nullable") { + val schema = ImageSchemaUtils.ColumnSchemaNullable + schema.fields.foreach { field => + assert(field.nullable === true, s"Field ${field.name} should be nullable") + } + } + + test("ImageSchemaNullable wraps ColumnSchemaNullable") { + val schema = ImageSchemaUtils.ImageSchemaNullable + assert(schema.fields.length === 1) + assert(schema.fields(0).name === "image") + assert(schema.fields(0).dataType === ImageSchemaUtils.ColumnSchemaNullable) + assert(schema.fields(0).nullable === true) + } + + test("isImage returns true for ImageSchema.columnSchema") { + assert(ImageSchemaUtils.isImage(ImageSchema.columnSchema) === true) + } + + test("isImage returns false for non-image types") { + assert(ImageSchemaUtils.isImage(StringType) === false) + assert(ImageSchemaUtils.isImage(BinaryType) === false) + assert(ImageSchemaUtils.isImage(IntegerType) === false) + } + + test("isImage returns false for different struct type") { + val differentSchema = StructType(Seq( + StructField("path", StringType, true) + )) + assert(ImageSchemaUtils.isImage(differentSchema) === false) + } + + test("isImage with StructField returns true for image column") { + val imageField = StructField("img", ImageSchema.columnSchema, true) + assert(ImageSchemaUtils.isImage(imageField) === true) + } + + test("isImage with StructField returns false for non-image column") { + val stringField = StructField("text", StringType, true) + assert(ImageSchemaUtils.isImage(stringField) === false) + } + + test("ColumnSchemaNullable matches ImageSchema.columnSchema structurally") { + // The nullable version should match structurally when ignoring nullability + val isMatch = DataType.equalsStructurally( + ImageSchemaUtils.ColumnSchemaNullable, + ImageSchema.columnSchema, + ignoreNullability = true + ) + assert(isMatch === true) + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/core/schema/VerifySchemaConstants.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/core/schema/VerifySchemaConstants.scala new file mode 100644 index 00000000000..19908e14fa3 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/core/schema/VerifySchemaConstants.scala @@ -0,0 +1,54 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.core.schema + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase + +class VerifySchemaConstants extends TestBase { + + test("score column kind constants have expected values") { + assert(SchemaConstants.ScoreColumnKind === "ScoreColumnKind") + assert(SchemaConstants.ScoreValueKind === "ScoreValueKind") + } + + test("label and score column names have expected values") { + assert(SchemaConstants.TrueLabelsColumn === "true_labels") + assert(SchemaConstants.ScoredLabelsColumn === "scored_labels") + assert(SchemaConstants.ScoresColumn === "scores") + assert(SchemaConstants.ScoredProbabilitiesColumn === "scored_probabilities") + } + + test("model and tag constants have expected values") { + assert(SchemaConstants.ScoreModelPrefix === "score_model") + assert(SchemaConstants.MMLTag === "mml") + assert(SchemaConstants.MLlibTag === "ml_attr") + } + + test("residual column names have expected values") { + assert(SchemaConstants.TreatmentResidualColumn === "treatment_residual") + assert(SchemaConstants.OutcomeResidualColumn === "outcome_residual") + } + + test("categorical metadata tag constants have expected values") { + assert(SchemaConstants.Ordinal === "ord") + assert(SchemaConstants.MLlibTypeTag === "type") + assert(SchemaConstants.ValuesString === "vals") + assert(SchemaConstants.ValuesInt === "vals_int") + assert(SchemaConstants.ValuesLong === "vals_long") + assert(SchemaConstants.ValuesDouble === "vals_double") + assert(SchemaConstants.ValuesBool === "vals_bool") + assert(SchemaConstants.HasNullLevels === "null_exists") + } + + test("ML kind constants have expected values") { + assert(SchemaConstants.ClassificationKind === "Classification") + assert(SchemaConstants.RegressionKind === "Regression") + } + + test("Spark native column name constants have expected values") { + assert(SchemaConstants.SparkPredictionColumn === "prediction") + assert(SchemaConstants.SparkRawPredictionColumn === "rawPrediction") + assert(SchemaConstants.SparkProbabilityColumn === "probability") + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/core/serialize/ValidateComplexParamSerializer.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/core/serialize/ValidateComplexParamSerializer.scala index 919e73a66ce..d349e0b20b9 100644 --- a/core/src/test/scala/com/microsoft/azure/synapse/ml/core/serialize/ValidateComplexParamSerializer.scala +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/core/serialize/ValidateComplexParamSerializer.scala @@ -3,12 +3,14 @@ package com.microsoft.azure.synapse.ml.core.serialize +import com.microsoft.azure.synapse.ml.core.env.StreamUtilities.using import com.microsoft.azure.synapse.ml.core.test.base.TestBase import com.microsoft.azure.synapse.ml.param.ByteArrayParam import org.apache.commons.io.FileUtils +import org.apache.hadoop.fs.Path import org.apache.spark.ml.param.{Param, ParamMap, Params} import org.apache.spark.ml.util._ -import org.apache.spark.ml.{ComplexParamsReadable, ComplexParamsWritable, Transformer} +import org.apache.spark.ml.{ComplexParamsReadable, ComplexParamsWritable, ObjectSerializer, Serializer, Transformer} import org.apache.spark.sql.types.StructType import org.apache.spark.sql.{DataFrame, Dataset} @@ -104,6 +106,43 @@ class ValidateComplexParamSerializer extends TestBase { assert(mpt1.getStringParam === mpt2.getStringParam) } + test("Complex Param serialization should read metadata written by the legacy SparkContext path") { + spark + val bytes = "foo".toCharArray.map(_.toByte) + + val mpt1 = new MixedParamTest("foo").setByteArray(bytes).setStringParam("foo") + mpt1.write.overwrite().save(saveFile) + + // Rewrite the metadata the way SynapseML wrote it before the reader moved off + // SparkContext.textFile, so this asserts that models saved by earlier versions still + // load rather than just round-tripping the current writer against the current reader. + val metadataDir = new File(saveFile, "metadata") + val metadataJson = spark.read.text(metadataDir.toString).first().getString(0) + FileUtils.deleteDirectory(metadataDir) + spark.sparkContext.parallelize(Seq(metadataJson), 1).saveAsTextFile(metadataDir.toString) + + val mpt2 = MixedParamTest.load(saveFile) + assert(mpt1.getByteArray === mpt2.getByteArray) + assert(mpt1.getStringParam === mpt2.getStringParam) + } + + test("Objects written the way earlier versions wrote them still load through the session path") { + spark + val obj = "round-trip payload".toCharArray.map(_.toByte) + val legacyPath = new Path(new File(tmpDir.toFile, "legacy-object").toString) + + // Reproduce the previous write path byte for byte: the FileSystem resolved from the + // SparkContext Hadoop configuration rather than from the session, writing through the same + // Serializer.write. Only the configuration lookup moved, so this pins that the on-disk format + // is unchanged and that objects written by earlier SynapseML versions still load. + using(legacyPath.getFileSystem(spark.sparkContext.hadoopConfiguration).create(legacyPath, true)) { os => + Serializer.write(obj, os) + }.get + + assert(new ObjectSerializer[Array[Byte]](spark).read(legacyPath) === obj) + assert(Serializer.readFromHDFS[Array[Byte]](spark, legacyPath) === obj) + } + override def afterAll(): Unit = { new File(saveFile).delete() new File(saveFile2).delete() diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/core/test/pipeline/PipelineTestCoverageSuite.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/core/test/pipeline/PipelineTestCoverageSuite.scala new file mode 100644 index 00000000000..1b593c49d35 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/core/test/pipeline/PipelineTestCoverageSuite.scala @@ -0,0 +1,191 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.core.test.pipeline + +import com.microsoft.azure.synapse.ml.build.BuildInfo +import com.microsoft.azure.synapse.ml.core.env.StreamUtilities.usingSource +import org.scalatest.funsuite.AnyFunSuite + +import java.io.File +import scala.annotation.tailrec +import scala.io.Source + +/** + * Guards against test suites that silently never run in CI. + * + * The UnitTests matrix runs `testOnly com.microsoft.azure.synapse.ml.$PACKAGE.**` unless a leg + * overrides it with an explicit TEST_CLASSES list. A suite in a package no leg names is not + * reported as skipped -- it simply never executes, and the build stays green. + */ +class PipelineTestCoverageSuite extends AnyFunSuite { + + private val rootPackage = "com.microsoft.azure.synapse.ml" + + /** Suite name prefixes launched by their own pipeline stages rather than the UnitTests matrix. */ + private val dedicatedStageSuites = Seq( + "nbtest.DatabricksCPUTests", + "nbtest.DatabricksGPUTests", + "nbtest.DatabricksRapidsTests", + "nbtest.SynapseTests", + "nbtest.FabricNotebookTests", + "nbtest.FabricSmokeTests", + "nbtest.FabricTestCleanup", + "nbtest.SynapseTestCleanup" + ).map(suffix => s"$rootPackage.$suffix") + + /** + * Suites that deliberately have no pipeline leg yet, listed here so the guard stays green + * without pretending they are covered. + * + * DatabricksCPUStreamingTests drives the "Deploying a Classifier" notebook, which + * DatabricksUtilities keeps out of the parallel CPU partitions because its server.stop() + * cancels every SparkContext job on Spark 4 and would kill notebooks sharing the cluster. + * Giving it a matrix leg of its own was tried and does not work yet: the extra concurrent + * cluster exhausts the instance pool so its libraries never finish installing, and the + * notebook itself still errors. Scheduling it needs pool capacity plus a notebook fix, + * neither of which belongs in a branch sync. + */ + private val unscheduledSuites = Seq( + "nbtest.DatabricksCPUStreamingTests" + ).map(suffix => s"$rootPackage.$suffix") + + /** + * ScalaTest entry points. Anything extending these, directly or transitively, is a suite. + * The `*Like` traits are separate entry points, not subtypes of the classes, so both spellings + * are needed -- `AnalyzeTextLROSuite` extends `AnyFunSuiteLike` directly. + */ + private val scalaTestBaseTypes: Set[String] = { + val styles = Set("Suite", "FunSuite", "FlatSpec", "WordSpec", "FreeSpec", "PropSpec", "FeatureSpec", "FunSpec") + val spellings = styles ++ styles.map("Any" + _) + spellings ++ spellings.map(_ + "Like") + "TestSuite" + "TestSuiteLike" + } + + private val repoRoot: File = BuildInfo.baseDirectory.getParentFile + + private def readFile(file: File): String = + usingSource(Source.fromFile(file, "UTF-8"))(_.mkString).get + + private def scalaFilesUnder(dir: File): Seq[File] = { + if (!dir.isDirectory) Seq.empty + else Option(dir.listFiles()).toSeq.flatten.flatMap { child => + if (child.isDirectory) scalaFilesUnder(child) + else if (child.getName.endsWith(".scala")) Seq(child) + else Seq.empty + } + } + + private def testSourceFiles: Seq[File] = + Option(repoRoot.listFiles()).toSeq.flatten + .filter(_.isDirectory) + .flatMap(module => scalaFilesUnder(new File(module, "src/test/scala"))) + + private case class Declaration(name: String, parents: Seq[String], isConcreteClass: Boolean) + + /** + * Declarations routinely wrap, e.g. `class Foo(bar: String)\n extends TestBase`, so match + * against a whitespace-flattened copy of the source. The segment between the name and + * `extends` may not cross another declaration keyword, which keeps a class without an + * `extends` clause from borrowing the next declaration's parents. + */ + private val declarationPattern = + raw"(? + val parents = m.group(4) + .split("\\bwith\\b") + .map(_.trim.takeWhile(c => c.isLetterOrDigit || c == '.' || c == '_')) + .map(name => name.split('.').lastOption.getOrElse(name)) + .filter(_.nonEmpty) + .toSeq + Declaration(m.group(2), parents, m.group(1) == "class" || m.group(1) == "case class") + }.toSeq + } + + /** Walks the local extends graph so suites are found by ancestry, not by naming convention. */ + private def suiteTypeNames(declarations: Seq[Declaration]): Set[String] = { + val parentsByName = declarations.groupBy(_.name).map { + case (name, decls) => name -> decls.flatMap(_.parents).toSet + } + @tailrec + def expand(known: Set[String]): Set[String] = { + val grown = known ++ parentsByName.collect { case (name, parents) if parents.exists(known) => name } + if (grown.size == known.size) known else expand(grown) + } + expand(scalaTestBaseTypes) + } + + private def matrixSpecs(pipeline: String): Seq[String] = { + val unitTestsJob = pipeline.split(raw"(?m)^- job: ") + .find(_.startsWith("UnitTests")) + .getOrElse(fail("Could not locate the UnitTests job in pipeline.yaml")) + val matrixBlock = unitTestsJob.split(raw"(?m)^ steps:").head + + // Anchor each package segment so the trailing ".**" is not swallowed by the segment matcher. + val fromTestClasses = raw"$rootPackage(?:\.\w+)*(?:\.\*\*)?".r.findAllIn(matrixBlock).toSeq + // A leg with TEST_CLASSES ignores its PACKAGE, so a package glob only counts when that leg + // does not also pin an explicit class list. + val fromPackages = matrixBlock.split(raw"(?m)^ \w+:") + .filterNot(_.contains("TEST_CLASSES:")) + .flatMap { leg => + raw"""PACKAGE:\s*"([\w.]+)"""".r.findAllMatchIn(leg).map(m => s"$rootPackage.${m.group(1)}.**") + }.toSeq + + (fromTestClasses ++ fromPackages).distinct + } + + private def isCovered(fqcn: String, specs: Seq[String]): Boolean = + specs.exists { spec => + if (spec.endsWith(".**")) fqcn.startsWith(spec.dropRight(2)) + else fqcn == spec + } + + test("Every test suite is claimed by a leg of the UnitTests matrix") { + val pipelineFile = new File(repoRoot, "pipeline.yaml") + assert(pipelineFile.exists(), s"pipeline.yaml not found at ${pipelineFile.getAbsolutePath}") + val specs = matrixSpecs(readFile(pipelineFile)) + assert(specs.nonEmpty, "Parsed no test specs out of the UnitTests matrix") + + val sources = testSourceFiles.map(file => file -> readFile(file)) + assert(sources.size > 100, s"Expected to scan the whole repo, only found ${sources.size} test files") + + val parsed = sources.map { case (file, source) => + val pkg = raw"(?m)^package\s+([\w.]+)".r.findFirstMatchIn(source).map(_.group(1)) + (file, pkg, parseDeclarations(source)) + } + val suiteTypes = suiteTypeNames(parsed.flatMap { case (_, _, decls) => decls }) + + val discovered = parsed.flatMap { case (file, pkg, decls) => + pkg.toSeq.flatMap { p => + decls + .filter(decl => decl.isConcreteClass && decl.parents.exists(suiteTypes)) + .map(decl => s"$p.${decl.name}" -> file.getName) + } + }.distinct + + // Without this the guard would pass vacuously if source parsing ever silently broke. + assert(discovered.size > 200, s"Only discovered ${discovered.size} suites; source parsing looks broken") + Seq( + "com.microsoft.azure.synapse.ml.services.language.AnalyzeTextLROSuite", // wraps, extends AnyFunSuiteLike + "com.microsoft.azure.synapse.ml.stages.EnsembleByKeySuite", + "com.microsoft.azure.synapse.ml.services.search.AzureSearchAuthSuite" + ).foreach { known => + assert(discovered.exists(_._1 == known), s"Suite discovery missed $known") + } + + val orphans = discovered + .filterNot { case (fqcn, _) => dedicatedStageSuites.exists(fqcn.startsWith) } + .filterNot { case (fqcn, _) => unscheduledSuites.exists(fqcn.startsWith) } + .filterNot { case (fqcn, _) => isCovered(fqcn, specs) } + .map { case (fqcn, fileName) => s"$fqcn ($fileName)" } + .sorted + + assert(orphans.isEmpty, + s"${orphans.size} test suite(s) are never run by CI. Add them to the UnitTests matrix in " + + s"pipeline.yaml:\n ${orphans.mkString("\n ")}") + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/core/utils/VerifyJarLoadingUtils.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/core/utils/VerifyJarLoadingUtils.scala new file mode 100644 index 00000000000..bc95c6ed732 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/core/utils/VerifyJarLoadingUtils.scala @@ -0,0 +1,25 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.core.utils + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase + +class VerifyJarLoadingUtils extends TestBase { + + test("className strips .class extension and converts slashes to dots") { + assert(JarLoadingUtils.className("com/example/MyClass.class") === "com.example.MyClass") + } + + test("className returns input unchanged if no .class extension") { + assert(JarLoadingUtils.className("com.example.MyClass") === "com.example.MyClass") + } + + test("className handles nested class paths") { + assert(JarLoadingUtils.className("a/b/c/D.class") === "a.b.c.D") + } + + test("className handles simple filename") { + assert(JarLoadingUtils.className("Main.class") === "Main") + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/core/utils/VerifyModelEquality.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/core/utils/VerifyModelEquality.scala new file mode 100644 index 00000000000..719072b0836 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/core/utils/VerifyModelEquality.scala @@ -0,0 +1,35 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.core.utils + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase + +class VerifyModelEquality extends TestBase { + + // These assertions hold both before and after the jaccardSimilarity fix in the companion + // production PR, so this suite stays independently mergeable. The discriminating + // partial-overlap tests ship with that fix rather than here. + + test("jaccardSimilarity of identical strings is 1.0") { + assert(ModelEquality.jaccardSimilarity("hello", "hello") === 1.0) + } + + test("jaccardSimilarity of strings sharing nothing is 0.0") { + assert(ModelEquality.jaccardSimilarity("abcd", "wxyz") === 0.0) + } + + test("jaccardSimilarity is bounded to [0, 1]") { + Seq(("abc", "def"), ("hello", "hello"), ("kitten", "sitting"), ("a", "ab")).foreach { + case (s1, s2) => + val score = ModelEquality.jaccardSimilarity(s1, s2) + assert(score >= 0.0 && score <= 1.0, s"$s1 vs $s2 produced $score") + } + } + + test("jaccardSimilarity is symmetric") { + Seq(("abc", "def"), ("kitten", "sitting"), ("hello", "hello")).foreach { case (s1, s2) => + assert(ModelEquality.jaccardSimilarity(s1, s2) === ModelEquality.jaccardSimilarity(s2, s1)) + } + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/core/utils/VerifyOsUtils.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/core/utils/VerifyOsUtils.scala new file mode 100644 index 00000000000..3543a4ff55c --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/core/utils/VerifyOsUtils.scala @@ -0,0 +1,15 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.core.utils + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase + +class VerifyOsUtils extends TestBase { + + test("IsWindows returns a boolean based on os.name property") { + val osName = System.getProperty("os.name").toLowerCase() + val expected = osName.indexOf("win") >= 0 + assert(OsUtils.IsWindows === expected) + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/explainers/VerifyExplainerSharedParams.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/explainers/VerifyExplainerSharedParams.scala new file mode 100644 index 00000000000..b5b68cb905a --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/explainers/VerifyExplainerSharedParams.scala @@ -0,0 +1,136 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.explainers + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.spark.ml.param.{ParamMap, Params} +import org.apache.spark.ml.util.Identifiable + +class VerifyExplainerSharedParams extends TestBase { + + // Test implementation that mixes in all the traits + private class TestExplainerParams(override val uid: String) + extends Params + with HasMetricsCol + with HasNumSamples + with HasTokensCol + with HasSuperpixelCol + with HasSamplingFraction + with HasExplainTarget { + override def copy(extra: ParamMap): Params = this + } + + test("HasMetricsCol sets and gets metrics column") { + val params = new TestExplainerParams(Identifiable.randomUID("test")) + params.setMetricsCol("metrics_output") + assert(params.getMetricsCol === "metrics_output") + } + + test("HasMetricsCol param has correct name and doc") { + val params = new TestExplainerParams(Identifiable.randomUID("test")) + assert(params.metricsCol.name === "metricsCol") + assert(params.metricsCol.doc.contains("fitting metrics")) + } + + test("HasNumSamples sets and gets number of samples") { + val params = new TestExplainerParams(Identifiable.randomUID("test")) + params.setNumSamples(100) + assert(params.getNumSamples === 100) + } + + test("HasNumSamples getNumSamplesOpt returns Some when set") { + val params = new TestExplainerParams(Identifiable.randomUID("test")) + params.setNumSamples(50) + assert(params.getNumSamplesOpt === Some(50)) + } + + test("HasNumSamples getNumSamplesOpt returns None when not set") { + val params = new TestExplainerParams(Identifiable.randomUID("test")) + assert(params.getNumSamplesOpt.isEmpty) + } + + test("HasNumSamples validates numSamples must be positive") { + val params = new TestExplainerParams(Identifiable.randomUID("test")) + assertThrows[IllegalArgumentException] { + params.setNumSamples(0) + } + assertThrows[IllegalArgumentException] { + params.setNumSamples(-1) + } + } + + test("HasTokensCol sets and gets tokens column") { + val params = new TestExplainerParams(Identifiable.randomUID("test")) + params.setTokensCol("tokens") + assert(params.getTokensCol === "tokens") + } + + test("HasTokensCol param has correct name and doc") { + val params = new TestExplainerParams(Identifiable.randomUID("test")) + assert(params.tokensCol.name === "tokensCol") + assert(params.tokensCol.doc.contains("tokens")) + } + + test("HasSuperpixelCol sets and gets superpixel column") { + val params = new TestExplainerParams(Identifiable.randomUID("test")) + params.setSuperpixelCol("superpixels") + assert(params.getSuperpixelCol === "superpixels") + } + + test("HasSuperpixelCol param has correct name and doc") { + val params = new TestExplainerParams(Identifiable.randomUID("test")) + assert(params.superpixelCol.name === "superpixelCol") + assert(params.superpixelCol.doc.contains("superpixel")) + } + + test("HasSamplingFraction sets and gets sampling fraction") { + val params = new TestExplainerParams(Identifiable.randomUID("test")) + params.setSamplingFraction(0.5) + assert(params.getSamplingFraction === 0.5) + } + + test("HasSamplingFraction validates fraction in range 0 to 1") { + val params = new TestExplainerParams(Identifiable.randomUID("test")) + // Valid values + params.setSamplingFraction(0.0) + params.setSamplingFraction(1.0) + params.setSamplingFraction(0.5) + + // Invalid values + assertThrows[IllegalArgumentException] { + params.setSamplingFraction(-0.1) + } + assertThrows[IllegalArgumentException] { + params.setSamplingFraction(1.1) + } + } + + test("HasExplainTarget has default targetCol value") { + val params = new TestExplainerParams(Identifiable.randomUID("test")) + assert(params.getTargetCol === "probability") + } + + test("HasExplainTarget sets and gets targetCol") { + val params = new TestExplainerParams(Identifiable.randomUID("test")) + params.setTargetCol("prediction") + assert(params.getTargetCol === "prediction") + } + + test("HasExplainTarget has default empty targetClasses") { + val params = new TestExplainerParams(Identifiable.randomUID("test")) + assert(params.getTargetClasses.isEmpty) + } + + test("HasExplainTarget sets and gets targetClasses") { + val params = new TestExplainerParams(Identifiable.randomUID("test")) + params.setTargetClasses(Array(0, 1, 2)) + assert(params.getTargetClasses === Array(0, 1, 2)) + } + + test("HasExplainTarget sets and gets targetClassesCol") { + val params = new TestExplainerParams(Identifiable.randomUID("test")) + params.setTargetClassesCol("classes") + assert(params.getTargetClassesCol === "classes") + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/explainers/VerifyRowUtils.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/explainers/VerifyRowUtils.scala new file mode 100644 index 00000000000..691242833d2 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/explainers/VerifyRowUtils.scala @@ -0,0 +1,100 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.explainers + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.spark.sql.Row +import org.apache.spark.sql.types._ + +class VerifyRowUtils extends TestBase { + + import RowUtils.RowCanGetAsDouble + + test("getAsDouble converts Byte to Double") { + val schema = StructType(Seq(StructField("value", ByteType))) + val row = Row(10.toByte) + val result = row.getAsDouble(0) + assert(result === 10.0) + } + + test("getAsDouble converts Short to Double") { + val schema = StructType(Seq(StructField("value", ShortType))) + val row = Row(100.toShort) + val result = row.getAsDouble(0) + assert(result === 100.0) + } + + test("getAsDouble converts Int to Double") { + val schema = StructType(Seq(StructField("value", IntegerType))) + val row = Row(42) + val result = row.getAsDouble(0) + assert(result === 42.0) + } + + test("getAsDouble converts Long to Double") { + val schema = StructType(Seq(StructField("value", LongType))) + val row = Row(1000L) + val result = row.getAsDouble(0) + assert(result === 1000.0) + } + + test("getAsDouble converts Float to Double") { + val schema = StructType(Seq(StructField("value", FloatType))) + val row = Row(3.14f) + val result = row.getAsDouble(0) + assert(Math.abs(result - 3.14) < 0.001) + } + + test("getAsDouble returns Double unchanged") { + val schema = StructType(Seq(StructField("value", DoubleType))) + val row = Row(2.718) + val result = row.getAsDouble(0) + assert(result === 2.718) + } + + test("getAsDouble by column name") { + val schema = StructType(Seq( + StructField("other", StringType), + StructField("value", IntegerType) + )) + // Create a row that knows about its schema + val row = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema( + Array("test", 42), + schema + ) + val result = row.getAsDouble("value") + assert(result === 42.0) + } + + test("getAsDouble throws for unsupported types") { + val schema = StructType(Seq(StructField("value", StringType))) + val row = Row("not a number") + assertThrows[Exception] { + row.getAsDouble(0) + } + } + + test("getAsDouble handles negative numbers") { + val schema = StructType(Seq(StructField("value", IntegerType))) + val row = Row(-100) + val result = row.getAsDouble(0) + assert(result === -100.0) + } + + test("getAsDouble handles zero") { + val schema = StructType(Seq(StructField("value", IntegerType))) + val row = Row(0) + val result = row.getAsDouble(0) + assert(result === 0.0) + } + + test("getAsDouble handles large Long values") { + val schema = StructType(Seq(StructField("value", LongType))) + val largeValue = Long.MaxValue / 2 + val row = Row(largeValue) + val result = row.getAsDouble(0) + // Note: some precision loss expected for very large longs + assert(result > 0) + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/exploratory/DataBalanceTestBase.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/exploratory/DataBalanceTestBase.scala index df70cb319fe..fca47e08f32 100644 --- a/core/src/test/scala/com/microsoft/azure/synapse/ml/exploratory/DataBalanceTestBase.scala +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/exploratory/DataBalanceTestBase.scala @@ -121,7 +121,8 @@ case class DistributionMetricsCalculator(refFeatureProbabilities: Array[Double], val averageObsRef = (obsFeatureProbabilities, refFeatureProbabilities).zipped.map((a, b) => (a + b) / 2d).toArray val entropyRefAvg = entropy(refFeatureProbabilities, Some(averageObsRef)) val entropyObsAvg = entropy(obsFeatureProbabilities, Some(averageObsRef)) - sqrt((entropyRefAvg + entropyObsAvg) / 2d) + val jsDivergenceBase2 = (entropyRefAvg + entropyObsAvg) / (2d * log(2d)) + sqrt(math.max(0d, jsDivergenceBase2)) } val infNormDistance: Double = absDiffObsRef.max val totalVariationDistance: Double = 0.5d * absDiffObsRef.sum diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/exploratory/DistributionBalanceMeasureSuite.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/exploratory/DistributionBalanceMeasureSuite.scala index 410a429b5df..7296cda0ce0 100644 --- a/core/src/test/scala/com/microsoft/azure/synapse/ml/exploratory/DistributionBalanceMeasureSuite.scala +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/exploratory/DistributionBalanceMeasureSuite.scala @@ -5,7 +5,7 @@ package com.microsoft.azure.synapse.ml.exploratory import com.microsoft.azure.synapse.ml.core.test.fuzzing.{TestObject, TransformerFuzzing} import org.apache.spark.ml.util.MLReadable -import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.{DataFrame, Row} import org.apache.spark.sql.functions.{array, col} class DistributionBalanceMeasureSuite extends DataBalanceTestBase with TransformerFuzzing[DistributionBalanceMeasure] { @@ -19,6 +19,8 @@ class DistributionBalanceMeasureSuite extends DataBalanceTestBase with Transform import DistributionMetrics._ import spark.implicits._ + private val jsDistanceTolerance = 1e-12 + private def distributionBalanceMeasure: DistributionBalanceMeasure = new DistributionBalanceMeasure() .setSensitiveCols(features) @@ -36,6 +38,94 @@ class DistributionBalanceMeasureSuite extends DataBalanceTestBase with Transform .setVerbose(true) .transform(sensitiveFeaturesDf) + private def jsDistance(observed: Seq[Double], reference: Seq[Double]): Double = { + require(observed.nonEmpty) + require(observed.length == reference.length) + Seq(observed, reference).foreach { distribution => + require(distribution.forall(probability => java.lang.Double.isFinite(probability) && probability >= 0d)) + require(math.abs(distribution.sum - 1d) <= jsDistanceTolerance) + } + val observedProbabilityCol = "observedProbability" + val referenceProbabilityCol = "referenceProbability" + val observedCountCol = "observedCount" + val referenceCountCol = "referenceCount" + val unusedCount = 0d + val probabilities = observed.zip(reference) + .map { case (observedProbability, referenceProbability) => + (observedProbability, referenceProbability, unusedCount, unusedCount) + } + .toDF(observedProbabilityCol, referenceProbabilityCol, observedCountCol, referenceCountCol) + val metrics = DistributionMetrics( + observed.length, + observedProbabilityCol, + observedCountCol, + referenceProbabilityCol, + referenceCountCol) + + probabilities.agg(metrics.jsDistance.alias(JSDISTANCE)).head().getAs[Double](JSDISTANCE) + } + + private def assertJsDistance(actual: Double, expected: Double): Unit = { + assert(!actual.isNaN) + assert(math.abs(actual - expected) <= jsDistanceTolerance) + } + + test("Jensen-Shannon distance has normalized endpoints and handles zero-probability support") { + assertJsDistance(jsDistance(Seq(0.5d, 0.5d), Seq(0.5d, 0.5d)), 0d) + assertJsDistance(jsDistance(Seq(1d, 0d), Seq(0d, 1d)), 1d) + assertJsDistance( + jsDistance(Seq(0.5d, 0.5d, 0d), Seq(0.5d, 0d, 0.5d)), + math.sqrt(0.5d)) + assertJsDistance( + jsDistance(Seq(0.9999999d, 0.0000001d), Seq(0.0000001d, 0.9999999d)), + 0.999998765189656d) + assertJsDistance( + jsDistance( + Seq(0.36096096457437404d, 0.0463302746586729d, 0.592708760766953d), + Seq(0.36096096457437415d, 0.04633027465867293d, 0.5927087607669529d)), + 0d) + } + + test("Jensen-Shannon distance is symmetric for an intermediate distribution") { + val observed = Seq(0.7d, 0.2d, 0.1d) + val reference = Seq(0.1d, 0.3d, 0.6d) + val expected = 0.5768458213445598 + val forward = jsDistance(observed, reference) + + assertJsDistance(forward, expected) + assertJsDistance(jsDistance(reference, observed), expected) + assertJsDistance(jsDistance(observed.reverse, reference.reverse), expected) + } + + test("Jensen-Shannon distance stays within its documented bounds") { + val distributions = Seq( + (Seq(1d, 0d), Seq(0d, 1d)), + (Seq(0.75d, 0.25d), Seq(0.25d, 0.75d)), + (Seq(0.5d, 0.5d), Seq(1d, 0d))) + + distributions.foreach { case (observed, reference) => + val distance = jsDistance(observed, reference) + assert(!distance.isNaN) + assert(distance >= -jsDistanceTolerance && distance <= 1d + jsDistanceTolerance) + } + } + + test("DistributionBalanceMeasure exposes normalized JS distance end-to-end without changing its schema") { + val source = Seq("red", "red", "red", "blue").toDF("color") + val measure = new DistributionBalanceMeasure() + .setSensitiveCols(Array("color")) + .setReferenceDistribution(Array(Map("red" -> 0.25d, "blue" -> 0.75d))) + val result = measure.transform(source) + val rows = result.collect() + + assert(result.schema === measure.transformSchema(source.schema)) + assert(rows.length === 1) + assert(rows.head.getAs[String]("FeatureName") === "color") + assertJsDistance( + rows.head.getAs[Row]("DistributionBalanceMeasure").getAs[Double](JSDISTANCE), + 0.4344213111034807) + } + private def actualFeature1: Map[String, Double] = (METRICS zip actual.filter(col("FeatureName") === feature1) .select(array(col("DistributionBalanceMeasure.*"))) @@ -53,7 +143,7 @@ class DistributionBalanceMeasureSuite extends DataBalanceTestBase with Transform // val (refProbs, refCounts) = Array.fill(numFeatures.toInt)(numFeatures).map(n => (1d / n, numRows / n)).unzip // val CALC = DistributionMetricsCalculator(refProbs, refCounts, obsProbs, obsCounts, numFeatures) val KLDIVERGENCE = 0.03775534151008829 - val JSDISTANCE = 0.09785224086736323 + val JSDISTANCE = 0.11753251925575922 val INFNORMDISTANCE = 0.1111111111111111 val TOTALVARIATIONDISTANCE = 0.1111111111111111 val WASSERSTEINDISTANCE = 0.07407407407407407 @@ -61,11 +151,11 @@ class DistributionBalanceMeasureSuite extends DataBalanceTestBase with Transform val CHISQUAREDPVALUE = 0.7165313105737893 } - test(s"DistributionBalanceMeasure can calculate Distribution Balance Measures for $feature1") { + test(s"DistributionBalanceMeasure normalizes JS distance without changing other measures for $feature1") { val actual = actualFeature1 val expected = ExpectedFeature1 assert(actual(KLDIVERGENCE) === expected.KLDIVERGENCE) - assert(actual(JSDISTANCE) === expected.JSDISTANCE) + assertJsDistance(actual(JSDISTANCE), expected.JSDISTANCE) assert(actual(INFNORMDISTANCE) === expected.INFNORMDISTANCE) assert(actual(TOTALVARIATIONDISTANCE) === expected.TOTALVARIATIONDISTANCE) assert(actual(WASSERSTEINDISTANCE) === expected.WASSERSTEINDISTANCE) @@ -90,7 +180,7 @@ class DistributionBalanceMeasureSuite extends DataBalanceTestBase with Transform // val (refProbs, refCounts) = Array.fill(numFeatures.toInt)(numFeatures).map(n => (1d / n, numRows / n)).unzip // val CALC = DistributionMetricsCalculator(refProbs, refCounts, obsProbs, obsCounts, numFeatures) val KLDIVERGENCE = 0.07551068302017659 - val JSDISTANCE = 0.14172745151398888 + val JSDISTANCE = 0.1702320179536471 val INFNORMDISTANCE = 0.1388888888888889 val TOTALVARIATIONDISTANCE = 0.16666666666666666 val WASSERSTEINDISTANCE = 0.08333333333333333 @@ -102,7 +192,7 @@ class DistributionBalanceMeasureSuite extends DataBalanceTestBase with Transform val actual = actualFeature2 val expected = ExpectedFeature2 assert(actual(KLDIVERGENCE) === expected.KLDIVERGENCE) - assert(actual(JSDISTANCE) === expected.JSDISTANCE) + assertJsDistance(actual(JSDISTANCE), expected.JSDISTANCE) assert(actual(INFNORMDISTANCE) === expected.INFNORMDISTANCE) assert(actual(TOTALVARIATIONDISTANCE) === expected.TOTALVARIATIONDISTANCE) assert(actual(WASSERSTEINDISTANCE) === expected.WASSERSTEINDISTANCE) @@ -182,7 +272,7 @@ class DistributionBalanceMeasureSuite extends DataBalanceTestBase with Transform // val refCounts = refProbs.map(_ * numRows) // val CALC = DistributionMetricsCalculator(refProbs, refCounts, obsProbs, obsCounts, numFeatures) val KLDIVERGENCE = 0.09399792940857671 - val JSDISTANCE = 0.15001917759832653 + val JSDISTANCE = 0.18019139596106415 val INFNORMDISTANCE = 0.19444444444444442 val TOTALVARIATIONDISTANCE = 0.19444444444444445 val WASSERSTEINDISTANCE = 0.12962962962962962 @@ -196,7 +286,7 @@ class DistributionBalanceMeasureSuite extends DataBalanceTestBase with Transform val actual = actualCustomDistFeature1 val expected = ExpectedCustomDistFeature1 assert(actual(KLDIVERGENCE) === expected.KLDIVERGENCE) - assert(actual(JSDISTANCE) === expected.JSDISTANCE) + assertJsDistance(actual(JSDISTANCE), expected.JSDISTANCE) assert(actual(INFNORMDISTANCE) === expected.INFNORMDISTANCE) assert(actual(TOTALVARIATIONDISTANCE) === expected.TOTALVARIATIONDISTANCE) assert(actual(WASSERSTEINDISTANCE) === expected.WASSERSTEINDISTANCE) @@ -222,7 +312,7 @@ class DistributionBalanceMeasureSuite extends DataBalanceTestBase with Transform // val refCounts = refProbs.map(_ * numRows) // val CALC = DistributionMetricsCalculator(refProbs, refCounts, obsProbs, obsCounts, numFeatures) val KLDIVERGENCE = Double.PositiveInfinity - val JSDISTANCE = 0.2100032735609124 + val JSDISTANCE = 0.25223963779252284 val INFNORMDISTANCE = 0.1111111111111111 val TOTALVARIATIONDISTANCE = 0.1111111111111111 val WASSERSTEINDISTANCE = 0.05555555555555555 @@ -237,7 +327,7 @@ class DistributionBalanceMeasureSuite extends DataBalanceTestBase with Transform val actual = actualCustomDistFeature2 val expected = ExpectedCustomDistFeature2 assert(actual(KLDIVERGENCE) === expected.KLDIVERGENCE) - assert(actual(JSDISTANCE) === expected.JSDISTANCE) + assertJsDistance(actual(JSDISTANCE), expected.JSDISTANCE) assert(actual(INFNORMDISTANCE) === expected.INFNORMDISTANCE) assert(actual(TOTALVARIATIONDISTANCE) === expected.TOTALVARIATIONDISTANCE) assert(actual(WASSERSTEINDISTANCE) === expected.WASSERSTEINDISTANCE) diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/fabric/FabricArtifactNamesSuite.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/fabric/FabricArtifactNamesSuite.scala new file mode 100644 index 00000000000..3f27176d5ed --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/fabric/FabricArtifactNamesSuite.scala @@ -0,0 +1,36 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.fabric + +import org.scalatest.funsuite.AnyFunSuite + +import java.time.LocalDateTime +import java.util.UUID + +class FabricArtifactNamesSuite extends AnyFunSuite { + private val testTime = LocalDateTime.of(2026, 8, 9, 2, 18, 35) + private val testId = UUID.fromString("01234567-89ab-cdef-0123-456789abcdef") + private val otherTestId = UUID.fromString("fedcba98-7654-3210-fedc-ba9876543210") + + test("Add a unique suffix to Spark Job Definition names") { + assert( + FabricArtifactNames.sjd("TestNotebook", testTime, testId) == + "TestNotebook-20260809-02-18-35-0123456789abcdef0123456789abcdef") + } + + test("Add a unique suffix to store artifact names") { + assert( + FabricArtifactNames.store("Lakehouse", testTime, testId) == + "Lakehouse202608090218350123456789abcdef0123456789abcdef") + } + + test("Distinguish artifacts created with the same timestamp") { + assert( + FabricArtifactNames.sjd("TestNotebook", testTime, testId) != + FabricArtifactNames.sjd("TestNotebook", testTime, otherTestId)) + assert( + FabricArtifactNames.store("Lakehouse", testTime, testId) != + FabricArtifactNames.store("Lakehouse", testTime, otherTestId)) + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/fabric/FabricOperations.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/fabric/FabricOperations.scala index 477db97d493..9e7335f4c4b 100644 --- a/core/src/test/scala/com/microsoft/azure/synapse/ml/fabric/FabricOperations.scala +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/fabric/FabricOperations.scala @@ -26,11 +26,31 @@ import java.net.URLEncoder import java.nio.file.{Files, Path} import java.time.LocalDateTime import java.time.format.DateTimeFormatter +import java.util.UUID import scala.annotation.tailrec import scala.concurrent.{ExecutionContext, Future, TimeoutException, blocking} import scala.util.Try import scala.util.control.Breaks.{break, breakable} +private[fabric] object FabricArtifactNames { + private val SjdTimestampFormat = DateTimeFormatter.ofPattern("yyyyMMdd-HH-mm-ss") + private val StoreTimestampFormat = DateTimeFormatter.ofPattern("yyyyMMddHHmmss") + + def sjd(runName: String): String = + sjd(runName, LocalDateTime.now(), UUID.randomUUID()) + + def store(storeName: String): String = + store(storeName, LocalDateTime.now(), UUID.randomUUID()) + + private[fabric] def sjd(runName: String, now: LocalDateTime, uniqueId: UUID): String = + s"$runName-${SjdTimestampFormat.format(now)}-${compact(uniqueId)}" + + private[fabric] def store(storeName: String, now: LocalDateTime, uniqueId: UUID): String = + s"$storeName${StoreTimestampFormat.format(now)}${compact(uniqueId)}" + + private def compact(uniqueId: UUID): String = uniqueId.toString.replace("-", "") +} + private[fabric] class FabricOperations(clientId: String, redirectUri: String, workspaceId: String) extends FabricInternalConnection(clientId, redirectUri, workspaceId) { @@ -95,14 +115,12 @@ private[fabric] class FabricOperations(clientId: String, redirectUri: String, wo def createSJDArtifact(path: String, artifactType: String): String = { val runName = getBlobNameFromFilepath(path).replace(".py", "") - - val dtf = DateTimeFormatter.ofPattern("yyyyMMdd-HH-mm-ss") - val now = dtf.format(LocalDateTime.now) + val displayName = FabricArtifactNames.sjd(runName) val reqBody: String = s""" |{ - | "displayName": "$runName-$now", + | "displayName": "$displayName", | "description": "Synapse Spark Job Definition $artifactType", | "artifactType": "$artifactType" |} @@ -114,13 +132,12 @@ private[fabric] class FabricOperations(clientId: String, redirectUri: String, wo def createStoreArtifact(): String = { val store = Secrets.ArtifactStore.capitalize - val dtf = DateTimeFormatter.ofPattern("yyyyMMddHHmmss") - val now = dtf.format(LocalDateTime.now) + val displayName = FabricArtifactNames.store(store) val reqBody: String = s""" |{ - | "displayName": "$store$now", + | "displayName": "$displayName", | "description": "SynapseML Test Infra $store", | "artifactType": "$store" |} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/io/binary/VerifyBinaryFileFormat.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/io/binary/VerifyBinaryFileFormat.scala new file mode 100644 index 00000000000..a28987e9324 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/io/binary/VerifyBinaryFileFormat.scala @@ -0,0 +1,123 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.io.binary + +import com.microsoft.azure.synapse.ml.core.schema.BinaryFileSchema +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.hadoop.fs.Path + +import java.io.File +import java.nio.file.Files + +class VerifyBinaryFileFormat extends TestBase { + + test("BinaryFileFormat shortName returns 'binary'") { + val format = new BinaryFileFormat() + assert(format.shortName() === "binary") + } + + test("BinaryFileFormat toString returns 'Binary'") { + val format = new BinaryFileFormat() + assert(format.toString === "Binary") + } + + test("BinaryFileFormat isSplitable returns false") { + val format = new BinaryFileFormat() + val result = format.isSplitable(spark, Map.empty, new Path("/test")) + assert(!result) + } + + test("BinaryFileFormat inferSchema returns BinaryFileSchema") { + val format = new BinaryFileFormat() + val schema = format.inferSchema(spark, Map.empty, Seq.empty) + assert(schema.isDefined) + assert(schema.get === BinaryFileSchema.Schema) + } + + test("BinaryFileFormat equals returns true for same type") { + val format1 = new BinaryFileFormat() + val format2 = new BinaryFileFormat() + assert(format1.equals(format2)) + } + + test("BinaryFileFormat equals returns false for different type") { + val format = new BinaryFileFormat() + assert(!format.equals("not a format")) + } + + test("BinaryFileFormat hashCode is consistent") { + val format1 = new BinaryFileFormat() + val format2 = new BinaryFileFormat() + assert(format1.hashCode() === format2.hashCode()) + } + + test("ConfUtils.getHConf returns SerializableConfiguration") { + import spark.implicits._ + val df = Seq(1, 2, 3).toDF("num") + val hConf = ConfUtils.getHConf(df) + assert(hConf != null) + } + + test("BinaryFileFormat can read binary files") { + // Create a temp directory with binary files + val tempDir = Files.createTempDirectory("binary-test").toFile + tempDir.deleteOnExit() + + val testFile = new File(tempDir, "test.bin") + Files.write(testFile.toPath, Array[Byte](1, 2, 3, 4, 5)) + testFile.deleteOnExit() + + val df = spark.read.format(classOf[BinaryFileFormat].getName).load(tempDir.getAbsolutePath) + assert(df.count() >= 1) + assert(df.schema === BinaryFileSchema.Schema) + } + + test("BinaryFileFormat reads file content correctly") { + val tempDir = Files.createTempDirectory("binary-content-test").toFile + tempDir.deleteOnExit() + + val testContent = "Hello, Binary!".getBytes + val testFile = new File(tempDir, "content.bin") + Files.write(testFile.toPath, testContent) + testFile.deleteOnExit() + + val df = spark.read.format(classOf[BinaryFileFormat].getName).load(tempDir.getAbsolutePath) + val row = df.collect().head + val struct = row.getStruct(0) + val bytes = struct.getAs[Array[Byte]]("bytes") + assert(bytes.sameElements(testContent)) + } + + test("BinaryFileFormat respects subsample option") { + val tempDir = Files.createTempDirectory("binary-subsample-test").toFile + tempDir.deleteOnExit() + + // Create multiple files + for (i <- 1 to 10) { + val testFile = new File(tempDir, s"file$i.bin") + Files.write(testFile.toPath, Array[Byte](i.toByte)) + testFile.deleteOnExit() + } + + // Read with subsample=0.0 should return fewer or no rows + val df = spark.read.format(classOf[BinaryFileFormat].getName) + .option("subsample", "0.0") + .load(tempDir.getAbsolutePath) + assert(df.count() === 0) + } + + test("BinaryFileFormat reads multiple files") { + val tempDir = Files.createTempDirectory("binary-multi-test").toFile + tempDir.deleteOnExit() + + for (i <- 1 to 3) { + val testFile = new File(tempDir, s"multi$i.bin") + Files.write(testFile.toPath, s"content$i".getBytes) + testFile.deleteOnExit() + } + + val df = spark.read.format(classOf[BinaryFileFormat].getName).load(tempDir.getAbsolutePath) + assert(df.count() === 3) + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/io/http/VerifyClients.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/io/http/VerifyClients.scala new file mode 100644 index 00000000000..aab8ffb099a --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/io/http/VerifyClients.scala @@ -0,0 +1,178 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.io.http + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase + +import scala.concurrent.ExecutionContext +import scala.concurrent.duration._ + +class VerifyClients extends TestBase { + + // Test SingleThreadedClient + private class TestSingleThreadedClient extends SingleThreadedClient { + override protected type Client = Unit + override protected type ResponseType = String + override protected type RequestType = String + override protected val internalClient: Client = () + + override protected def sendRequestWithContext( + request: RequestWithContext): ResponseWithContext = { + request.request match { + case Some(req) => ResponseWithContext(Some(s"response-$req"), request.context) + case None => ResponseWithContext(None, request.context) + } + } + } + + test("SingleThreadedClient processes requests sequentially") { + val client = new TestSingleThreadedClient + val requests = Iterator( + client.RequestWithContext(Some("a"), Some("ctx-a")), + client.RequestWithContext(Some("b"), Some("ctx-b")), + client.RequestWithContext(Some("c"), Some("ctx-c")) + ) + + val responses = client.sendRequestsWithContext(requests).toList + assert(responses.length === 3) + assert(responses(0).response === Some("response-a")) + assert(responses(0).context === Some("ctx-a")) + assert(responses(1).response === Some("response-b")) + assert(responses(2).response === Some("response-c")) + } + + test("SingleThreadedClient handles empty requests") { + val client = new TestSingleThreadedClient + val requests = Iterator( + client.RequestWithContext(None, Some("ctx")) + ) + + val responses = client.sendRequestsWithContext(requests).toList + assert(responses.length === 1) + assert(responses.head.response === None) + assert(responses.head.context === Some("ctx")) + } + + test("SingleThreadedClient handles empty iterator") { + val client = new TestSingleThreadedClient + val requests = Iterator.empty.asInstanceOf[Iterator[client.RequestWithContext]] + val responses = client.sendRequestsWithContext(requests).toList + assert(responses.isEmpty) + } + + // Test AsyncClient + private class TestAsyncClient(conc: Int, to: Duration) + (implicit ec: ExecutionContext) + extends AsyncClient(conc, to) { + + override protected type Client = Unit + override protected type ResponseType = String + override protected type RequestType = String + override protected val internalClient: Client = () + + override protected def sendRequestWithContext( + request: RequestWithContext): ResponseWithContext = { + request.request match { + case Some(req) => + // Simulate some work + Thread.sleep(10) + ResponseWithContext(Some(s"async-$req"), request.context) + case None => + ResponseWithContext(None, request.context) + } + } + } + + test("AsyncClient processes requests concurrently") { + implicit val ec: ExecutionContext = ExecutionContext.global + val client = new TestAsyncClient(4, 30.seconds) + + val requests = Iterator( + client.RequestWithContext(Some("1"), None), + client.RequestWithContext(Some("2"), None), + client.RequestWithContext(Some("3"), None), + client.RequestWithContext(Some("4"), None) + ) + + val responses = client.sendRequestsWithContext(requests).toList + assert(responses.length === 4) + assert(responses.map(_.response).forall(_.isDefined)) + assert(responses.flatMap(_.response).toSet === Set("async-1", "async-2", "async-3", "async-4")) + } + + test("AsyncClient preserves context") { + implicit val ec: ExecutionContext = ExecutionContext.global + val client = new TestAsyncClient(2, 30.seconds) + + val requests = Iterator( + client.RequestWithContext(Some("a"), Some("context-a")), + client.RequestWithContext(Some("b"), Some("context-b")) + ) + + val responses = client.sendRequestsWithContext(requests).toList + assert(responses.length === 2) + // Note: order may vary due to concurrency, but contexts should match requests + responses.foreach { resp => + resp.response match { + case Some("async-a") => assert(resp.context === Some("context-a")) + case Some("async-b") => assert(resp.context === Some("context-b")) + case _ => fail("Unexpected response") + } + } + } + + test("AsyncClient handles empty requests") { + implicit val ec: ExecutionContext = ExecutionContext.global + val client = new TestAsyncClient(2, 30.seconds) + + val requests = Iterator( + client.RequestWithContext(None, Some("ctx")) + ) + + val responses = client.sendRequestsWithContext(requests).toList + assert(responses.length === 1) + assert(responses.head.response === None) + } + + test("AsyncClient respects concurrency parameter") { + implicit val ec: ExecutionContext = ExecutionContext.global + val client = new TestAsyncClient(2, 30.seconds) + assert(client.concurrency === 2) + } + + test("AsyncClient respects timeout parameter") { + implicit val ec: ExecutionContext = ExecutionContext.global + val client = new TestAsyncClient(2, 45.seconds) + assert(client.timeout === 45.seconds) + } + + // Test RequestWithContext and ResponseWithContext + test("RequestWithContext can be created with request only") { + val client = new TestSingleThreadedClient + val req = new client.RequestWithContext(Some("test")) + assert(req.request === Some("test")) + assert(req.context === None) + } + + test("RequestWithContext can be created with request and context") { + val client = new TestSingleThreadedClient + val req = client.RequestWithContext(Some("test"), Some("ctx")) + assert(req.request === Some("test")) + assert(req.context === Some("ctx")) + } + + test("ResponseWithContext can be created with response only") { + val client = new TestSingleThreadedClient + val resp = new client.ResponseWithContext(Some("test")) + assert(resp.response === Some("test")) + assert(resp.context === None) + } + + test("ResponseWithContext can be created with response and context") { + val client = new TestSingleThreadedClient + val resp = client.ResponseWithContext(Some("test"), Some("ctx")) + assert(resp.response === Some("test")) + assert(resp.context === Some("ctx")) + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/io/http/VerifyHTTPSchema.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/io/http/VerifyHTTPSchema.scala new file mode 100644 index 00000000000..5cd1c3843d9 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/io/http/VerifyHTTPSchema.scala @@ -0,0 +1,179 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.io.http + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.http.ProtocolVersion +import org.apache.http.message.BasicHeader + +class VerifyHTTPSchema extends TestBase { + + test("HeaderData stores name and value") { + val header = HeaderData("Content-Type", "application/json") + assert(header.name === "Content-Type") + assert(header.value === "application/json") + } + + test("HeaderData.toHTTPCore creates BasicHeader") { + val header = HeaderData("Authorization", "Bearer token123") + val httpHeader = header.toHTTPCore + assert(httpHeader.getName === "Authorization") + assert(httpHeader.getValue === "Bearer token123") + } + + test("HeaderData can be created from HTTP Header") { + val basicHeader = new BasicHeader("X-Custom", "custom-value") + val headerData = new HeaderData(basicHeader) + assert(headerData.name === "X-Custom") + assert(headerData.value === "custom-value") + } + + test("ProtocolVersionData stores protocol info") { + val pvd = ProtocolVersionData("HTTP", 1, 1) + assert(pvd.protocol === "HTTP") + assert(pvd.major === 1) + assert(pvd.minor === 1) + } + + test("ProtocolVersionData.toHTTPCore creates ProtocolVersion") { + val pvd = ProtocolVersionData("HTTP", 2, 0) + val pv = pvd.toHTTPCore + assert(pv.getProtocol === "HTTP") + assert(pv.getMajor === 2) + assert(pv.getMinor === 0) + } + + test("ProtocolVersionData can be created from ProtocolVersion") { + val pv = new ProtocolVersion("HTTP", 1, 0) + val pvd = new ProtocolVersionData(pv) + assert(pvd.protocol === "HTTP") + assert(pvd.major === 1) + assert(pvd.minor === 0) + } + + test("StatusLineData stores status info") { + val pvd = ProtocolVersionData("HTTP", 1, 1) + val sld = StatusLineData(pvd, 200, "OK") + assert(sld.protocolVersion === pvd) + assert(sld.statusCode === 200) + assert(sld.reasonPhrase === "OK") + } + + test("RequestLineData stores request info") { + val pvd = Some(ProtocolVersionData("HTTP", 1, 1)) + val rld = RequestLineData("GET", "https://example.com", pvd) + assert(rld.method === "GET") + assert(rld.uri === "https://example.com") + assert(rld.protocolVersion === pvd) + } + + test("RequestLineData works without protocol version") { + val rld = RequestLineData("POST", "/api/data", None) + assert(rld.method === "POST") + assert(rld.uri === "/api/data") + assert(rld.protocolVersion.isEmpty) + } + + test("EntityData stores content info") { + val content = "test content".getBytes + val entity = EntityData( + content = content, + contentEncoding = None, + contentLength = Some(content.length.toLong), + contentType = Some(HeaderData("Content-Type", "text/plain")), + isChunked = false, + isRepeatable = true, + isStreaming = false + ) + assert(entity.content === content) + assert(entity.contentLength === Some(content.length.toLong)) + assert(entity.isChunked === false) + assert(entity.isRepeatable === true) + } + + test("HTTPResponseData stores response info") { + val pvd = ProtocolVersionData("HTTP", 1, 1) + val statusLine = StatusLineData(pvd, 200, "OK") + val response = HTTPResponseData( + headers = Array(HeaderData("Content-Type", "application/json")), + entity = None, + statusLine = statusLine, + locale = "en-US" + ) + assert(response.headers.length === 1) + assert(response.statusLine.statusCode === 200) + assert(response.locale === "en-US") + assert(response.entity.isEmpty) + } + + test("HTTPRequestData stores request info") { + val requestLine = RequestLineData("GET", "https://api.example.com/data", None) + val request = HTTPRequestData( + requestLine = requestLine, + headers = Array(HeaderData("Accept", "application/json")), + entity = None + ) + assert(request.requestLine.method === "GET") + assert(request.headers.length === 1) + assert(request.entity.isEmpty) + } + + test("HTTPRequestData with entity") { + val content = """{"key": "value"}""".getBytes + val entity = EntityData( + content = content, + contentEncoding = None, + contentLength = Some(content.length.toLong), + contentType = Some(HeaderData("Content-Type", "application/json")), + isChunked = false, + isRepeatable = true, + isStreaming = false + ) + val requestLine = RequestLineData("POST", "/api/submit", None) + val request = HTTPRequestData( + requestLine = requestLine, + headers = Array(), + entity = Some(entity) + ) + assert(request.entity.isDefined) + assert(request.entity.get.content === content) + } + + test("HTTPSchema.Response has schema") { + val schema = HTTPSchema.Response + assert(schema != null) + } + + test("HTTPSchema.Request has schema") { + val schema = HTTPSchema.Request + assert(schema != null) + } + + test("HTTPSchema.stringToResponse creates response") { + val response = HTTPSchema.stringToResponse("test body", 200, "OK") + assert(response.statusLine.statusCode === 200) + assert(response.statusLine.reasonPhrase === "OK") + assert(response.entity.isDefined) + } + + test("HTTPSchema.emptyResponse creates response without entity") { + val response = HTTPSchema.emptyResponse(404, "Not Found") + assert(response.statusLine.statusCode === 404) + assert(response.statusLine.reasonPhrase === "Not Found") + assert(response.entity.isEmpty) + } + + test("HTTPSchema.binaryToResponse creates response with binary content") { + val content = Array[Byte](1, 2, 3, 4, 5) + val response = HTTPSchema.binaryToResponse(content, 200, "OK") + assert(response.entity.isDefined) + assert(response.entity.get.content === content) + } + + test("HeaderValues.PlatformInfo returns a string") { + val platformInfo = HeaderValues.PlatformInfo + assert(platformInfo != null) + assert(platformInfo.nonEmpty) + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/io/http/VerifyRESTHelpers.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/io/http/VerifyRESTHelpers.scala new file mode 100644 index 00000000000..9deb68ff705 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/io/http/VerifyRESTHelpers.scala @@ -0,0 +1,49 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.io.http + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase + +// scalastyle:off magic.number +class VerifyRESTHelpers extends TestBase { + + test("retry succeeds on first try with empty backoffs") { + val result = RESTHelpers.retry(List.empty[Int], () => 42) + assert(result === 42) + } + + test("retry succeeds on first try with non-empty backoffs") { + val result = RESTHelpers.retry(List(0, 0), () => "ok") + assert(result === "ok") + } + + test("retry retries on failure and eventually succeeds") { + var attempts = 0 + val result = Console.withOut(new java.io.ByteArrayOutputStream()) { + // Zero backoffs exercise the same retry path without a real Thread.sleep. + RESTHelpers.retry(List(0, 0, 0), () => { + attempts += 1 + if (attempts < 3) throw new RuntimeException("fail") + "success" + }) + } + assert(result === "success") + assert(attempts === 3) + } + + test("retry throws when all retries exhausted") { + Console.withOut(new java.io.ByteArrayOutputStream()) { + intercept[RuntimeException] { + RESTHelpers.retry(List(0), () => throw new RuntimeException("always fails")) + } + } + } + + test("retry with empty backoff list throws immediately") { + intercept[RuntimeException] { + RESTHelpers.retry(List.empty[Int], () => throw new RuntimeException("immediate")) + } + } +} +// scalastyle:on magic.number diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/io/image/VerifyImageUtils.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/io/image/VerifyImageUtils.scala new file mode 100644 index 00000000000..f8dca6ce09f --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/io/image/VerifyImageUtils.scala @@ -0,0 +1,159 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.io.image + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase + +import java.awt.image.BufferedImage + +class VerifyImageUtils extends TestBase { + + test("channelsToType returns TYPE_BYTE_GRAY for 1 channel") { + assert(ImageUtils.channelsToType(1) === BufferedImage.TYPE_BYTE_GRAY) + } + + test("channelsToType returns TYPE_3BYTE_BGR for 3 channels") { + assert(ImageUtils.channelsToType(3) === BufferedImage.TYPE_3BYTE_BGR) + } + + test("channelsToType returns TYPE_4BYTE_ABGR for 4 channels") { + assert(ImageUtils.channelsToType(4) === BufferedImage.TYPE_4BYTE_ABGR) + } + + test("channelsToType throws for unsupported channel count") { + assertThrows[UnsupportedOperationException] { + ImageUtils.channelsToType(2) + } + assertThrows[UnsupportedOperationException] { + ImageUtils.channelsToType(5) + } + assertThrows[UnsupportedOperationException] { + ImageUtils.channelsToType(0) + } + } + + test("toBufferedImage creates image from byte array - grayscale") { + val width = 2 + val height = 2 + val nChannels = 1 + val bytes = Array[Byte](10, 20, 30, 40) + + val img = ImageUtils.toBufferedImage(bytes, width, height, nChannels) + + assert(img.getWidth === width) + assert(img.getHeight === height) + assert(img.getType === BufferedImage.TYPE_BYTE_GRAY) + } + + test("toBufferedImage creates image from byte array - RGB") { + val width = 2 + val height = 2 + val nChannels = 3 + // BGR format: 2x2 pixels = 12 bytes + val bytes = Array[Byte]( + 0, 0, 100.toByte, // pixel (0,0) + 0, 100.toByte, 0, // pixel (1,0) + 100.toByte, 0, 0, // pixel (0,1) + 50, 50, 50 // pixel (1,1) + ) + + val img = ImageUtils.toBufferedImage(bytes, width, height, nChannels) + + assert(img.getWidth === width) + assert(img.getHeight === height) + assert(img.getType === BufferedImage.TYPE_3BYTE_BGR) + } + + test("toBufferedImage creates image from byte array - RGBA") { + val width = 2 + val height = 1 + val nChannels = 4 + // ABGR format: 2x1 pixels = 8 bytes + val bytes = Array[Byte](0, 0, 100.toByte, -1, 100.toByte, 0, 0, -1) + + val img = ImageUtils.toBufferedImage(bytes, width, height, nChannels) + + assert(img.getWidth === width) + assert(img.getHeight === height) + assert(img.getType === BufferedImage.TYPE_4BYTE_ABGR) + } + + test("safeRead returns None for null bytes") { + // scalastyle:off null + val result = ImageUtils.safeRead(null) + // scalastyle:on null + assert(result.isEmpty) + } + + test("safeRead returns None for invalid image bytes") { + val invalidBytes = Array[Byte](1, 2, 3, 4, 5) + val result = ImageUtils.safeRead(invalidBytes) + assert(result.isEmpty) + } + + test("toSparkImage converts BufferedImage to Row format") { + val img = new BufferedImage(10, 10, BufferedImage.TYPE_3BYTE_BGR) + val row = ImageUtils.toSparkImage(img, Some("/path/to/image.jpg")) + + assert(row != null) + // The row should contain an inner row with image data + val innerRow = row.getAs[org.apache.spark.sql.Row](0) + assert(innerRow.getAs[Option[String]](0) === Some("/path/to/image.jpg")) + assert(innerRow.getAs[Int](1) === 10) // height + assert(innerRow.getAs[Int](2) === 10) // width + assert(innerRow.getAs[Int](3) === 3) // nChannels + } + + test("toSparkImage works without path") { + val img = new BufferedImage(5, 5, BufferedImage.TYPE_BYTE_GRAY) + val row = ImageUtils.toSparkImage(img, None) + + assert(row != null) + val innerRow = row.getAs[org.apache.spark.sql.Row](0) + assert(innerRow.getAs[Int](3) === 1) // grayscale = 1 channel + } + + test("toSparkImageTuple returns correct tuple for grayscale image") { + val img = new BufferedImage(4, 3, BufferedImage.TYPE_BYTE_GRAY) + val (path, height, width, nChannels, mode, decoded) = ImageUtils.toSparkImageTuple(img, Some("/test")) + + assert(path === Some("/test")) + assert(height === 3) + assert(width === 4) + assert(nChannels === 1) + assert(decoded.length === 4 * 3 * 1) // width * height * channels + } + + test("toSparkImageTuple returns correct tuple for RGB image") { + val img = new BufferedImage(4, 3, BufferedImage.TYPE_3BYTE_BGR) + val (path, height, width, nChannels, mode, decoded) = ImageUtils.toSparkImageTuple(img) + + assert(path === None) + assert(height === 3) + assert(width === 4) + assert(nChannels === 3) + assert(decoded.length === 4 * 3 * 3) + } + + test("toSparkImageTuple returns correct tuple for RGBA image") { + val img = new BufferedImage(2, 2, BufferedImage.TYPE_4BYTE_ABGR) + val (_, height, width, nChannels, _, decoded) = ImageUtils.toSparkImageTuple(img) + + assert(height === 2) + assert(width === 2) + assert(nChannels === 4) + assert(decoded.length === 2 * 2 * 4) + } + + test("roundtrip: toSparkImage then toBufferedImage preserves dimensions") { + val original = new BufferedImage(8, 6, BufferedImage.TYPE_3BYTE_BGR) + val sparkRow = ImageUtils.toSparkImage(original) + val innerRow = sparkRow.getAs[org.apache.spark.sql.Row](0) + + val reconstructed = ImageUtils.toBufferedImage(innerRow) + + assert(reconstructed.getWidth === original.getWidth) + assert(reconstructed.getHeight === original.getHeight) + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/logging/VerifyFeatureNames.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/logging/VerifyFeatureNames.scala new file mode 100644 index 00000000000..e3c3fba8065 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/logging/VerifyFeatureNames.scala @@ -0,0 +1,61 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.logging + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase + +class VerifyFeatureNames extends TestBase { + + test("AiServices constants have expected values") { + assert(FeatureNames.AiServices.Face === "aiservice-face") + assert(FeatureNames.AiServices.Form === "aiservice-form") + assert(FeatureNames.AiServices.Language === "aiservice-language") + assert(FeatureNames.AiServices.OpenAI === "aiservice-openai") + assert(FeatureNames.AiServices.Search === "aiservice-search") + assert(FeatureNames.AiServices.Speech === "aiservice-speech") + assert(FeatureNames.AiServices.Text === "aiservice-text") + assert(FeatureNames.AiServices.Translate === "aiservice-translate") + assert(FeatureNames.AiServices.Vision === "aiservice-vision") + } + + test("ML feature constants have expected values") { + assert(FeatureNames.AutoML === "automl") + assert(FeatureNames.Causal === "causal") + assert(FeatureNames.Explainers === "explainers") + assert(FeatureNames.Featurize === "featurize") + assert(FeatureNames.Geospatial === "geospatial") + assert(FeatureNames.Image === "image") + assert(FeatureNames.IsolationForest === "isolationforest") + assert(FeatureNames.NearestNeighbor === "nearestneighbor") + assert(FeatureNames.Recommendation === "recommendation") + } + + test("Deep learning and model feature constants have expected values") { + assert(FeatureNames.DeepLearning === "deeplearning") + assert(FeatureNames.OpenCV === "opencv") + assert(FeatureNames.LightGBM === "lightgbm") + assert(FeatureNames.VowpalWabbit === "vowpalwabbit") + } + + test("Core constant has expected value") { + assert(FeatureNames.Core === "core") + } + + test("All AiServices constants start with 'aiservice-' prefix") { + val aiServices = Seq( + FeatureNames.AiServices.Face, + FeatureNames.AiServices.Form, + FeatureNames.AiServices.Language, + FeatureNames.AiServices.OpenAI, + FeatureNames.AiServices.Search, + FeatureNames.AiServices.Speech, + FeatureNames.AiServices.Text, + FeatureNames.AiServices.Translate, + FeatureNames.AiServices.Vision + ) + aiServices.foreach { name => + assert(name.startsWith("aiservice-"), s"$name should start with 'aiservice-'") + } + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/logging/VerifySynapseMLLogging.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/logging/VerifySynapseMLLogging.scala new file mode 100644 index 00000000000..fdbf3f070e0 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/logging/VerifySynapseMLLogging.scala @@ -0,0 +1,186 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.logging + +import com.microsoft.azure.synapse.ml.build.BuildInfo +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.spark.sql.SparkSession + +class VerifySynapseMLLogging extends TestBase { + + test("RequiredLogFields stores uid, className, and method") { + val fields = RequiredLogFields("test-uid-123", "TestClass", "testMethod") + assert(fields.uid === "test-uid-123") + assert(fields.className === "TestClass") + assert(fields.method === "testMethod") + } + + test("RequiredLogFields.toMap contains all required fields") { + val fields = RequiredLogFields("uid1", "MyClass", "myMethod") + val map = fields.toMap + + assert(map("modelUid") === "uid1") + assert(map("className") === "MyClass") + assert(map("method") === "myMethod") + assert(map("libraryVersion") === BuildInfo.version) + assert(map("libraryName") === "SynapseML") + assert(map("protocolVersion") === "0.0.1") + } + + test("RequiredLogFields.toMap size is 6") { + val fields = RequiredLogFields("uid", "class", "method") + assert(fields.toMap.size === 6) + } + + test("RequiredErrorFields stores errorType and errorMessage") { + val fields = RequiredErrorFields("java.lang.RuntimeException", "Test error message") + assert(fields.errorType === "java.lang.RuntimeException") + assert(fields.errorMessage === "Test error message") + } + + test("RequiredErrorFields.toMap contains error fields") { + val fields = RequiredErrorFields("ErrorType", "ErrorMessage") + val map = fields.toMap + + assert(map("errorType") === "ErrorType") + assert(map("errorMessage") === "ErrorMessage") + } + + test("RequiredErrorFields can be created from Exception") { + val exception = new RuntimeException("Test exception message") + val fields = new RequiredErrorFields(exception) + + assert(fields.errorType === "java.lang.RuntimeException") + assert(fields.errorMessage === "Test exception message") + } + + test("RequiredErrorFields handles exception with no message") { + // scalastyle:off null + val exception = new RuntimeException(None.orNull: String) + val fields = new RequiredErrorFields(exception) + + assert(fields.errorType === "java.lang.RuntimeException") + assert(Option(fields.errorMessage).isEmpty) + // scalastyle:on null + } + + test("RequiredErrorFields.toMap is JSON-serializable when the exception has no message") { + // Regression: Exception.getMessage is null when an exception is constructed + // without a message. spray-json's JsString rejects null, so an unguarded + // value made getPayload(...).toJson throw IllegalArgumentException and mask + // whatever error was actually being logged. + // scalastyle:off null + val exception = new RuntimeException(None.orNull: String) + // scalastyle:on null + val map = new RequiredErrorFields(exception).toMap + + assert(map("errorMessage") === "") + assert(map("errorType") === "java.lang.RuntimeException") + + import spray.json.DefaultJsonProtocol._ + import spray.json._ + val json = map.toJson.compactPrint + assert(json.contains("\"errorMessage\":\"\"")) + } + + test("SynapseMLLogging.HadoopKeysToLog contains expected mappings") { + val keys = SynapseMLLogging.HadoopKeysToLog + + assert(keys("trident.artifact.id") === "artifactId") + assert(keys("trident.workspace.id") === "workspaceId") + assert(keys("trident.capacity.id") === "capacityId") + assert(keys("trident.artifact.workspace.id") === "artifactWorkspaceId") + assert(keys("trident.lakehouse.id") === "lakehouseId") + assert(keys("trident.activity.id") === "livyId") + assert(keys("trident.artifact.type") === "artifactType") + assert(keys("trident.tenant.id") === "tenantId") + } + + test("SynapseMLLogging.HadoopKeysToLog size is 8") { + assert(SynapseMLLogging.HadoopKeysToLog.size === 8) + } + + test("SynapseMLLogging.LoggedClasses is a mutable set") { + try { + SynapseMLLogging.LoggedClasses.add("TestClass") + assert(SynapseMLLogging.LoggedClasses.contains("TestClass")) + } finally { + // Clean up to avoid leaking state into other tests + SynapseMLLogging.LoggedClasses.remove("TestClass") + } + } + + /** Runs `f` with `spark` as the active session, restoring a previously-active session if there + * was one. + * + * `getHadoopConfEntries` resolves its configuration through `SparkSession.getActiveSession`, so + * these tests need one. When another suite has left a session active on this thread it is + * restored exactly, so nothing is overwritten. + * + * When there was none, the shared session is deliberately left active rather than cleared. + * `TestBase` never establishes one — `getOrCreate` only calls `setActiveSession` on the branch + * that actually constructs the session, not when it returns an existing one — so "restoring" an + * empty previous value would clear the active session for every suite that later runs on this + * thread. `EnsembleByKey.transformSchema` falls back to `getActiveSession` and silently defaults + * `spark.sql.caseSensitive` to `false` when there is none, which turns three `EnsembleByKeySuite` + * tests red when both suites share a JVM. Leaving the shared session active is the canonical + * state, and is what every suite reading `getActiveSession` already assumes. + */ + private def withActiveSharedSession[T](f: => T): T = { + val previous = SparkSession.getActiveSession + SparkSession.setActiveSession(spark) + try f finally previous.foreach(SparkSession.setActiveSession) + } + + test("getHadoopConfEntries reads cluster-level Hadoop configuration") { + // Fabric sets the trident.* keys on the cluster Hadoop conf. getHadoopConfEntries now derives + // its conf from the session instead of spark.sparkContext, so this pins that existing + // telemetry still resolves. + withActiveSharedSession { + val hc = spark.sparkContext.hadoopConfiguration + try { + hc.set("trident.workspace.id", "ws-from-cluster") + assert(SynapseMLLogging.getHadoopConfEntries.get("workspaceId").contains("ws-from-cluster")) + } finally { + hc.unset("trident.workspace.id") + } + } + } + + test("getHadoopConfEntries reads session-level overrides") { + withActiveSharedSession { + try { + spark.conf.set("trident.artifact.id", "artifact-from-session") + assert(SynapseMLLogging.getHadoopConfEntries.get("artifactId").contains("artifact-from-session")) + } finally { + spark.conf.unset("trident.artifact.id") + } + } + } + + test("getHadoopConfEntries returns only known telemetry field names") { + withActiveSharedSession { + val known = SynapseMLLogging.HadoopKeysToLog.values.toSet + assert(SynapseMLLogging.getHadoopConfEntries.keySet.subsetOf(known)) + } + } + + test("getHadoopConfEntries is empty when no session is active") { + // Runs on its own thread so that clearing the active session cannot strand suites that share + // the main test thread; SparkSession's active-session slot is a thread local. + var failure: Option[Throwable] = None + val thread = new Thread(new Runnable { + override def run(): Unit = + try { + SparkSession.clearActiveSession() + assert(SynapseMLLogging.getHadoopConfEntries.isEmpty) + } catch { + case e: Throwable => failure = Some(e) + } + }) + thread.start() + thread.join() + failure.foreach(e => throw e) + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/logging/common/VerifyPlatformDetails.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/logging/common/VerifyPlatformDetails.scala index 3ca5b1f3f6a..1e1c2444878 100644 --- a/core/src/test/scala/com/microsoft/azure/synapse/ml/logging/common/VerifyPlatformDetails.scala +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/logging/common/VerifyPlatformDetails.scala @@ -7,24 +7,61 @@ import com.microsoft.azure.synapse.ml.core.test.base.TestBase class VerifyPlatformDetails extends TestBase { - test("currentPlatform returns a non-null non-empty string") { - val platform = PlatformDetails.currentPlatform() - assert(platform != null) + test("Platform constants have expected values") { + assert(PlatformDetails.PlatformSynapseInternal === "synapse_internal") + assert(PlatformDetails.PlatformSynapse === "synapse") + assert(PlatformDetails.PlatformBinder === "binder") + assert(PlatformDetails.PlatformDatabricks === "databricks") + assert(PlatformDetails.PlatformUnknown === "unknown") + assert(PlatformDetails.SynapseProjectName === "Microsoft.ProjectArcadia") + } + + test("CurrentPlatform returns a string") { + val platform = PlatformDetails.CurrentPlatform assert(platform.nonEmpty) } - test("currentPlatform returns one of the known platform values") { - val known = Set( + test("currentPlatform returns a valid platform string") { + val platform = PlatformDetails.currentPlatform() + val validPlatforms = Set( PlatformDetails.PlatformSynapseInternal, PlatformDetails.PlatformSynapse, - PlatformDetails.PlatformDatabricks, PlatformDetails.PlatformBinder, + PlatformDetails.PlatformDatabricks, PlatformDetails.PlatformUnknown ) - assert(known.contains(PlatformDetails.currentPlatform())) + assert(validPlatforms.contains(platform)) + } + + test("runningOnSynapseInternal agrees with CurrentPlatform") { + assert(PlatformDetails.runningOnSynapseInternal() === + (PlatformDetails.CurrentPlatform == PlatformDetails.PlatformSynapseInternal)) } - test("runningOnFabric is consistent with runningOnSynapseInternal") { + test("runningOnSynapse agrees with CurrentPlatform") { + assert(PlatformDetails.runningOnSynapse() === + (PlatformDetails.CurrentPlatform == PlatformDetails.PlatformSynapse)) + } + + test("runningOnSynapse and runningOnSynapseInternal are mutually exclusive") { + assert(!(PlatformDetails.runningOnSynapse() && PlatformDetails.runningOnSynapseInternal())) + } + + test("runningOnFabric returns same as runningOnSynapseInternal") { assert(PlatformDetails.runningOnFabric() === PlatformDetails.runningOnSynapseInternal()) } + + test("CurrentPlatform returns a known platform value") { + val platform = PlatformDetails.CurrentPlatform + // Expected platforms when running tests on a local/dev environment + val expectedOnDev = Set(PlatformDetails.PlatformUnknown, PlatformDetails.PlatformBinder) + // Allow-list of platforms that may legitimately appear in CI (e.g., Synapse or Databricks) + val ciPlatforms = Set( + PlatformDetails.PlatformSynapseInternal, + PlatformDetails.PlatformSynapse, + PlatformDetails.PlatformDatabricks + ) + // Verify that the platform is either a dev-expected value or a known CI platform + assert(expectedOnDev.contains(platform) || ciPlatforms.contains(platform)) + } } diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/logging/common/VerifyScrubber.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/logging/common/VerifyScrubber.scala index 5db71ef9d3f..f97a237c6c6 100644 --- a/core/src/test/scala/com/microsoft/azure/synapse/ml/logging/common/VerifyScrubber.scala +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/logging/common/VerifyScrubber.scala @@ -7,50 +7,67 @@ import com.microsoft.azure.synapse.ml.core.test.base.TestBase class VerifyScrubber extends TestBase { - test("SASScrubber replaces SAS signature in URL") { - val url = "https://storage.blob.core.windows.net/container?sv=2021-06-08" + - "&sig=abcdef1234567890abcdef1234567890abcdef12345%3d&se=2024-01-01" - val scrubbed = SASScrubber.scrub(url) - assert(scrubbed.contains("sig=####")) - assert(!scrubbed.contains("abcdef1234567890")) - } - - test("SASScrubber leaves strings without SAS tokens unchanged") { - val message = "This is a normal log message with no SAS token" - assert(SASScrubber.scrub(message) === message) - } - - test("SASScrubber handles multiple SAS tokens in same string") { - val url1 = "https://storage1.blob.core.windows.net/c1?sig=abcdef1234567890abcdef1234567890abcdef12345%3d" - val url2 = "https://storage2.blob.core.windows.net/c2?sig=123456abcdef7890123456abcdef7890123456abc78%3d" - val combined = s"$url1 and $url2" - val scrubbed = SASScrubber.scrub(combined) - assert(scrubbed === "https://storage1.blob.core.windows.net/c1?sig=####" + - " and https://storage2.blob.core.windows.net/c2?sig=####") - } - - test("SASScrubber is case insensitive") { - val upperUrl = "https://storage.blob.core.windows.net/container" + - "?SIG=ABCDEF1234567890ABCDEF1234567890ABCDEF12345%3D&se=2024-01-01" - val mixedUrl = "https://storage.blob.core.windows.net/container" + - "?Sig=AbCdEf1234567890AbCdEf1234567890AbCdEf12345%3d&se=2024-01-01" - val scrubbedUpper = SASScrubber.scrub(upperUrl) - val scrubbedMixed = SASScrubber.scrub(mixedUrl) - assert(scrubbedUpper.contains("sig=####")) - assert(scrubbedMixed.contains("sig=####")) - assert(!scrubbedUpper.contains("ABCDEF1234567890")) - assert(!scrubbedMixed.contains("AbCdEf1234567890")) - } - - test("SASScrubber preserves rest of URL around the signature") { - val url = "https://storage.blob.core.windows.net/container?sv=2021-06-08" + - "&sig=abcdef1234567890abcdef1234567890abcdef12345%3d&se=2024-01-01&sp=r" - val scrubbed = SASScrubber.scrub(url) - assert(scrubbed.contains("sv=2021-06-08")) - assert(scrubbed.contains("se=2024-01-01")) - assert(scrubbed.contains("sp=r")) - assert(scrubbed.contains("sig=####")) - assert(scrubbed === "https://storage.blob.core.windows.net/container?sv=2021-06-08" + - "&sig=####&se=2024-01-01&sp=r") + test("SASScrubber scrubs SAS signature from URL") { + // SAS tokens typically contain sig= followed by base64-like encoded signature + // Dummy URL for testing - not a real endpoint + // scalastyle:off line.size.limit + val urlWithSas = "https://storage.blob.core.windows.net/container/file?sig=abcdefghijklmnopqrstuvwxyz0123456789ABCDEFG%3d" + // scalastyle:on line.size.limit + val result = SASScrubber.scrub(urlWithSas) + assert(result.contains("sig=####")) + assert(!result.contains("abcdefghijklmnopqrstuvwxyz")) + } + + test("SASScrubber handles multiple SAS signatures in one string") { + // Use signatures that match the pattern: sig= followed by 43-63 alphanumeric/% chars ending with %3d + val sig1 = "sig=abcdefghijklmnopqrstuvwxyz0123456789ABCDEFG%3d" + val sig2 = "sig=XYZ987wvu654tsr321qpo098nml765kji432hgf109AB%3d" + val message = s"URL1: $sig1 and URL2: $sig2" + val result = SASScrubber.scrub(message) + // Both signatures should be replaced + assert(!result.contains("abcdefghijklmnopqrstuvwxyz")) + assert(!result.contains("XYZ987wvu654")) + assert(result.contains("sig=####")) + } + + test("SASScrubber leaves non-SAS content unchanged") { + val message = "This is a normal log message without any signatures" + val result = SASScrubber.scrub(message) + assert(result === message) + } + + test("SASScrubber is case insensitive for sig parameter") { + val lowerCase = "https://test.com?sig=abcdefghijklmnopqrstuvwxyz0123456789ABCDEFG%3d" + val upperCase = "https://test.com?SIG=abcdefghijklmnopqrstuvwxyz0123456789ABCDEFG%3d" + val mixedCase = "https://test.com?SiG=abcdefghijklmnopqrstuvwxyz0123456789ABCDEFG%3d" + + assert(SASScrubber.scrub(lowerCase).contains("sig=####")) + assert(SASScrubber.scrub(upperCase).contains("sig=####")) + assert(SASScrubber.scrub(mixedCase).contains("sig=####")) + } + + test("SASScrubber handles empty string") { + assert(SASScrubber.scrub("") === "") + } + + test("SASScrubber handles string with only sig= but invalid signature") { + // Too short signature should not be scrubbed + val shortSig = "https://test.com?sig=abc" + assert(SASScrubber.scrub(shortSig) === shortSig) + } + + test("SASScrubber preserves text before and after signature") { + val message = "Prefix text sig=abcdefghijklmnopqrstuvwxyz0123456789ABCDEFG%3d suffix text" + val result = SASScrubber.scrub(message) + assert(result.startsWith("Prefix text")) + assert(result.endsWith("suffix text")) + assert(result.contains("sig=####")) + } + + test("SASScrubber implements Scrubber trait") { + val scrubber: Scrubber = SASScrubber + val message = "Test sig=abcdefghijklmnopqrstuvwxyz0123456789ABCDEFG%3d" + val result = scrubber.scrub(message) + assert(result.contains("sig=####")) } } diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/FabricNotebookTests.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/FabricNotebookTests.scala index 49206afbad6..65cc88a666f 100644 --- a/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/FabricNotebookTests.scala +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/FabricNotebookTests.scala @@ -176,8 +176,10 @@ object FabricNotebookTests { ) private val ExecutorShutdownTimeoutSeconds = 30L - private val StoreArtifactName = "^(Lakehouse|Warehouse)\\d{14}$".r - private val SJDArtifactName = "^(.+)-\\d{8}-\\d{2}-\\d{2}-\\d{2}$".r + private val UniqueArtifactId = "[0-9a-fA-F]{32}" + private val StoreArtifactName = s"^(Lakehouse|Warehouse)\\d{14}(?:$UniqueArtifactId)?$$".r + private val SJDArtifactName = + s"^(.+)-\\d{8}-\\d{2}-\\d{2}-\\d{2}(?:-$UniqueArtifactId)?$$".r private val TestSJDNames = (IncludedNotebooks :+ "OnePlusOne").toSet private[nbtest] def shutdownExecutor(executorService: ExecutorService): Unit = { diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/FabricTestArtifactTrackerSuite.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/FabricTestArtifactTrackerSuite.scala index 54dee70e3c1..359478dc6e5 100644 --- a/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/FabricTestArtifactTrackerSuite.scala +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/FabricTestArtifactTrackerSuite.scala @@ -61,10 +61,19 @@ class FabricTestArtifactTrackerSuite extends AnyFunSuite { test("Recognize only SynapseML Fabric test artifact names") { assert(FabricNotebookTests.isTestArtifactName("Lakehouse20260808010917")) + assert(FabricNotebookTests.isTestArtifactName( + "Lakehouse202608080109170123456789abcdef0123456789abcdef")) assert(FabricNotebookTests.isTestArtifactName( "ExploreAlgorithmsRegressionQuickstartTrainRegressor-20260808-01-09-17")) + assert(FabricNotebookTests.isTestArtifactName( + "ExploreAlgorithmsRegressionQuickstartTrainRegressor-20260808-01-09-17-" + + "0123456789abcdef0123456789abcdef")) assert(FabricNotebookTests.isTestArtifactName("OnePlusOne-20260808-01-09-17")) assert(!FabricNotebookTests.isTestArtifactName("LakehouseForManualTesting")) + assert(!FabricNotebookTests.isTestArtifactName( + "Lakehouse20260808010917-not-a-unique-id")) + assert(!FabricNotebookTests.isTestArtifactName( + "Lakehouse20260808010917-0123456789abcdef0123456789abcdef")) assert(!FabricNotebookTests.isTestArtifactName( "ExploreAlgorithmsAdHocNotebook-20260808-01-09-17")) assert(!FabricNotebookTests.isTestArtifactName("CustomerNotebook-20260808-01-09-17")) diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyByteArrayParam.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyByteArrayParam.scala new file mode 100644 index 00000000000..cde25744473 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyByteArrayParam.scala @@ -0,0 +1,78 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.param + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.spark.ml.param.{ParamMap, Params} + +class VerifyByteArrayParam extends TestBase { + + private class TestParamsHolder extends Params { + override val uid: String = "test-holder" + val bytesParam = new ByteArrayParam(this, "bytes", "A byte array param") + override def copy(extra: ParamMap): Params = this + } + + test("ByteArrayParam can be created with basic constructor") { + val holder = new TestParamsHolder + assert(holder.bytesParam.name === "bytes") + assert(holder.bytesParam.doc === "A byte array param") + } + + test("ByteArrayParam accepts empty byte array") { + val holder = new TestParamsHolder + holder.set(holder.bytesParam, Array.empty[Byte]) + assert(holder.get(holder.bytesParam).exists(_.isEmpty)) + } + + test("ByteArrayParam accepts byte array with data") { + val holder = new TestParamsHolder + val data = Array[Byte](1, 2, 3, 4, 5) + holder.set(holder.bytesParam, data) + assert(holder.get(holder.bytesParam).exists(_.sameElements(data))) + } + + test("ByteArrayParam accepts large byte array") { + val holder = new TestParamsHolder + val data = Array.fill(1000)(42.toByte) + holder.set(holder.bytesParam, data) + assert(holder.get(holder.bytesParam).exists(_.length === 1000)) + } + + test("ByteArrayParam accepts byte array with all byte values") { + val holder = new TestParamsHolder + val data = (-128 to 127).map(_.toByte).toArray + holder.set(holder.bytesParam, data) + assert(holder.get(holder.bytesParam).exists(_.length === 256)) + } + + test("ByteArrayParam custom validator accepts and rejects per its predicate") { + val holder = new Params { + override val uid: String = "test" + val nonEmptyBytes = new ByteArrayParam( + this, "nonEmpty", "Non-empty byte array", + (arr: Array[Byte]) => arr.nonEmpty + ) + override def copy(extra: ParamMap): Params = this + } + holder.set(holder.nonEmptyBytes, Array[Byte](1, 2, 3)) + assert(holder.get(holder.nonEmptyBytes).exists(_.length === 3)) + assertThrows[IllegalArgumentException] { + holder.set(holder.nonEmptyBytes, Array.empty[Byte]) + } + } + + test("ByteArrayParam can be cleared") { + val holder = new TestParamsHolder + holder.set(holder.bytesParam, Array[Byte](1, 2, 3)) + assert(holder.isSet(holder.bytesParam)) + holder.clear(holder.bytesParam) + assert(!holder.isSet(holder.bytesParam)) + } + + test("ByteArrayParam returns None when not set") { + val holder = new TestParamsHolder + assert(holder.get(holder.bytesParam).isEmpty) + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyDataFrameParam.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyDataFrameParam.scala new file mode 100644 index 00000000000..485be61190f --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyDataFrameParam.scala @@ -0,0 +1,163 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.param + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.spark.ml.linalg.{DenseVector, Vectors} +import org.apache.spark.ml.param.{ParamMap, Params} +import org.apache.spark.sql.{DataFrame, Row} +import org.apache.spark.sql.types._ + +class VerifyDataFrameParam extends TestBase { + + import spark.implicits._ + + private class TestParamsHolder extends Params { + override val uid: String = "test-holder" + val dfParam = new DataFrameParam(this, "dataFrame", "A dataframe param") + override def copy(extra: ParamMap): Params = this + } + + // DataFrameParam basic tests + test("DataFrameParam can be created with basic constructor") { + val holder = new TestParamsHolder + assert(holder.dfParam.name === "dataFrame") + assert(holder.dfParam.doc === "A dataframe param") + } + + test("DataFrameParam accepts DataFrame") { + val holder = new TestParamsHolder + val df = Seq(("a", 1), ("b", 2)).toDF("str", "num") + holder.set(holder.dfParam, df) + assert(holder.isSet(holder.dfParam)) + } + + test("DataFrameParam pyValue returns DF reference") { + val holder = new TestParamsHolder + val df = Seq(1, 2, 3).toDF("num") + val pyVal = holder.dfParam.pyValue(df) + assert(pyVal === "dataFrameDF") + } + + test("DataFrameParam pyLoadLine generates Python code") { + val holder = new TestParamsHolder + val pyCode = holder.dfParam.pyLoadLine(1) + assert(pyCode.contains("spark.read.parquet")) + assert(pyCode.contains("model-1.model")) + assert(pyCode.contains("complexParams")) + assert(pyCode.contains("dataFrame")) + } + + test("DataFrameParam rValue returns DF reference") { + val holder = new TestParamsHolder + val df = Seq(1, 2, 3).toDF("num") + val rVal = holder.dfParam.rValue(df) + assert(rVal === "dataFrameDF") + } + + test("DataFrameParam rLoadLine generates R code") { + val holder = new TestParamsHolder + val rCode = holder.dfParam.rLoadLine(2) + assert(rCode.contains("spark_read_parquet")) + assert(rCode.contains("model-2.model")) + assert(rCode.contains("complexParams")) + assert(rCode.contains("dataFrame")) + } + + // DataFrameEquality tests + test("DataFrameEquality compares equal DataFrames correctly") { + val holder = new TestParamsHolder + val df1 = Seq(("a", 1), ("b", 2)).toDF("str", "num") + val df2 = Seq(("a", 1), ("b", 2)).toDF("str", "num") + // Should not throw + holder.dfParam.assertEquality(df1, df2) + } + + test("DataFrameEquality detects different DataFrames") { + val holder = new TestParamsHolder + val df1 = Seq(("a", 1), ("b", 2)).toDF("str", "num") + val df2 = Seq(("a", 1), ("c", 3)).toDF("str", "num") + assertThrows[AssertionError] { + holder.dfParam.assertEquality(df1, df2) + } + } + + test("DataFrameEquality throws for non-DataFrame types") { + val holder = new TestParamsHolder + assertThrows[AssertionError] { + holder.dfParam.assertEquality("not a df", "also not a df") + } + } + + // DataFrameEquality implicit tests + test("DataFrameEquality handles doubles with tolerance") { + val holder = new TestParamsHolder + val df1 = Seq(1.0, 2.0, 3.0).toDF("num") + val df2 = Seq(1.00001, 2.00001, 3.00001).toDF("num") + // Should not throw due to tolerance + holder.dfParam.assertEquality(df1, df2) + } + + test("DataFrameEquality handles NaN values") { + val holder = new TestParamsHolder + val df1 = Seq(Double.NaN, 2.0).toDF("num") + val df2 = Seq(Double.NaN, 2.0).toDF("num") + holder.dfParam.assertEquality(df1, df2) + } + + test("DataFrameEquality handles DenseVector columns") { + val holder = new TestParamsHolder + // Create DataFrames with vector columns using VectorAssembler + import org.apache.spark.ml.feature.VectorAssembler + val baseData1 = Seq((1.0, 2.0), (3.0, 4.0)).toDF("a", "b") + val baseData2 = Seq((1.0, 2.0), (3.0, 4.0)).toDF("a", "b") + val assembler = new VectorAssembler().setInputCols(Array("a", "b")).setOutputCol("vec") + val df1 = assembler.transform(baseData1).select("vec") + val df2 = assembler.transform(baseData2).select("vec") + holder.dfParam.assertEquality(df1, df2) + } + + test("DataFrameEquality handles binary columns") { + val holder = new TestParamsHolder + val df1 = Seq(Array[Byte](1, 2, 3)).toDF("bytes") + val df2 = Seq(Array[Byte](1, 2, 3)).toDF("bytes") + holder.dfParam.assertEquality(df1, df2) + } + + test("DataFrameEquality detects different column names") { + val holder = new TestParamsHolder + val df1 = Seq(1, 2, 3).toDF("col1") + val df2 = Seq(1, 2, 3).toDF("col2") + assertThrows[AssertionError] { + holder.dfParam.assertEquality(df1, df2) + } + } + + test("DataFrameEquality detects different row counts") { + val holder = new TestParamsHolder + val df1 = Seq(1, 2, 3).toDF("num") + val df2 = Seq(1, 2).toDF("num") + assertThrows[AssertionError] { + holder.dfParam.assertEquality(df1, df2) + } + } + + test("DataFrameParam with custom validator") { + val holder = new Params { + override val uid: String = "test" + val nonEmptyDf = new DataFrameParam( + this, "nonEmpty", "Non-empty dataframe", + (df: DataFrame) => df.count() > 0 + ) + override def copy(extra: ParamMap): Params = this + } + val df = Seq(1, 2, 3).toDF("num") + holder.set(holder.nonEmptyDf, df) + } + + test("DataFrameParam sortInDataframeEquality is true") { + val holder = new TestParamsHolder + assert(holder.dfParam.sortInDataframeEquality) + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyDataTypeParam.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyDataTypeParam.scala new file mode 100644 index 00000000000..afeb1a4d793 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyDataTypeParam.scala @@ -0,0 +1,126 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.param + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.spark.ml.param.{ParamMap, Params} +import org.apache.spark.sql.types._ + +class VerifyDataTypeParam extends TestBase { + + private class TestParamsHolder extends Params { + override val uid: String = "test-holder" + val dataTypeParam = new DataTypeParam(this, "dataType", "A data type param") + override def copy(extra: ParamMap): Params = this + } + + test("DataTypeParam can be created with basic constructor") { + val holder = new TestParamsHolder + assert(holder.dataTypeParam.name === "dataType") + assert(holder.dataTypeParam.doc === "A data type param") + } + + test("DataTypeParam accepts StringType") { + val holder = new TestParamsHolder + holder.set(holder.dataTypeParam, StringType) + assert(holder.get(holder.dataTypeParam).contains(StringType)) + } + + test("DataTypeParam accepts IntegerType") { + val holder = new TestParamsHolder + holder.set(holder.dataTypeParam, IntegerType) + assert(holder.get(holder.dataTypeParam).contains(IntegerType)) + } + + test("DataTypeParam accepts DoubleType") { + val holder = new TestParamsHolder + holder.set(holder.dataTypeParam, DoubleType) + assert(holder.get(holder.dataTypeParam).contains(DoubleType)) + } + + test("DataTypeParam accepts BooleanType") { + val holder = new TestParamsHolder + holder.set(holder.dataTypeParam, BooleanType) + assert(holder.get(holder.dataTypeParam).contains(BooleanType)) + } + + test("DataTypeParam accepts ArrayType") { + val holder = new TestParamsHolder + val arrayType = ArrayType(StringType) + holder.set(holder.dataTypeParam, arrayType) + assert(holder.get(holder.dataTypeParam).contains(arrayType)) + } + + test("DataTypeParam accepts MapType") { + val holder = new TestParamsHolder + val mapType = MapType(StringType, IntegerType) + holder.set(holder.dataTypeParam, mapType) + assert(holder.get(holder.dataTypeParam).contains(mapType)) + } + + test("DataTypeParam accepts StructType") { + val holder = new TestParamsHolder + val structType = StructType(Seq( + StructField("name", StringType), + StructField("age", IntegerType) + )) + holder.set(holder.dataTypeParam, structType) + assert(holder.get(holder.dataTypeParam).contains(structType)) + } + + test("DataTypeParam accepts nested StructType") { + val holder = new TestParamsHolder + val nestedType = StructType(Seq( + StructField("outer", StructType(Seq( + StructField("inner", StringType) + ))) + )) + holder.set(holder.dataTypeParam, nestedType) + assert(holder.get(holder.dataTypeParam).contains(nestedType)) + } + + test("DataTypeParam accepts TimestampType") { + val holder = new TestParamsHolder + holder.set(holder.dataTypeParam, TimestampType) + assert(holder.get(holder.dataTypeParam).contains(TimestampType)) + } + + test("DataTypeParam accepts DateType") { + val holder = new TestParamsHolder + holder.set(holder.dataTypeParam, DateType) + assert(holder.get(holder.dataTypeParam).contains(DateType)) + } + + test("DataTypeParam accepts BinaryType") { + val holder = new TestParamsHolder + holder.set(holder.dataTypeParam, BinaryType) + assert(holder.get(holder.dataTypeParam).contains(BinaryType)) + } + + test("DataTypeParam custom validator accepts and rejects per its predicate") { + val holder = new Params { + override val uid: String = "test" + val numericOnlyParam = new DataTypeParam( + this, "numericOnly", "Only numeric types", + (dt: DataType) => dt.isInstanceOf[NumericType] + ) + override def copy(extra: ParamMap): Params = this + } + holder.set(holder.numericOnlyParam, IntegerType) + holder.set(holder.numericOnlyParam, DoubleType) + holder.set(holder.numericOnlyParam, FloatType) + assert(holder.get(holder.numericOnlyParam).contains(FloatType)) + assertThrows[IllegalArgumentException] { + holder.set(holder.numericOnlyParam, StringType) + } + } + + test("DataTypeParam can be cleared") { + val holder = new TestParamsHolder + holder.set(holder.dataTypeParam, StringType) + assert(holder.isSet(holder.dataTypeParam)) + holder.clear(holder.dataTypeParam) + assert(!holder.isSet(holder.dataTypeParam)) + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyEstimatorArrayParam.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyEstimatorArrayParam.scala new file mode 100644 index 00000000000..5c0ed4a6089 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyEstimatorArrayParam.scala @@ -0,0 +1,93 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.param + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.spark.ml.Estimator +import org.apache.spark.ml.classification.{LogisticRegression, DecisionTreeClassifier} +import org.apache.spark.ml.feature.StringIndexer +import org.apache.spark.ml.param.{ParamMap, Params} + +import java.util.{ArrayList => JArrayList} + +class VerifyEstimatorArrayParam extends TestBase { + + private class TestParamsHolder extends Params { + override val uid: String = "test-holder" + val estimatorsParam = new EstimatorArrayParam(this, "estimators", "An array of estimators") + override def copy(extra: ParamMap): Params = this + } + + test("EstimatorArrayParam can be created with basic constructor") { + val holder = new TestParamsHolder + assert(holder.estimatorsParam.name === "estimators") + assert(holder.estimatorsParam.doc === "An array of estimators") + } + + test("EstimatorArrayParam accepts empty array") { + val holder = new TestParamsHolder + holder.set(holder.estimatorsParam, Array.empty[Estimator[_]]) + assert(holder.get(holder.estimatorsParam).exists(_.isEmpty)) + } + + test("EstimatorArrayParam accepts array with single estimator") { + val holder = new TestParamsHolder + val estimators = Array[Estimator[_]](new LogisticRegression()) + holder.set(holder.estimatorsParam, estimators) + assert(holder.get(holder.estimatorsParam).exists(_.length === 1)) + } + + test("EstimatorArrayParam accepts array with multiple estimators") { + val holder = new TestParamsHolder + val estimators = Array[Estimator[_]]( + new LogisticRegression(), + new DecisionTreeClassifier(), + new StringIndexer() + ) + holder.set(holder.estimatorsParam, estimators) + assert(holder.get(holder.estimatorsParam).exists(_.length === 3)) + } + + test("EstimatorArrayParam w() method accepts Java List") { + val holder = new TestParamsHolder + val javaList = new JArrayList[Estimator[_]]() + javaList.add(new LogisticRegression()) + javaList.add(new DecisionTreeClassifier()) + + val paramPair = holder.estimatorsParam.w(javaList) + assert(paramPair.param === holder.estimatorsParam) + assert(paramPair.value.length === 2) + } + + test("EstimatorArrayParam custom validator accepts and rejects per its predicate") { + val holder = new Params { + override val uid: String = "test" + val nonEmptyEstimators = new EstimatorArrayParam( + this, "nonEmpty", "Non-empty estimator array", + (arr: Array[Estimator[_]]) => arr.nonEmpty + ) + override def copy(extra: ParamMap): Params = this + } + val estimators = Array[Estimator[_]](new LogisticRegression()) + holder.set(holder.nonEmptyEstimators, estimators) + assert(holder.get(holder.nonEmptyEstimators).exists(_.length === 1)) + assertThrows[IllegalArgumentException] { + holder.set(holder.nonEmptyEstimators, Array.empty[Estimator[_]]) + } + } + + test("EstimatorArrayParam can be cleared") { + val holder = new TestParamsHolder + val estimators = Array[Estimator[_]](new LogisticRegression()) + holder.set(holder.estimatorsParam, estimators) + assert(holder.isSet(holder.estimatorsParam)) + holder.clear(holder.estimatorsParam) + assert(!holder.isSet(holder.estimatorsParam)) + } + + test("EstimatorArrayParam returns None when not set") { + val holder = new TestParamsHolder + assert(holder.get(holder.estimatorsParam).isEmpty) + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyEvaluatorParam.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyEvaluatorParam.scala new file mode 100644 index 00000000000..f7ef0222800 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyEvaluatorParam.scala @@ -0,0 +1,95 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.param + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.spark.ml.evaluation.{ + BinaryClassificationEvaluator, MulticlassClassificationEvaluator, RegressionEvaluator +} +import org.apache.spark.ml.param.{ParamMap, Params} + +class VerifyEvaluatorParam extends TestBase { + + private class TestParamsHolder extends Params { + override val uid: String = "test-holder" + val evaluatorParam = new EvaluatorParam(this, "evaluator", "An evaluator param") + override def copy(extra: ParamMap): Params = this + } + + test("EvaluatorParam can be created with basic constructor") { + val holder = new TestParamsHolder + assert(holder.evaluatorParam.name === "evaluator") + assert(holder.evaluatorParam.doc === "An evaluator param") + } + + test("EvaluatorParam accepts BinaryClassificationEvaluator") { + val holder = new TestParamsHolder + val evaluator = new BinaryClassificationEvaluator() + holder.set(holder.evaluatorParam, evaluator) + assert(holder.isSet(holder.evaluatorParam)) + } + + test("EvaluatorParam accepts MulticlassClassificationEvaluator") { + val holder = new TestParamsHolder + val evaluator = new MulticlassClassificationEvaluator() + holder.set(holder.evaluatorParam, evaluator) + assert(holder.isSet(holder.evaluatorParam)) + } + + test("EvaluatorParam accepts RegressionEvaluator") { + val holder = new TestParamsHolder + val evaluator = new RegressionEvaluator() + holder.set(holder.evaluatorParam, evaluator) + assert(holder.isSet(holder.evaluatorParam)) + } + + test("EvaluatorParam assertEquality passes for same evaluator type") { + val holder = new TestParamsHolder + val eval1 = new BinaryClassificationEvaluator() + .setMetricName("areaUnderROC") + .setLabelCol("label") + val eval2 = new BinaryClassificationEvaluator() + .setMetricName("areaUnderROC") + .setLabelCol("label") + holder.evaluatorParam.assertEquality(eval1, eval2) + } + + test("EvaluatorParam assertEquality throws for non-Evaluator types") { + val holder = new TestParamsHolder + assertThrows[AssertionError] { + holder.evaluatorParam.assertEquality("not an evaluator", "also not") + } + } + + test("EvaluatorParam custom validator accepts and rejects per its predicate") { + val holder = new Params { + override val uid: String = "test" + val binaryOnly = new EvaluatorParam( + this, "binaryOnly", "Binary evaluator only", + _.isInstanceOf[BinaryClassificationEvaluator] + ) + override def copy(extra: ParamMap): Params = this + } + val evaluator = new BinaryClassificationEvaluator() + holder.set(holder.binaryOnly, evaluator) + assert(holder.get(holder.binaryOnly).contains(evaluator)) + assertThrows[IllegalArgumentException] { + holder.set(holder.binaryOnly, new RegressionEvaluator()) + } + } + + test("EvaluatorParam can be cleared") { + val holder = new TestParamsHolder + val evaluator = new RegressionEvaluator() + holder.set(holder.evaluatorParam, evaluator) + assert(holder.isSet(holder.evaluatorParam)) + holder.clear(holder.evaluatorParam) + assert(!holder.isSet(holder.evaluatorParam)) + } + + test("EvaluatorParam returns None when not set") { + val holder = new TestParamsHolder + assert(holder.get(holder.evaluatorParam).isEmpty) + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyGlobalParams.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyGlobalParams.scala new file mode 100644 index 00000000000..35e349887bf --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyGlobalParams.scala @@ -0,0 +1,101 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.param + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.spark.ml.param.{Param, ParamMap, Params} +import org.apache.spark.ml.util.Identifiable +import org.scalatest.TestData + +class VerifyGlobalParams extends TestBase { + + // Test keys + case object TestStringKey extends GlobalKey[String] + case object TestIntKey extends GlobalKey[Int] + case object AnotherStringKey extends GlobalKey[String] + + // GlobalParams is process-wide mutable state, so every test must start from a clean slate + // and must not leak keys into suites that share the forked JVM. + private def resetGlobalState(): Unit = { + GlobalParams.resetGlobalParam(TestStringKey) + GlobalParams.resetGlobalParam(TestIntKey) + GlobalParams.resetGlobalParam(AnotherStringKey) + } + + protected override def beforeEach(td: TestData): Unit = { + super.beforeEach(td) + resetGlobalState() + } + + protected override def afterEach(td: TestData): Unit = { + resetGlobalState() + super.afterEach(td) + } + + test("setGlobalParam and getGlobalParam work for String") { + GlobalParams.setGlobalParam(TestStringKey, "test-value") + val result = GlobalParams.getGlobalParam(TestStringKey) + assert(result === Some("test-value")) + } + + test("setGlobalParam and getGlobalParam work for Int") { + GlobalParams.setGlobalParam(TestIntKey, 42) + val result = GlobalParams.getGlobalParam(TestIntKey) + assert(result === Some(42)) + } + + test("getGlobalParam returns None for unset key") { + val result = GlobalParams.getGlobalParam(TestStringKey) + assert(result.isEmpty) + } + + test("resetGlobalParam removes the parameter") { + GlobalParams.setGlobalParam(TestStringKey, "value") + assert(GlobalParams.getGlobalParam(TestStringKey).isDefined) + GlobalParams.resetGlobalParam(TestStringKey) + assert(GlobalParams.getGlobalParam(TestStringKey).isEmpty) + } + + test("setGlobalParam overwrites existing value") { + GlobalParams.setGlobalParam(TestStringKey, "first") + GlobalParams.setGlobalParam(TestStringKey, "second") + assert(GlobalParams.getGlobalParam(TestStringKey) === Some("second")) + } + + test("multiple keys can be set independently") { + GlobalParams.setGlobalParam(TestStringKey, "string-value") + GlobalParams.setGlobalParam(TestIntKey, 100) + GlobalParams.setGlobalParam(AnotherStringKey, "another-value") + + assert(GlobalParams.getGlobalParam(TestStringKey) === Some("string-value")) + assert(GlobalParams.getGlobalParam(TestIntKey) === Some(100)) + assert(GlobalParams.getGlobalParam(AnotherStringKey) === Some("another-value")) + } + + test("registerParam and getParam work together") { + // Create a test Params implementation + class TestParams(override val uid: String) extends Params { + val testParam = new Param[String](this, "testParam", "test param") + override def copy(extra: ParamMap): Params = this + } + + val params = new TestParams(Identifiable.randomUID("test")) + GlobalParams.registerParam(params.testParam, TestStringKey) + GlobalParams.setGlobalParam(TestStringKey, "global-value") + + val result = GlobalParams.getParam(params.testParam) + assert(result === Some("global-value")) + } + + test("getParam returns None for unregistered param") { + class TestParams(override val uid: String) extends Params { + val unregisteredParam = new Param[String](this, "unregisteredParam", "not registered") + override def copy(extra: ParamMap): Params = this + } + + val params = new TestParams(Identifiable.randomUID("test")) + val result = GlobalParams.getParam(params.unregisteredParam) + assert(result.isEmpty) + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyModelParam.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyModelParam.scala new file mode 100644 index 00000000000..b209567d4b2 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyModelParam.scala @@ -0,0 +1,101 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.param + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.spark.ml.Model +import org.apache.spark.ml.classification.{LogisticRegression, LogisticRegressionModel} +import org.apache.spark.ml.feature.{StringIndexer, VectorAssembler} +import org.apache.spark.ml.param.{ParamMap, Params} + +class VerifyModelParam extends TestBase { + + import spark.implicits._ + + private class TestParamsHolder extends Params { + override val uid: String = "test-holder" + val modelParam = new ModelParam(this, "model", "A model param") + override def copy(extra: ParamMap): Params = this + } + + // Helper to create a trained model + private def createTrainedModel(): LogisticRegressionModel = { + val data = Seq( + (0.0, 1.0, 0.0), + (1.0, 0.0, 1.0), + (0.0, 1.0, 0.0), + (1.0, 0.0, 1.0) + ).toDF("label", "f1", "f2") + + val assembler = new VectorAssembler() + .setInputCols(Array("f1", "f2")) + .setOutputCol("features") + val assembled = assembler.transform(data) + + val lr = new LogisticRegression() + .setMaxIter(5) + .setLabelCol("label") + .setFeaturesCol("features") + lr.fit(assembled) + } + + test("ModelParam can be created with basic constructor") { + val holder = new TestParamsHolder + assert(holder.modelParam.name === "model") + assert(holder.modelParam.doc === "A model param") + } + + test("ModelParam accepts LogisticRegressionModel") { + val holder = new TestParamsHolder + val model = createTrainedModel() + holder.set(holder.modelParam, model) + assert(holder.isSet(holder.modelParam)) + } + + test("ModelParam pyValue returns model reference") { + val holder = new TestParamsHolder + val model = createTrainedModel() + val pyVal = holder.modelParam.pyValue(model) + assert(pyVal === "modelModel") + } + + test("ModelParam pyLoadLine generates Python code") { + val holder = new TestParamsHolder + val pyCode = holder.modelParam.pyLoadLine(1) + assert(pyCode.contains("Pipeline.load")) + assert(pyCode.contains("model-1.model")) + assert(pyCode.contains("complexParams")) + } + + test("ModelParam rValue returns model reference") { + val holder = new TestParamsHolder + val model = createTrainedModel() + val rVal = holder.modelParam.rValue(model) + assert(rVal === "modelModel") + } + + test("ModelParam rLoadLine generates R code") { + val holder = new TestParamsHolder + val rCode = holder.modelParam.rLoadLine(2) + assert(rCode.contains("ml_load")) + assert(rCode.contains("model-2.model")) + // Spark 4.1 extracts the stage via sparklyr internals instead of ml_stages; see RCodegenSuite. + assert(rCode.contains("new_ml_pipeline_stage")) + assert(rCode.contains("getStages")) + } + + test("ModelParam can be cleared") { + val holder = new TestParamsHolder + val model = createTrainedModel() + holder.set(holder.modelParam, model) + assert(holder.isSet(holder.modelParam)) + holder.clear(holder.modelParam) + assert(!holder.isSet(holder.modelParam)) + } + + test("ModelParam returns None when not set") { + val holder = new TestParamsHolder + assert(holder.get(holder.modelParam).isEmpty) + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyPipelineStageParams.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyPipelineStageParams.scala new file mode 100644 index 00000000000..21dc403404e --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyPipelineStageParams.scala @@ -0,0 +1,159 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.param + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.spark.ml.{Transformer, Estimator, Model, PipelineStage} +import org.apache.spark.ml.feature.{Tokenizer, HashingTF, StringIndexer, StringIndexerModel} +import org.apache.spark.ml.param.{ParamMap, Params} +import org.apache.spark.sql.{DataFrame, Dataset} +import org.apache.spark.sql.types.StructType + +class VerifyPipelineStageParams extends TestBase { + + // Test class that holds the params + private class TestParamsHolder extends Params { + override val uid: String = "test-holder" + + val transformerParam = new TransformerParam(this, "transformer", "A transformer param") + val estimatorParam = new EstimatorParam(this, "estimator", "An estimator param") + val pipelineStageParam = new PipelineStageParam(this, "pipelineStage", "A pipeline stage param") + + override def copy(extra: ParamMap): Params = this + } + + // TransformerParam tests + test("TransformerParam can be created with basic constructor") { + val holder = new TestParamsHolder + assert(holder.transformerParam.name === "transformer") + assert(holder.transformerParam.doc === "A transformer param") + } + + test("TransformerParam accepts valid Transformer") { + val holder = new TestParamsHolder + val tokenizer = new Tokenizer().setInputCol("text").setOutputCol("words") + holder.set(holder.transformerParam, tokenizer) + assert(holder.get(holder.transformerParam).contains(tokenizer)) + } + + test("TransformerParam custom validator accepts and rejects per its predicate") { + val holder = new Params { + override val uid: String = "test" + val validatedParam = new TransformerParam( + this, "validated", "validated param", + (t: Transformer) => t.isInstanceOf[Tokenizer] + ) + override def copy(extra: ParamMap): Params = this + } + val tokenizer = new Tokenizer() + holder.set(holder.validatedParam, tokenizer) + assert(holder.get(holder.validatedParam).contains(tokenizer)) + assertThrows[IllegalArgumentException] { + holder.set(holder.validatedParam, new HashingTF()) + } + } + + test("TransformerParam rLoadLine generates correct R code") { + val holder = new TestParamsHolder + val rCode = holder.transformerParam.rLoadLine(1) + assert(rCode.contains("ml_load")) + assert(rCode.contains("model-1.model")) + assert(rCode.contains("complexParams")) + assert(rCode.contains("transformer")) + // Spark 4.1 extracts the stage via sparklyr internals instead of ml_stages; see RCodegenSuite. + assert(rCode.contains("new_ml_pipeline_stage")) + assert(rCode.contains("getStages")) + } + + // EstimatorParam tests + test("EstimatorParam can be created with basic constructor") { + val holder = new TestParamsHolder + assert(holder.estimatorParam.name === "estimator") + assert(holder.estimatorParam.doc === "An estimator param") + } + + test("EstimatorParam accepts valid Estimator") { + val holder = new TestParamsHolder + val indexer = new StringIndexer().setInputCol("label").setOutputCol("indexedLabel") + holder.set(holder.estimatorParam, indexer) + } + + test("EstimatorParam rLoadLine generates correct R code") { + val holder = new TestParamsHolder + val rCode = holder.estimatorParam.rLoadLine(2) + assert(rCode.contains("ml_load")) + assert(rCode.contains("model-2.model")) + assert(rCode.contains("complexParams")) + assert(rCode.contains("estimator")) + } + + // PipelineStageParam tests + test("PipelineStageParam can be created with basic constructor") { + val holder = new TestParamsHolder + assert(holder.pipelineStageParam.name === "pipelineStage") + assert(holder.pipelineStageParam.doc === "A pipeline stage param") + } + + test("PipelineStageParam accepts Transformer") { + val holder = new TestParamsHolder + val tokenizer = new Tokenizer() + holder.set(holder.pipelineStageParam, tokenizer) + } + + test("PipelineStageParam accepts Estimator") { + val holder = new TestParamsHolder + val indexer = new StringIndexer() + holder.set(holder.pipelineStageParam, indexer) + } + + test("PipelineStageParam rLoadLine generates correct R code") { + val holder = new TestParamsHolder + val rCode = holder.pipelineStageParam.rLoadLine(3) + assert(rCode.contains("ml_load")) + assert(rCode.contains("model-3.model")) + assert(rCode.contains("pipelineStage")) + // Spark 4.1 extracts the stage via sparklyr internals instead of ml_stages; see RCodegenSuite. + assert(rCode.contains("new_ml_pipeline_stage")) + assert(rCode.contains("getStages")) + } + + // PipelineStageWrappable trait tests + test("PipelineStageWrappable pyValue returns model reference") { + val holder = new TestParamsHolder + val tokenizer = new Tokenizer() + val pyVal = holder.transformerParam.pyValue(tokenizer) + assert(pyVal === "transformerModel") + } + + test("PipelineStageWrappable pyLoadLine generates Python code") { + val holder = new TestParamsHolder + val pyCode = holder.transformerParam.pyLoadLine(1) + assert(pyCode.contains("Pipeline.load")) + assert(pyCode.contains("model-1.model")) + assert(pyCode.contains("complexParams")) + assert(pyCode.contains("getStages()")) + } + + test("PipelineStageWrappable rValue returns model reference") { + val holder = new TestParamsHolder + val tokenizer = new Tokenizer() + val rVal = holder.transformerParam.rValue(tokenizer) + assert(rVal === "transformerModel") + } + + test("PipelineStageWrappable assertEquality succeeds for same transformer") { + val holder = new TestParamsHolder + val t1 = new Tokenizer().setInputCol("a").setOutputCol("b") + val t2 = new Tokenizer().setInputCol("a").setOutputCol("b") + // Should not throw + holder.transformerParam.assertEquality(t1, t2) + } + + test("PipelineStageWrappable assertEquality throws for non-PipelineStage") { + val holder = new TestParamsHolder + assertThrows[AssertionError] { + holder.transformerParam.assertEquality("not a stage", "also not a stage") + } + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyPythonWrappableParam.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyPythonWrappableParam.scala new file mode 100644 index 00000000000..fc5264333e4 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyPythonWrappableParam.scala @@ -0,0 +1,126 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.param + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.spark.ml.param.{ParamMap, Params} +import spray.json._ +import spray.json.DefaultJsonProtocol._ + +class VerifyPythonWrappableParam extends TestBase { + + test("PythonPrinter converts JsNull to Python None") { + val result = PythonPrinter(JsNull) + assert(result === "None") + } + + test("PythonPrinter converts JsTrue to Python True") { + val result = PythonPrinter(JsTrue) + assert(result === "True") + } + + test("PythonPrinter converts JsFalse to Python False") { + val result = PythonPrinter(JsFalse) + assert(result === "False") + } + + test("PythonPrinter converts JsNumber correctly") { + assert(PythonPrinter(JsNumber(42)) === "42") + assert(PythonPrinter(JsNumber(3.14)) === "3.14") + assert(PythonPrinter(JsNumber(-100)) === "-100") + } + + test("PythonPrinter converts JsString correctly") { + val result = PythonPrinter(JsString("hello")) + assert(result === "\"hello\"") + } + + test("PythonPrinter converts JsArray correctly") { + val arr = JsArray(JsNumber(1), JsNumber(2), JsNumber(3)) + val result = PythonPrinter(arr) + assert(result === "[1,2,3]") + } + + test("PythonPrinter converts JsObject correctly") { + val obj = JsObject("key" -> JsString("value")) + val result = PythonPrinter(obj) + assert(result.contains("key")) + assert(result.contains("value")) + } + + test("PythonPrinter converts nested structures") { + val nested = JsObject( + "bool" -> JsTrue, + "null" -> JsNull, + "number" -> JsNumber(42) + ) + val result = PythonPrinter(nested) + assert(result.contains("True")) + assert(result.contains("None")) + assert(result.contains("42")) + } + + test("pyDefaultRender with JsonFormat") { + val result = PythonWrappableParam.pyDefaultRender("test") + assert(result === "\"test\"") + } + + test("pyDefaultRender with Int") { + val result = PythonWrappableParam.pyDefaultRender(42) + assert(result === "42") + } + + test("pyDefaultRender with Boolean true") { + val result = PythonWrappableParam.pyDefaultRender(true) + assert(result === "True") + } + + test("pyDefaultRender with Boolean false") { + val result = PythonWrappableParam.pyDefaultRender(false) + assert(result === "False") + } + + test("pyDefaultRender with custom jsonFunc") { + val result = PythonWrappableParam.pyDefaultRender( + List(1, 2, 3), + (v: List[Int]) => v.toJson.compactPrint + ) + assert(result === "[1,2,3]") + } + + // Test PythonWrappableParam trait implementation + private class TestPythonParam(parent: Params, override val name: String, doc: String) + extends org.apache.spark.ml.param.Param[String](parent, name, doc) + with PythonWrappableParam[String] + + private class TestParams extends Params { + override val uid: String = "test-uid" + val stringParam = new TestPythonParam(this, "testString", "A test string param") + override def copy(extra: ParamMap): Params = this + } + + test("PythonWrappableParam.pyValue renders value correctly") { + val params = new TestParams + val result = params.stringParam.pyValue("hello") + assert(result === "\"hello\"") + } + + test("PythonWrappableParam.pyName returns param name") { + val params = new TestParams + val result = params.stringParam.pyName("anyValue") + assert(result === "testString") + } + + test("PythonWrappableParam.pyConstructorLine generates correct format") { + val params = new TestParams + val result = params.stringParam.pyConstructorLine("world") + assert(result === "testString=\"world\"") + } + + test("PythonWrappableParam.pySetterLine generates correct format") { + val params = new TestParams + val result = params.stringParam.pySetterLine("value") + assert(result === "setTestString(\"value\")") + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyRWrappableParam.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyRWrappableParam.scala new file mode 100644 index 00000000000..ef069b43669 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyRWrappableParam.scala @@ -0,0 +1,152 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.param + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.spark.ml.param.{ParamMap, Params} +import spray.json._ +import spray.json.DefaultJsonProtocol._ + +class VerifyRWrappableParam extends TestBase { + + test("RPrinter converts JsNull to R NULL") { + val result = RPrinter(JsNull) + assert(result === "NULL") + } + + test("RPrinter converts JsTrue to R TRUE") { + val result = RPrinter(JsTrue) + assert(result === "TRUE") + } + + test("RPrinter converts JsFalse to R FALSE") { + val result = RPrinter(JsFalse) + assert(result === "FALSE") + } + + test("RPrinter converts integer JsNumber with L suffix") { + val result = RPrinter(JsNumber(42)) + assert(result === "42L") + } + + test("RPrinter converts double JsNumber without L suffix") { + val result = RPrinter(JsNumber(3.14)) + assert(result === "3.14") + } + + test("RPrinter converts JsString correctly") { + val result = RPrinter(JsString("hello")) + assert(result === "\"hello\"") + } + + test("RPrinter converts empty JsArray to c()") { + val arr = JsArray() + val result = RPrinter(arr) + assert(result === "c()") + } + + test("RPrinter converts JsArray of numbers to list()") { + val arr = JsArray(JsNumber(1), JsNumber(2), JsNumber(3)) + val result = RPrinter(arr) + assert(result === "list(1L,2L,3L)") + } + + test("RPrinter converts empty JsObject to c()") { + val obj = JsObject() + val result = RPrinter(obj) + assert(result === "c()") + } + + test("RPrinter converts JsObject to list2env") { + val obj = JsObject("key" -> JsString("value")) + val result = RPrinter(obj) + assert(result.contains("list2env")) + assert(result.contains("key")) + assert(result.contains("value")) + } + + test("RPrinter converts nested JsObject correctly") { + val nested = JsObject( + "bool" -> JsTrue, + "null" -> JsNull, + "number" -> JsNumber(42) + ) + val result = RPrinter(nested) + assert(result.contains("TRUE")) + assert(result.contains("NULL")) + assert(result.contains("42L")) + } + + test("RPrinter converts JsArray of JsObjects correctly") { + val arr = JsArray( + JsObject("a" -> JsNumber(1)), + JsObject("b" -> JsNumber(2)) + ) + val result = RPrinter(arr) + assert(result.contains("list2env")) + } + + test("rDefaultRender with JsonFormat for String") { + val result = RWrappableParam.rDefaultRender("test") + assert(result === "\"test\"") + } + + test("rDefaultRender with JsonFormat for Int") { + val result = RWrappableParam.rDefaultRender(42) + assert(result === "42L") + } + + test("rDefaultRender with JsonFormat for Boolean true") { + val result = RWrappableParam.rDefaultRender(true) + assert(result === "TRUE") + } + + test("rDefaultRender with JsonFormat for Boolean false") { + val result = RWrappableParam.rDefaultRender(false) + assert(result === "FALSE") + } + + test("rDefaultRender with custom jsonFunc") { + val result = RWrappableParam.rDefaultRender( + List(1, 2, 3), + (v: List[Int]) => v.toJson.compactPrint + ) + assert(result === "list(1L,2L,3L)") + } + + // Test RWrappableParam trait implementation + private class TestRParam(parent: Params, override val name: String, doc: String) + extends org.apache.spark.ml.param.Param[String](parent, name, doc) + with RWrappableParam[String] + + private class TestParams extends Params { + override val uid: String = "test-uid" + val stringParam = new TestRParam(this, "testString", "A test string param") + override def copy(extra: ParamMap): Params = this + } + + test("RWrappableParam.rValue renders value correctly") { + val params = new TestParams + val result = params.stringParam.rValue("hello") + assert(result === "\"hello\"") + } + + test("RWrappableParam.rName returns param name") { + val params = new TestParams + val result = params.stringParam.rName("anyValue") + assert(result === "testString") + } + + test("RWrappableParam.rConstructorLine generates correct format") { + val params = new TestParams + val result = params.stringParam.rConstructorLine("world") + assert(result === "testString=\"world\"") + } + + test("RWrappableParam.rSetterLine generates correct format") { + val params = new TestParams + val result = params.stringParam.rSetterLine("value") + assert(result === "setTestString(\"value\")") + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyUDFParam.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyUDFParam.scala new file mode 100644 index 00000000000..4971e0c3753 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/param/VerifyUDFParam.scala @@ -0,0 +1,107 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.param + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.spark.ml.param.{ParamMap, Params} +import org.apache.spark.sql.expressions.UserDefinedFunction +import org.apache.spark.sql.functions.udf + +class VerifyUDFParam extends TestBase { + + private class TestParamsHolder extends Params { + override val uid: String = "test-holder" + val udfParam = new UDFParam(this, "udf", "A UDF param") + override def copy(extra: ParamMap): Params = this + } + + test("UDFParam can be created with basic constructor") { + val holder = new TestParamsHolder + assert(holder.udfParam.name === "udf") + assert(holder.udfParam.doc === "A UDF param") + } + + test("UDFParam accepts simple UDF") { + val holder = new TestParamsHolder + val myUdf = udf((x: Int) => x * 2) + holder.set(holder.udfParam, myUdf) + assert(holder.isSet(holder.udfParam)) + } + + test("UDFParam accepts string transformation UDF") { + val holder = new TestParamsHolder + val myUdf = udf((s: String) => s.toUpperCase) + holder.set(holder.udfParam, myUdf) + assert(holder.isSet(holder.udfParam)) + } + + test("UDFParam accepts multi-argument UDF") { + val holder = new TestParamsHolder + val myUdf = udf((a: Int, b: Int) => a + b) + holder.set(holder.udfParam, myUdf) + assert(holder.isSet(holder.udfParam)) + } + + test("UDFParam with custom validator rejects values the validator refuses") { + val holder = new Params { + override val uid: String = "test" + // Only accept UDFs, and only when the validator agrees; here nothing is acceptable + val validatedUdf = new UDFParam( + this, "validated", "Validated UDF", + (_: UserDefinedFunction) => false + ) + override def copy(extra: ParamMap): Params = this + } + val myUdf = udf((x: Double) => x * x) + assertThrows[IllegalArgumentException] { + holder.set(holder.validatedUdf, myUdf) + } + } + + test("UDFParam with custom validator accepts values the validator allows") { + val holder = new Params { + override val uid: String = "test" + val validatedUdf = new UDFParam( + this, "validated", "Validated UDF", + (_: UserDefinedFunction) => true + ) + override def copy(extra: ParamMap): Params = this + } + val myUdf = udf((x: Double) => x * x) + holder.set(holder.validatedUdf, myUdf) + assert(holder.isSet(holder.validatedUdf)) + } + + test("UDFParam can be cleared") { + val holder = new TestParamsHolder + val myUdf = udf((x: Int) => x) + holder.set(holder.udfParam, myUdf) + assert(holder.isSet(holder.udfParam)) + holder.clear(holder.udfParam) + assert(!holder.isSet(holder.udfParam)) + } + + test("UDFParam returns None when not set") { + val holder = new TestParamsHolder + assert(holder.get(holder.udfParam).isEmpty) + } + + test("UDFParam assertEquality accepts the same UDF and rejects mismatched ones") { + val holder = new TestParamsHolder + val intUdf = udf((x: Int) => x * 2) + val stringUdf = udf((x: Int) => x.toString) + holder.udfParam.assertEquality(intUdf, intUdf) + // UDFs with different return types must not compare equal + assertThrows[AssertionError] { + holder.udfParam.assertEquality(intUdf, stringUdf) + } + } + + test("UDFParam assertEquality throws for non-UDF types") { + val holder = new TestParamsHolder + assertThrows[AssertionError] { + holder.udfParam.assertEquality("not a udf", "also not a udf") + } + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/stages/EnsembleByKeyResolutionSuite.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/stages/EnsembleByKeyResolutionSuite.scala new file mode 100644 index 00000000000..517b8efbaa8 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/stages/EnsembleByKeyResolutionSuite.scala @@ -0,0 +1,163 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.stages + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.spark.SparkException +import org.apache.spark.ml.Pipeline +import org.apache.spark.ml.linalg.{SQLDataTypes, Vector} +import org.apache.spark.sql.functions.{col, struct} +import org.apache.spark.sql.types.{DoubleType, IntegerType, Metadata, StringType, StructField, StructType} +import org.apache.spark.sql.{AnalysisException, DataFrame, Row} + +/** Covers the duplicate attribute resolution rules that EnsembleByKey mirrors from Spark's + * `AttributeSeq.resolve`. + */ +class EnsembleByKeyResolutionSuite extends TestBase { + + private val duplicateKey = "__is_duplicate" + + test("custom stage identifiers should not affect internal column resolution") { + val input = spark.createDataFrame(Seq(("group", 1.0), ("group", 3.0))).toDF("key", "score") + Seq("ensemble.by.key", "ensemble`by`key").foreach { uid => + val transformer = new EnsembleByKey(uid) + .setKey("key").setCol("score").setCollapseGroup(false) + assert(transformer.transformSchema(input.schema) === transformer.transform(input).schema) + } + } + + test("non-collapsed output should retain rows with null grouping keys") { + val schema = StructType(Array( + StructField("id", IntegerType, nullable = false), + StructField("key", StringType), + StructField("score", DoubleType, nullable = false))) + val missingKey = Option.empty[String].orNull + val input = spark.createDataFrame(java.util.Arrays.asList( + Row(0, missingKey, 1.0), + Row(1, missingKey, 3.0), + Row(2, "group", 5.0)), schema) + val transformed = new EnsembleByKey() + .setKey("key").setCol("score").setCollapseGroup(false).transform(input) + + assert(transformed.orderBy("id").collect().map(row => + (row.getInt(1), Option(row.getString(0)), row.getDouble(3))) === + Array((0, None, 2.0), (1, None, 2.0), (2, Some("group"), 5.0))) + } + + test("vector mean schema should match Spark for all-null inputs") { + val schema = StructType(Array( + StructField("key", StringType, nullable = false), + StructField("features", SQLDataTypes.VectorType))) + val missingVector = Option.empty[Vector].orNull + val input = spark.createDataFrame(java.util.Arrays.asList( + Row("group", missingVector), + Row("group", missingVector)), schema) + val transformer = new EnsembleByKey().setKey("key").setCol("features") + val transformed = transformer.transform(input) + + assert(transformer.transformSchema(input.schema) === transformed.schema) + assert(!transformed.schema("mean(features)").nullable) + intercept[SparkException](transformed.collect()) + } + + test("duplicated qualifier attributes should follow Spark expression identity") { + val base = spark.createDataFrame(Seq(("top", "nested", 1.0), ("top", "nested", 3.0))) + .toDF("group", "nestedGroup", "score") + val nestedGroup = struct(col("nestedGroup").alias("group")).alias("dup") + val shared = base.select(col("group"), col("group"), nestedGroup, col("score")) + val transformer = new EnsembleByKey().setKey("dup.group").setCol("score") + + assert(distinctExpressions(shared, "group") === 1) + Seq("dup" -> "top", "other" -> "nested").foreach { case (alias, expected) => + val transformed = assertSchemaAgrees(transformer, shared.as(alias)) + withClue(s"$alias: ") { + assert(transformed.head().getString(0) === expected) + assert(transformed.select("mean(score)").head().getDouble(0) === 2.0) + } + } + + val ambiguous = base.select(col("group"), nestedGroup, col("score")).as("dup") + .crossJoin(spark.createDataFrame(Seq(Tuple1("side"))).toDF("group").as("dup")) + assert(distinctExpressions(ambiguous, "group") === 2) + intercept[AnalysisException](ambiguous.select("dup.group")) + val error = intercept[IllegalArgumentException](transformer.transform(ambiguous)) + assert(error.getMessage.contains("dup.group is ambiguous")) + } + + test("duplicated unqualified attributes sharing one expression should aggregate") { + val base = spark.createDataFrame(Seq(("group", 1.0), ("group", 3.0))).toDF("key", "score") + val duplicated = base.select(col("key"), col("score"), col("score")) + val transformer = new EnsembleByKey().setKey("key").setCol("score") + + assert(duplicated.schema.fieldNames === Array("key", "score", "score")) + assert(distinctExpressions(duplicated, "score") === 1) + assert(duplicated.select("score").columns === Array("score")) + + val transformed = transformer.transform(duplicated) + assert(transformed.schema.fieldNames === Array("key", "mean(score)")) + assert(transformed.schema("mean(score)") === StructField("mean(score)", DoubleType)) + assert(transformed.head().getDouble(1) === 2.0) + + assert(transformer.transformSchema(duplicated.schema) === transformed.schema) + val pipelineModel = new Pipeline().setStages(Array(transformer)).fit(duplicated) + assert(pipelineModel.transform(duplicated).collect() === transformed.collect()) + } + + test("union duplicate attributes should follow Spark duplicate pruning") { + val base = spark.createDataFrame(Seq(("group", 1.0), ("group", 3.0))).toDF("key", "score") + val duplicated = base.select(col("key"), col("score"), col("score")) + val unioned = duplicated.union(duplicated) + assert(unioned.schema.fieldNames === Array("key", "score", "score")) + assert(unioned.schema.fields.last.metadata.contains(duplicateKey)) + assert(distinctExpressions(unioned, "score") === 2) + assert(unioned.select("score").columns === Array("score")) + + val transformed = assertSchemaAgrees(new EnsembleByKey().setKey("key").setCol("score"), unioned) + assert(transformed.schema.fieldNames === Array("key", "mean(score)")) + assert(transformed.head().getDouble(1) === 2.0) + + val qualified = assertSchemaAgrees( + new EnsembleByKey().setKey("key").setCol("u.score"), unioned.as("u")) + assert(qualified.schema.fieldNames === Array("key", "mean(u.score)")) + assert(qualified.head().getDouble(1) === 2.0) + } + + test("duplicate pruning should not override qualifier selection") { + // The only `group` attribute of `u` carries Spark's duplicate marker while `v.group` does not, + // so pruning before qualifier selection would silently resolve `u.group` to `v.group`. + val base = spark.createDataFrame(Seq(("u", 1.0), ("u", 3.0))).toDF("group", "score") + val duplicated = base.select(col("group"), col("group"), col("score")) + val tagged = duplicated.union(duplicated).toDF("other", "group", "score") + assert(tagged.schema("group").metadata.contains(duplicateKey)) + assert(!tagged.schema("other").metadata.contains(duplicateKey)) + + val untagged = spark.createDataFrame(Seq(Tuple1("v"))).toDF("group") + val joined = tagged.as("u").crossJoin(untagged.as("v")) + assert(joined.schema.fieldNames === Array("other", "group", "score", "group")) + assert(joined.select("u.group").head().getString(0) === "u") + + val transformed = assertSchemaAgrees( + new EnsembleByKey().setKey("u.group").setCol("score"), joined) + assert(transformed.schema.fieldNames === Array("group", "mean(score)")) + assert(transformed.schema("group").metadata === Metadata.empty) + assert(transformed.head().getString(0) === "u") + assert(transformed.head().getDouble(1) === 2.0) + + val nonCollapsed = new EnsembleByKey().setKey("u.group").setCol("score").setCollapseGroup(false) + val schemaError = intercept[IllegalArgumentException](nonCollapsed.transformSchema(joined.schema)) + val transformError = intercept[IllegalArgumentException](nonCollapsed.transform(joined)) + assert(schemaError.getMessage.contains("multiple columns are named group")) + assert(transformError.getMessage.contains("multiple columns are named group")) + } + + private def distinctExpressions(input: DataFrame, name: String): Int = { + input.queryExecution.analyzed.output.filter(_.name == name).map(_.exprId).distinct.length + } + + private def assertSchemaAgrees(transformer: EnsembleByKey, input: DataFrame): DataFrame = { + val transformed = transformer.transform(input) + assert(transformer.transformSchema(input.schema) === transformed.schema) + transformed + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/stages/EnsembleByKeySuite.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/stages/EnsembleByKeySuite.scala index 1a624cf4431..fb3a022994d 100644 --- a/core/src/test/scala/com/microsoft/azure/synapse/ml/stages/EnsembleByKeySuite.scala +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/stages/EnsembleByKeySuite.scala @@ -5,9 +5,14 @@ package com.microsoft.azure.synapse.ml.stages import com.microsoft.azure.synapse.ml.core.test.base.TestBase import com.microsoft.azure.synapse.ml.core.test.fuzzing.{TestObject, TransformerFuzzing} +import org.apache.spark.ml.Pipeline import org.apache.spark.ml.feature.VectorAssembler -import org.apache.spark.ml.linalg.DenseVector -import org.apache.spark.sql.DataFrame +import org.apache.spark.ml.linalg.{DenseVector, SQLDataTypes} +import org.apache.spark.sql.{AnalysisException, DataFrame, Row, SparkSession} +import org.apache.spark.sql.catalyst.expressions.{Cast, RowOrdering} +import org.apache.spark.sql.functions.{array, col, expr, lit, map, struct} +import org.apache.spark.sql.types.{CalendarIntervalType, DoubleType, MapType, Metadata, StringType, + StructField, StructType} class EnsembleByKeySuite extends TestBase with TransformerFuzzing[EnsembleByKey] { @@ -53,6 +58,629 @@ class EnsembleByKeySuite extends TestBase with TransformerFuzzing[EnsembleByKey] df1.show() } + test("transformSchema should match mixed aggregate output for default and explicit names") { + val input = mixedTypeDF + val inputNames = Array("doubleScore", "floatScore", "features") + val defaultNames = inputNames.map(name => s"mean($name)") + val explicitNames = Array("averageDouble", "averageFloat", "averageFeatures") + val keyNames = Array("group", "region") + + assert(input.schema("features").metadata !== Metadata.empty) + + Seq(defaultNames -> false, explicitNames -> true).foreach { case (outputNames, useExplicitNames) => + Seq(true, false).foreach { collapseGroup => + val transformer = new EnsembleByKey() + .setKeys(keyNames).setCols(inputNames).setCollapseGroup(collapseGroup) + if (useExplicitNames) { + transformer.setColNames(outputNames) + } + + val transformedSchema = transformer.transformSchema(input.schema) + val actualSchema = transformer.transform(input).schema + val expectedNames = if (collapseGroup) { + keyNames ++ outputNames + } else { + keyNames ++ input.columns.filterNot((keyNames ++ outputNames).contains) ++ outputNames + } + + withClue(s"explicitNames=$useExplicitNames, collapseGroup=$collapseGroup: ") { + assert(transformedSchema === actualSchema) + assert(actualSchema.fieldNames === expectedNames) + assert(actualSchema(outputNames(0)) === StructField(outputNames(0), DoubleType)) + assert(actualSchema(outputNames(1)) === StructField(outputNames(1), DoubleType)) + assert(actualSchema(outputNames(2)) === + StructField(outputNames(2), SQLDataTypes.VectorType, nullable = false)) + } + } + } + } + + test("non-collapsed output should overwrite numeric and vector columns") { + val input = mixedTypeDF + val overwrittenNames = Array("doubleScore", "floatScore", "features") + val transformer = new EnsembleByKey() + .setKeys("group", "region").setCols(overwrittenNames) + .setColNames(overwrittenNames).setCollapseGroup(false) + + val transformedSchema = transformer.transformSchema(input.schema) + val transformed = transformer.transform(input) + + assert(transformed.schema === transformedSchema) + assert(transformed.columns === + Array("group", "region", "id", "component1", "component2") ++ overwrittenNames) + assert(transformed.schema("features").metadata === Metadata.empty) + assert(!transformed.schema("features").nullable) + + val actual = transformed.orderBy("id") + .select("doubleScore", "floatScore", "features") + .collect() + .map(row => (row.getDouble(0), row.getDouble(1), row.getAs[DenseVector](2))) + val expected = Array( + (1.0, 1.0, new DenseVector(Array(1.0, 0.1))), + (2.0, 2.0, new DenseVector(Array(2.0, -2.5))), + (2.0, 2.0, new DenseVector(Array(2.0, -2.5)))) + + assert(actual === expected) + } + + test("non-collapsed output should replace case-variant columns consistently") { + val input = spark.createDataFrame(Seq((0, "group", 1.0, "lower", "upper"))) + .toDF("id", "key", "score", "features", "FEATURES") + + Seq(false -> Array("key", "id", "score", "features"), + true -> Array("key", "id", "score", "FEATURES", "features")) + .foreach { case (caseSensitive, expectedNames) => + withCaseSensitiveAnalysis(caseSensitive) { + val transformer = new EnsembleByKey() + .setKey("key").setCol("score").setColName("features").setCollapseGroup(false) + + val transformedSchema = transformer.transformSchema(input.schema) + val actualSchema = transformer.transform(input).schema + + assert(transformedSchema === actualSchema) + assert(actualSchema.fieldNames === expectedNames) + } + } + } + + test("default output names should follow updated input columns before transform") { + val transformer = new EnsembleByKey().setKeys("group", "region").setCol("doubleScore") + assert(transformer.getDefault(transformer.colNames).isEmpty) + transformer.transformSchema(mixedTypeDF.schema) + assert(transformer.getDefault(transformer.colNames).isEmpty) + assert(transformer.getColNames === Array("mean(doubleScore)")) + transformer.transform(mixedTypeDF) + assert(transformer.getDefault(transformer.colNames).isEmpty) + transformer.setCols("doubleScore", "floatScore") + + assert(transformer.transformSchema(mixedTypeDF.schema).fieldNames === + Array("group", "region", "mean(doubleScore)", "mean(floatScore)")) + assert(transformer.getColNames === Array("mean(doubleScore)", "mean(floatScore)")) + } + + test("grouping keys should resolve case-insensitively to input field names") { + withCaseSensitiveAnalysis(false) { + Seq(true, false).foreach { collapseGroup => + val transformer = new EnsembleByKey() + .setKeys("GROUP", "REGION").setCol("doubleScore").setCollapseGroup(collapseGroup) + + val transformedSchema = transformer.transformSchema(mixedTypeDF.schema) + val actualSchema = transformer.transform(mixedTypeDF).schema + + withClue(s"collapseGroup=$collapseGroup: ") { + assert(transformedSchema === actualSchema) + assert(actualSchema.fieldNames.take(2) === Array("GROUP", "REGION")) + } + } + } + } + + test("grouping key resolution should honor case-sensitive analysis") { + withCaseSensitiveAnalysis(true) { + val input = spark.createDataFrame(Seq(("lower", "upper", 1.0))) + .toDF("group", "GROUP", "score") + val transformer = new EnsembleByKey().setKey("group").setCol("score") + + assert(transformer.transformSchema(input.schema) === transformer.transform(input).schema) + + val error = intercept[IllegalArgumentException] { + new EnsembleByKey().setKey("Group").setCol("score").transformSchema(input.schema) + } + assert(error.getMessage.contains("Group does not exist")) + } + } + + test("transformSchema should match output when grouping column retention is disabled") { + withSQLConf("spark.sql.retainGroupColumns", "false") { + Seq(true, false).foreach { collapseGroup => + val transformer = new EnsembleByKey() + .setKeys("group", "region").setCol("doubleScore").setCollapseGroup(collapseGroup) + val transformedSchema = transformer.transformSchema(mixedTypeDF.schema) + val actualSchema = transformer.transform(mixedTypeDF).schema + + assert(transformedSchema === actualSchema) + assert(actualSchema.fieldNames.take(2) === Array("group", "region")) + } + } + } + + test("transform should use the dataset session for grouping column retention") { + val disabledSession = spark.newSession() + disabledSession.conf.set("spark.sql.retainGroupColumns", false) + val disabledInput = disabledSession.createDataFrame(Seq(("group", 1.0))).toDF("group", "score") + + withActiveSession(spark) { + val transformer = new EnsembleByKey().setKey("group").setCol("score") + assert(transformer.transformSchema(disabledInput.schema) === transformer.transform(disabledInput).schema) + } + + val enabledSession = spark.newSession() + enabledSession.conf.set("spark.sql.retainGroupColumns", true) + val enabledInput = enabledSession.createDataFrame(Seq(("group", 1.0))).toDF("group", "score") + + withSQLConf("spark.sql.retainGroupColumns", "false") { + withActiveSession(spark) { + val transformer = new EnsembleByKey().setKey("group").setCol("score") + val transformed = transformer.transform(enabledInput) + val pipelineModel = new Pipeline().setStages(Array(transformer)).fit(enabledInput) + + assert(transformer.transformSchema(enabledInput.schema) === transformed.schema) + assert(pipelineModel.transform(enabledInput).schema === transformed.schema) + assert(transformed.columns === Array("group", "mean(score)")) + } + } + } + + test("configuration parsing should match Spark boolean parsing") { + withSQLConf("spark.sql.caseSensitive", " false ") { + val transformer = new EnsembleByKey().setKey("GROUP").setCol("doubleScore") + assert(transformer.transformSchema(mixedTypeDF.schema) === transformer.transform(mixedTypeDF).schema) + } + + withSQLConf("spark.sql.retainGroupColumns", " true ") { + val transformer = new EnsembleByKey().setKey("group").setCol("doubleScore") + assert(transformer.transformSchema(mixedTypeDF.schema) === transformer.transform(mixedTypeDF).schema) + } + + withSQLConf("spark.sql.retainGroupColumns", " false ") { + val transformer = new EnsembleByKey().setKey("group").setCol("doubleScore") + assert(transformer.transformSchema(mixedTypeDF.schema) === transformer.transform(mixedTypeDF).schema) + } + } + + test("no active session should expose the documented case-resolution limitation") { + withSQLConf("spark.sql.caseSensitive", "true") { + val input = spark.createDataFrame(Seq((0, "group", 1.0, 2.0, 3.0))) + .toDF("id", "key", "score", "features", "FEATURES") + val transformer = new EnsembleByKey() + .setKey("key").setCol("score").setColName("features").setCollapseGroup(false) + val assembler = new VectorAssembler() + .setInputCols(Array("FEATURES")).setOutputCol("vector") + val pipeline = new Pipeline().setStages(Array(transformer, assembler)) + + withoutActiveSession { + val transformedSchema = transformer.transformSchema(input.schema) + val actualSchema = transformer.transform(input).schema + + assert(transformedSchema.fieldNames === Array("key", "id", "score", "features")) + assert(actualSchema.fieldNames === Array("key", "id", "score", "FEATURES", "features")) + // Assert the limitation on the transformed schema rather than on a downstream stage. + // Spark 3.5 rejected pipeline.fit here because VectorAssembler resolved its input columns + // with a case-sensitive StructType lookup, while Spark 4 routes that lookup through + // SQLConf.get.resolver and so applies the same session-less case-insensitive fallback + // used above -- the two agree and the pipeline builds. The schema itself is the stable + // contract across both versions: the fallback really did drop FEATURES. + val fieldError = intercept[IllegalArgumentException](transformedSchema("FEATURES")) + assert(fieldError.getMessage.contains("FEATURES")) + } + pipeline.fit(input) + } + } + + test("transform should use the dataset session for column resolution") { + val sensitiveSession = spark.newSession() + sensitiveSession.conf.set("spark.sql.caseSensitive", true) + val sensitiveInput = sensitiveSession.createDataFrame(Seq(("group", 1.0))).toDF("group", "score") + + withCaseSensitiveAnalysis(false) { + val transformer = new EnsembleByKey().setKey("GROUP").setCol("SCORE") + assert(transformer.transformSchema(sensitiveInput.schema).fieldNames === Array("GROUP", "mean(SCORE)")) + assert(intercept[IllegalArgumentException](transformer.transform(sensitiveInput)) + .getMessage.contains("does not exist")) + } + + val insensitiveSession = spark.newSession() + insensitiveSession.conf.set("spark.sql.caseSensitive", false) + val insensitiveInput = insensitiveSession.createDataFrame(Seq(("group", 1.0))).toDF("group", "score") + + withCaseSensitiveAnalysis(true) { + val transformer = new EnsembleByKey().setKey("GROUP").setCol("SCORE") + assert(transformer.transform(insensitiveInput).schema.fieldNames === Array("GROUP", "mean(SCORE)")) + } + + withoutActiveSession { + val transformer = new EnsembleByKey().setKey("GROUP").setCol("SCORE") + assert(intercept[IllegalArgumentException](transformer.transform(sensitiveInput)) + .getMessage.contains("does not exist")) + } + } + + test("nested and quoted field references should match Spark resolution") { + val nestedInput = spark.createDataFrame(Seq(("a", 1.0), ("a", 3.0))) + .toDF("nestedKey", "score") + .select(struct(col("nestedKey").alias("key")).alias("nested"), col("score")) + + Seq(true, false).foreach { collapseGroup => + val transformer = new EnsembleByKey() + .setKey("nested.key").setCol("score").setCollapseGroup(collapseGroup) + val transformedSchema = transformer.transformSchema(nestedInput.schema) + val transformed = transformer.transform(nestedInput) + + assert(transformedSchema === transformed.schema) + assert(transformed.schema.fieldNames.head === "key") + assert(transformed.select("mean(score)").head().getDouble(0) === 2.0) + } + + val dottedInput = spark.createDataFrame(Seq(("a", 1.0), ("a", 3.0))).toDF("a.b", "score") + val dottedTransformer = new EnsembleByKey().setKey("`a.b`").setCol("score") + + assert(dottedTransformer.transformSchema(dottedInput.schema) === dottedTransformer.transform(dottedInput).schema) + } + + test("nested key nullability should include nullable ancestor structs") { + val inputSchema = StructType(Array( + StructField( + "nested", + StructType(Array(StructField("key", StringType, nullable = false))), + nullable = true), + StructField("score", DoubleType, nullable = false))) + val rows = java.util.Arrays.asList( + Row(Row("a"), 1.0), + Row(Row("a"), 3.0)) + val input = spark.createDataFrame(rows, inputSchema) + + Seq(true, false).foreach { collapseGroup => + val transformer = new EnsembleByKey() + .setKey("nested.key").setCol("score").setCollapseGroup(collapseGroup) + val transformedSchema = transformer.transformSchema(input.schema) + val actualSchema = transformer.transform(input).schema + + assert(transformedSchema === actualSchema) + assert(actualSchema("key").nullable) + } + } + + test("non-collapsed nested keys should reject unsafe leaf-name collisions") { + val collisionInput = spark.createDataFrame(Seq(("row-1", "group", 1.0))) + .toDF("id", "nestedId", "score") + .select(col("id"), struct(col("nestedId").alias("id")).alias("meta"), col("score")) + val collisionTransformer = new EnsembleByKey() + .setKey("meta.id").setCol("score").setCollapseGroup(false) + + assertConsistentSchemaError( + collisionTransformer, collisionInput, "ambiguous between a nested field and a dataset qualifier") + + val duplicateInput = spark.createDataFrame(Seq(("left", "right", 1.0))) + .toDF("leftKey", "rightKey", "score") + .select( + struct(col("leftKey").alias("key")).alias("left"), + struct(col("rightKey").alias("key")).alias("right"), + col("score")) + val duplicateTransformer = new EnsembleByKey() + .setKeys("left.key", "right.key").setCol("score").setCollapseGroup(false) + + assertConsistentSchemaError(duplicateTransformer, duplicateInput, "must resolve to distinct output columns") + } + + test("non-collapsed duplicate grouping keys should fail consistently") { + Seq("true", "false").foreach { retainGroupColumns => + withSQLConf("spark.sql.retainGroupColumns", retainGroupColumns) { + val transformer = new EnsembleByKey() + .setKeys("group", "group").setCol("doubleScore").setCollapseGroup(false) + + assertConsistentSchemaError( + transformer, + mixedTypeDF, + "must resolve to distinct output columns") + } + } + + val collapsed = new EnsembleByKey() + .setKeys("group", "group").setCol("doubleScore").setCollapseGroup(true) + assert(collapsed.transformSchema(mixedTypeDF.schema) === collapsed.transform(mixedTypeDF).schema) + } + + test("nested keys should preserve unreferenced duplicate top-level columns") { + val input = spark.createDataFrame(Seq(("group", 1.0, 10.0))) + .toDF("key", "score", "duplicate") + .select( + struct(col("key").alias("value")).alias("nested"), + col("score"), + col("duplicate").alias("duplicate"), + col("duplicate").alias("duplicate")) + val transformer = new EnsembleByKey() + .setKey("nested.value").setCol("score").setCollapseGroup(false) + + val transformed = transformer.transform(input) + assert(transformer.transformSchema(input.schema) === transformed.schema) + assert(transformed.schema.fieldNames === + Array("value", "nested", "score", "duplicate", "duplicate", "mean(score)")) + } + + test("quoted field references should ignore quoted-regex column settings") { + withSQLConf("spark.sql.parser.quotedRegexColumnNames", "true") { + val keyInput = spark.createDataFrame(Seq(("a", 1.0), ("a", 3.0))).toDF("a.b", "score") + val keyTransformer = new EnsembleByKey().setKey("`a.b`").setCol("score") + assert(keyTransformer.transformSchema(keyInput.schema) === keyTransformer.transform(keyInput).schema) + + val colInput = spark.createDataFrame(Seq(("group", 1.0), ("group", 3.0))).toDF("group", "s.c") + val colTransformer = new EnsembleByKey().setKey("group").setCol("`s.c`") + assert(colTransformer.transformSchema(colInput.schema) === colTransformer.transform(colInput).schema) + } + } + + test("literal dotted aggregate columns should require Spark quoting") { + val input = spark.createDataFrame(Seq(("group", 1.0), ("group", 3.0))).toDF("group", "s.c") + val quotedTransformer = new EnsembleByKey().setKey("group").setCol("`s.c`") + + assert(quotedTransformer.transformSchema(input.schema) === quotedTransformer.transform(input).schema) + + val plainTransformer = new EnsembleByKey().setKey("group").setCol("s.c") + assertConsistentSchemaError(plainTransformer, input, "s.c does not exist") + } + + test("qualified and collection field references should match Spark resolution") { + val qualifiedInput = mixedTypeDF.as("source") + val qualifiedTransformer = new EnsembleByKey().setKey("source.group").setCol("doubleScore") + assert(qualifiedTransformer.transformSchema(qualifiedInput.schema) === + qualifiedTransformer.transform(qualifiedInput).schema) + + val collectionBase = spark.createDataFrame(Seq(("group", 1.0))).toDF("key", "score") + val arrayInput = collectionBase.select( + array(struct(col("key").alias("field"))).alias("items"), + col("score")) + val arrayTransformer = new EnsembleByKey().setKey("items.field").setCol("score") + val arrayResult = arrayTransformer.transform(arrayInput) + assert(arrayTransformer.transformSchema(arrayInput.schema) === arrayResult.schema) + assert(arrayResult.collect().head.getSeq[String](0) === Seq("group")) + + val nullableArrayInput = collectionBase.select( + array(struct(expr("CAST(NULL AS STRING)").alias("field"))).alias("items"), + col("score")) + val nullableArrayTransformer = new EnsembleByKey().setKey("items.field").setCol("score") + val nullableArrayResult = nullableArrayTransformer.transform(nullableArrayInput) + assert(nullableArrayTransformer.transformSchema(nullableArrayInput.schema) === nullableArrayResult.schema) + assert(Option(nullableArrayResult.collect().head.getSeq[String](0).head).isEmpty) + + val mapInput = collectionBase.select( + map(lit("field"), col("key")).alias("values"), + col("score")) + val mapTransformer = new EnsembleByKey().setKey("values.field").setCol("score") + assert(mapTransformer.transformSchema(mapInput.schema) === mapTransformer.transform(mapInput).schema) + + val invalidMapInput = collectionBase.select( + map(struct(lit(1).alias("part")), col("key")).alias("values"), + col("score")) + val invalidMapTransformer = new EnsembleByKey().setKey("values.field").setCol("score") + assertConsistentSchemaError(invalidMapTransformer, invalidMapInput, "does not accept string keys") + } + + test("map key extraction should follow Spark cast coercion") { + val base = spark.createDataFrame(Seq(("group", 1.0), ("group", 3.0))).toDF("key", "score") + Seq( + "values.true" -> map(lit(true), col("key")), + "values.field" -> map(lit("field").cast("binary"), col("key")), + "values.1" -> map(lit(1), col("key")), + "values.2020-01-01" -> map(lit("2020-01-01").cast("date"), col("key")) + ).foreach { case (reference, values) => + val input = base.select(values.alias("values"), col("score")) + val transformed = assertSchemaAgrees(new EnsembleByKey().setKey(reference).setCol("score"), input) + withClue(s"$reference: ") { + assert(transformed.head().getString(0) === "group") + assert(transformed.select("mean(score)").head().getDouble(0) === 2.0) + } + } + } + + test("map keys Spark cannot order should be rejected consistently") { + val base = spark.createDataFrame(Seq(("group", 1.0), ("group", 3.0))).toDF("key", "score") + val input = base.select( + map(expr("make_interval(0, 0, 0, 1, 0, 0, 0)"), col("key")).alias("values"), + col("score")) + val keyType = input.schema("values").dataType.asInstanceOf[MapType].keyType + + assert(keyType === CalendarIntervalType) + assert(Cast.canCast(StringType, keyType), "the key type is castable from a string literal") + assert(!RowOrdering.isOrderable(keyType), "the key type is not orderable, so GetMapValue fails") + intercept[AnalysisException](input.select(expr("values[make_interval(0, 0, 0, 1, 0, 0, 0)]")).schema) + + val transformer = new EnsembleByKey().setKey("values.1 days").setCol("score") + assertConsistentSchemaError(transformer, input, "map key type CalendarIntervalType is not orderable") + assertConsistentSchemaError(transformer, input, "Use a map column whose key type is orderable") + } + + test("extracted grouping values Spark cannot order should be rejected consistently") { + val input = spark.range(1).select( + map(lit("outer"), map(lit("inner"), lit(1))).alias("values"), + lit(1.0).alias("score")) + val transformer = new EnsembleByKey().setKey("values.outer").setCol("score") + + assertConsistentSchemaError(transformer, input, "Spark cannot use as a grouping key") + } + + test("map extraction should reject dataset qualifier collisions") { + val input = spark.createDataFrame(Seq(("group", 1.0))) + .toDF("key", "score") + .select(map(lit("field"), col("score")).alias("values"), col("score"), col("key").alias("field")) + .as("values") + val transformer = new EnsembleByKey().setKey("values.field").setCol("score") + + assertConsistentSchemaError(transformer, input, "ambiguous between a nested field and a dataset qualifier") + } + + test("nested key output names should preserve configured casing") { + val input = spark.createDataFrame(Seq(("group", 1.0))) + .toDF("key", "score") + .select(struct(col("key").alias("Key")).alias("nested"), col("score")) + val transformer = new EnsembleByKey().setKey("nested.key").setCol("score") + + assert(transformer.transformSchema(input.schema) === transformer.transform(input).schema) + assert(transformer.transform(input).schema.fieldNames.head === "key") + } + + test("qualified references should preserve qualifier identity") { + val left = spark.createDataFrame(Seq((1, "left", 1.0))).toDF("id", "group", "score").as("left") + val right = spark.createDataFrame(Seq((1, "right"))).toDF("id", "group").as("right") + val joined = left.join(right, Seq("id")) + + Seq("left", "right").foreach { qualifier => + val transformer = new EnsembleByKey().setKey(s"$qualifier.group").setCol("score") + withClue(s"$qualifier: ") { + assert(assertSchemaAgrees(transformer, joined).head().getString(0) === qualifier) + } + } + + assertConsistentSchemaError( + new EnsembleByKey().setKey("right.group").setCol("score").setCollapseGroup(false), + joined, + "multiple columns are named group when collapseGroup is false") + + val invalidQualifier = new EnsembleByKey().setKey("wrong.group").setCol("score") + assert(invalidQualifier.transformSchema(joined.schema).fieldNames === Array("group", "mean(score)")) + val error = intercept[IllegalArgumentException](invalidQualifier.transform(joined)) + assert(error.getMessage.contains("does not match a dataset qualifier")) + } + + test("non-collapsed qualified references should preserve unrelated duplicates") { + val left = spark.createDataFrame(Seq((1, "group", 1.0))).toDF("id", "group", "score").as("left") + val right = spark.createDataFrame(Seq((1, 2.0))).toDF("id", "score").as("right") + val joined = left.join(right, Seq("id")) + val transformer = new EnsembleByKey() + .setKey("left.group").setCol("left.score") + .setColName("average").setCollapseGroup(false) + val transformed = assertSchemaAgrees(transformer, joined) + + assert(transformed.schema.fieldNames === Array("group", "id", "score", "score", "average")) + assert(transformed.head().getDouble(4) === 1.0) + } + + test("qualified aggregates should compare derived aggregate outputs") { + val left = spark.createDataFrame(Seq((1, "group", 1.0), (2, "group", 3.0))) + .toDF("id", "group", "score").as("left") + val right = spark.createDataFrame(Seq((1, 5.0))).toDF("id", "score").as("right") + val joined = left.join(right, Seq("id"), "left_outer") + assert(joined.schema.fields.filter(_.name == "score").map(_.nullable) === Array(false, true)) + + Seq("left.score" -> 2.0, "right.score" -> 5.0).foreach { case (reference, expected) => + val transformed = assertSchemaAgrees(new EnsembleByKey().setKey("group").setCol(reference), joined) + withClue(s"$reference: ") { + assert(transformed.schema.last === StructField(s"mean($reference)", DoubleType)) + assert(transformed.head().getDouble(1) === expected) + } + } + + assertConsistentSchemaError( + new EnsembleByKey().setKey("right.score").setCol("left.score"), + joined, + "incompatible declared outputs") + + val nestedLeft = spark.createDataFrame(Seq((1, 1.0), (1, 3.0))).toDF("id", "value") + .select(col("id"), struct(col("value")).alias("s")).as("left") + val nestedRight = spark.createDataFrame(Seq((1, 5.0f))).toDF("id", "value") + .select(col("id"), struct(col("value")).alias("s")).as("right") + val nested = assertSchemaAgrees( + new EnsembleByKey().setKey("id").setCol("right.s.value"), + nestedLeft.join(nestedRight, Seq("id"))) + assert(nested.schema.last === StructField("mean(right.s.value)", DoubleType)) + assert(nested.head().getDouble(1) === 5.0) + + val stringRight = spark.createDataFrame(Seq((1, "5"))).toDF("id", "value") + .select(col("id"), struct(col("value")).alias("s")).as("right") + assertConsistentSchemaError( + new EnsembleByKey().setKey("id").setCol("right.s.value"), + nestedLeft.join(stringRight, Seq("id")), + "incompatible declared outputs") + } + + test("multipart qualifiers should agree with schema-only interpretations") { + val base = spark.createDataFrame(Seq(("top", "nested", 1.0), ("top", "nested", 3.0))) + .toDF("group", "nestedGroup", "score") + val viewName = s"ensembleView${System.nanoTime()}" + val input = base.select( + col("group"), struct(col("nestedGroup").alias("group")).alias(viewName), col("score")) + val transformer = new EnsembleByKey().setKey(s"global_temp.$viewName.group").setCol("score") + + assert(assertSchemaAgrees(transformer, input.as("global_temp")).head().getString(0) === "nested") + + input.createOrReplaceGlobalTempView(viewName) + try { + val view = spark.table(s"global_temp.$viewName") + assert(assertSchemaAgrees(transformer, view).head().getString(0) === "top") + } finally { + spark.catalog.dropGlobalTempView(viewName) + } + + val conflicting = base.select( + col("score").alias("group"), + struct(col("nestedGroup").alias("group")).alias("view"), + col("score")) + assertConsistentSchemaError( + new EnsembleByKey().setKey("global_temp.view.group").setCol("score"), + conflicting, + "ambiguous between a nested field and a dataset qualifier") + } + + test("schema and runtime should reject invalid column configurations") { + val invalidConfigurations = Seq( + new EnsembleByKey().setCol("doubleScore") -> "keys must be set and non-empty", + new EnsembleByKey().setKeys(Array.empty[String]).setCol("doubleScore") -> + "keys must be set and non-empty", + new EnsembleByKey().setKey("group") -> "cols must be set and non-empty", + new EnsembleByKey().setKey("group").setCols(Array.empty[String]) -> + "cols must be set and non-empty", + new EnsembleByKey().setKey("missingKey").setCol("doubleScore") -> "missingKey does not exist", + new EnsembleByKey().setKey("group").setCol("missingCol") -> "missingCol does not exist", + new EnsembleByKey().setKey("group").setCols("doubleScore", "floatScore") + .setColName("average") -> "must have the same length", + new EnsembleByKey().setKey("group").setCol("doubleScore").setColName("GROUP") + .setCollapseGroup(false) -> "cannot overwrite grouping keys" + ) + + invalidConfigurations.foreach { case (transformer, expectedMessage) => + assertConsistentSchemaError(transformer, mixedTypeDF, expectedMessage) + } + + withCaseSensitiveAnalysis(false) { + val ambiguousInput = spark.createDataFrame(Seq(("lower", "upper", 1.0))) + .toDF("group", "GROUP", "score") + val keyTransformer = new EnsembleByKey().setKey("group").setCol("score") + assert(keyTransformer.transformSchema(ambiguousInput.schema).fieldNames === + Array("group", "mean(score)")) + val error = intercept[IllegalArgumentException](keyTransformer.transform(ambiguousInput)) + assert(error.getMessage.contains("group is ambiguous")) + + val ambiguousAggregateInput = spark.createDataFrame(Seq(("group", 1.0, 2.0))) + .toDF("group", "score", "SCORE") + val aggregateTransformer = new EnsembleByKey().setKey("group").setCol("score") + assert(aggregateTransformer.transformSchema(ambiguousAggregateInput.schema).fieldNames === + Array("group", "mean(score)")) + val aggregateError = + intercept[IllegalArgumentException](aggregateTransformer.transform(ambiguousAggregateInput)) + assert(aggregateError.getMessage.contains("score is ambiguous")) + } + } + + test("transformSchema should reject unsupported aggregate types") { + val input = spark.createDataFrame(Seq(("foo", 1))).toDF("group", "score") + val transformer = new EnsembleByKey().setKey("group").setCol("score") + + val error = intercept[IllegalArgumentException] { + transformer.transformSchema(input.schema) + } + + assert(error.getMessage === "Cannot operate on type IntegerType with strategy mean") + } + lazy val testDF: DataFrame = { val initialTestDF = spark.createDataFrame( Seq((0, "foo", 1.0, .1), @@ -64,6 +692,19 @@ class EnsembleByKeySuite extends TestBase with TransformerFuzzing[EnsembleByKey] .setOutputCol("v1").transform(initialTestDF) } + lazy val mixedTypeDF: DataFrame = { + val initialTestDF = spark.createDataFrame( + Seq((0, "west", "foo", 1.0, 1.0f, 1.0, 0.1), + (1, "east", "bar", 4.0, 4.0f, 4.0, -2.0), + (2, "east", "bar", 0.0, 0.0f, 0.0, -3.0))) + .toDF("id", "region", "group", "doubleScore", "floatScore", "component1", "component2") + + new VectorAssembler() + .setInputCols(Array("component1", "component2")) + .setOutputCol("features") + .transform(initialTestDF) + } + lazy val testModel: EnsembleByKey = new EnsembleByKey().setKey("label1").setCol("score1") .setCollapseGroup(false).setVectorDims(Map("v1"->2)) @@ -98,4 +739,45 @@ class EnsembleByKeySuite extends TestBase with TransformerFuzzing[EnsembleByKey] def testObjects(): Seq[TestObject[EnsembleByKey]] = Seq(new TestObject(testModel, testDF)) def reader: EnsembleByKey.type = EnsembleByKey + + private def withCaseSensitiveAnalysis[T](value: Boolean)(action: => T): T = { + withSQLConf("spark.sql.caseSensitive", value.toString)(action) + } + + private def withSQLConf[T](configName: String, value: String)(action: => T): T = { + val previousValue = spark.conf.get(configName) + spark.conf.set(configName, value) + try action finally spark.conf.set(configName, previousValue) + } + + private def withActiveSession[T](session: SparkSession)(action: => T): T = { + val previousSession = SparkSession.getActiveSession + SparkSession.setActiveSession(session) + try action finally { + previousSession.fold(SparkSession.clearActiveSession())(SparkSession.setActiveSession) + } + } + + private def withoutActiveSession[T](action: => T): T = { + val previousSession = SparkSession.getActiveSession + SparkSession.clearActiveSession() + try action finally previousSession.foreach(SparkSession.setActiveSession) + } + + private def assertSchemaAgrees(transformer: EnsembleByKey, input: DataFrame): DataFrame = { + val transformed = transformer.transform(input) + assert(transformer.transformSchema(input.schema) === transformed.schema) + transformed + } + + private def assertConsistentSchemaError( + transformer: EnsembleByKey, + input: DataFrame, + expectedMessage: String + ): Unit = { + val schemaError = intercept[IllegalArgumentException](transformer.transformSchema(input.schema)) + val transformError = intercept[IllegalArgumentException](transformer.transform(input)) + assert(schemaError.getMessage.contains(expectedMessage)) + assert(transformError.getMessage.contains(expectedMessage)) + } } diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/stages/LumpFeaturesSuite.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/stages/LumpFeaturesSuite.scala new file mode 100644 index 00000000000..a32579be74b --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/stages/LumpFeaturesSuite.scala @@ -0,0 +1,577 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.stages + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import com.microsoft.azure.synapse.ml.core.test.fuzzing.{EstimatorFuzzing, TestObject, TransformerFuzzing} +import org.apache.spark.ml.param.ParamMap +import org.apache.spark.ml.util.MLReadable +import org.apache.spark.sql.functions.col +import org.apache.spark.sql.types._ +import org.apache.spark.sql.{Column, DataFrame, Row} +import spray.json._ + +import java.io.File +import scala.collection.JavaConverters._ + +/** Shared input fixture and projection helper used by both the estimator suite + * [[LumpFeaturesSuite]] and the model suite [[LumpFeaturesModelSuite]]. + */ +trait LumpFeaturesTestData extends TestBase { + + import spark.implicits._ + + protected lazy val df: DataFrame = Seq( + ("apple", "red"), + ("apple", "red"), + ("apple", "blue"), + ("banana", "red"), + ("cherry", "green") + ).toDF("f1", "f2") + + protected val other: String = "__other__" + + protected def dump(data: DataFrame, cols: Seq[String]): List[String] = { + val projected = data.select(cols.map(col): _*) + projected.collect().map { r => + cols.indices.map(i => if (r.isNullAt(i)) "" else r.get(i).toString).mkString("|") + }.sorted.toList + } +} + +//scalastyle:off null +class LumpFeaturesSuite extends EstimatorFuzzing[LumpFeatures] with LumpFeaturesTestData { + + import spark.implicits._ + + test("multi-column top-K retains learned values and lumps the rest") { + val model = new LumpFeatures().setLumpRules(Map("f1" -> 1, "f2" -> 1)).fit(df) + assert(model.getKeptValues("f1") == Seq("apple")) + assert(model.getKeptValues("f2") == Seq("red")) + val out = model.transform(df) + assert(out.columns.toSeq == Seq("f1", "f2")) + assert(dump(out, Seq("f1", "f2")) == List( + s"$other|$other", s"$other|red", s"apple|$other", "apple|red", "apple|red")) + } + + test("stable ranking is count desc then value asc for ties") { + val model = new LumpFeatures().setLumpRules(Map("f1" -> 2)).fit(df) + assert(model.getKeptValues("f1") == Seq("apple", "banana")) + assert(dump(model.transform(df), Seq("f1")) == List(other, "apple", "apple", "apple", "banana")) + } + + test("rare and unseen non-null values map to the other bucket") { + val model = new LumpFeatures().setLumpRules(Map("f1" -> 1)).fit(df) + val scoring = Seq(("banana", "red"), ("durian", "red"), ("apple", "red")).toDF("f1", "f2") + assert(dump(model.transform(scoring), Seq("f1")) == List(other, other, "apple")) + } + + test("handleNull keep preserves null; handleNull other maps null to otherValue") { + val dfN = Seq(("apple", "red"), ("apple", "red"), (null, "red"), ("banana", "red")) + .toDF("f1", "f2") + val keepModel = new LumpFeatures().setLumpRules(Map("f1" -> 1)).setHandleNull("keep").fit(dfN) + assert(dump(keepModel.transform(dfN), Seq("f1")) == List("", other, "apple", "apple")) + val otherModel = new LumpFeatures().setLumpRules(Map("f1" -> 1)).setHandleNull("other").fit(dfN) + assert(dump(otherModel.transform(dfN), Seq("f1")) == List(other, other, "apple", "apple")) + } + + test("all-null column yields empty learned levels and maps values accordingly") { + val dfAllNull = Seq((null.asInstanceOf[String], "red"), (null.asInstanceOf[String], "blue")) + .toDF("f1", "f2") + val model = new LumpFeatures().setLumpRules(Map("f1" -> 2)).setHandleNull("keep").fit(dfAllNull) + assert(model.getKeptValues("f1").isEmpty) + assert(dump(model.transform(dfAllNull), Seq("f1")) == List("", "")) + val scoring = Seq(("zebra", "red")).toDF("f1", "f2") + assert(dump(model.transform(scoring), Seq("f1")) == List(other)) + } + + test("empty learned levels via manually constructed model maps all non-null to otherValue") { + val model = new LumpFeaturesModel() + .setLumpRules(Map("f1" -> 2)) + .setKeptValuesJson(Map("f1" -> "{\"topK\":2,\"values\":[]}")) + .setHandleNull("keep") + assert(dump(model.transform(df), Seq("f1")) == List(other, other, other, other, other)) + } + + test("fit rejects otherValue collision even when the value is rare") { + val dfCollide = Seq(("apple", "red"), ("apple", "red"), (other, "blue")).toDF("f1", "f2") + val ex = intercept[IllegalArgumentException] { + new LumpFeatures().setLumpRules(Map("f1" -> 1)).fit(dfCollide) + } + assert(ex.getMessage.toLowerCase.contains("othervalue")) + } + + test("fitted model rejects direct otherValue mutation to an observed non-retained category") { + val model = new LumpFeatures().setLumpRules(Map("f1" -> 1)).fit(df) + assert(model.getKeptValues("f1") == Seq("apple")) + val ex = intercept[IllegalArgumentException] { model.setOtherValue("banana") } + assert(ex.getMessage.toLowerCase.contains("othervalue")) + assert(model.getOtherValue == other) + assert(dump(model.transform(df), Seq("f1")) == List(other, other, "apple", "apple", "apple")) + } + + test("generic otherValue Param mutation is rejected and leaves the fitted model unchanged") { + val model = new LumpFeatures().setLumpRules(Map("f1" -> 1)).fit(df) + val ex = intercept[IllegalArgumentException] { + model.set(model.otherValue, "banana") + } + assert(ex.getMessage.toLowerCase.contains("othervalue")) + assert(model.getOtherValue == other) + assert(dump(model.transform(df), Seq("f1")) == List(other, other, "apple", "apple", "apple")) + } + + test("generic clear and mutation cannot remove or corrupt fitted learned state") { + val model = new LumpFeatures() + .setLumpRules(Map("f1" -> 1)) + .setOtherValue("fallback") + .fit(df) + val learned = model.getKeptValuesJson + + intercept[IllegalArgumentException] { model.clear(model.getParam("keptValuesJson")) } + assert(model.getKeptValuesJson == learned) + + intercept[IllegalArgumentException] { model.set(model.keptValuesJson, Map.empty[String, String]) } + assert(model.getKeptValuesJson == learned) + + intercept[IllegalArgumentException] { model.clear(model.otherValue) } + assert(model.getOtherValue == "fallback") + intercept[IllegalArgumentException] { model.setOtherValue("banana") } + } + + test("fit validates rules, K, column names, presence and string type") { + intercept[IllegalArgumentException] { + new LumpFeatures().setLumpRules(Map.empty[String, Int]).fit(df) + } + intercept[IllegalArgumentException] { + new LumpFeatures().fit(df) + } + intercept[IllegalArgumentException] { + new LumpFeatures().setLumpRules(Map("f1" -> 0)).fit(df) + } + intercept[IllegalArgumentException] { + new LumpFeatures().setLumpRules(Map("f1" -> -3)).fit(df) + } + intercept[IllegalArgumentException] { + new LumpFeatures().setLumpRules(Map("" -> 1)).fit(df) + } + intercept[IllegalArgumentException] { + new LumpFeatures().setLumpRules(Map("missing" -> 1)).fit(df) + } + val dfInt = Seq(("apple", 1), ("banana", 2)).toDF("f1", "n") + intercept[IllegalArgumentException] { + new LumpFeatures().setLumpRules(Map("n" -> 1)).fit(dfInt) + } + } + + test("model validates state and rule consistency") { + val mismatch = new LumpFeaturesModel() + .setLumpRules(Map("f1" -> 1)) + .setKeptValuesJson(Map("f2" -> "{\"topK\":1,\"values\":[\"red\"]}")) + intercept[IllegalArgumentException] { mismatch.transform(df).collect() } + val tooMany = new LumpFeaturesModel() + .setLumpRules(Map("f1" -> 1)) + .setKeptValuesJson(Map("f1" -> "{\"topK\":1,\"values\":[\"apple\",\"banana\"]}")) + intercept[IllegalArgumentException] { tooMany.transform(df).collect() } + } + + test("schema matches transform exactly with cleared metadata and correct nullability") { + val schema = StructType(Seq( + StructField("f1", StringType, nullable = true), + StructField("f2", StringType, nullable = true), + StructField("keep_me", IntegerType, nullable = false))) + val rows = Seq(Row("apple", "red", 1), Row("banana", "red", 2), Row(null, "blue", 3)) + val md = new MetadataBuilder().putString("foo", "bar").build() + val base = spark.createDataFrame(rows.asJava, schema) + val dfMeta = base.withColumn("f1", col("f1").as("f1", md)) + + val keepModel = new LumpFeatures().setLumpRules(Map("f1" -> 1)).setHandleNull("keep").fit(dfMeta) + val declaredKeep = keepModel.transformSchema(dfMeta.schema) + assert(declaredKeep == keepModel.transform(dfMeta).schema) + assert(declaredKeep.fieldNames.toSeq == Seq("f1", "f2", "keep_me")) + assert(declaredKeep("f1").dataType == StringType) + assert(declaredKeep("f1").nullable) + assert(declaredKeep("f1").metadata == Metadata.empty) + assert(declaredKeep("f2") == dfMeta.schema("f2")) + assert(declaredKeep("keep_me") == dfMeta.schema("keep_me")) + assert(keepModel.transform(dfMeta).schema("f1").metadata == Metadata.empty) + + val otherModel = new LumpFeatures().setLumpRules(Map("f1" -> 1)).setHandleNull("other").fit(dfMeta) + val declaredOther = otherModel.transformSchema(dfMeta.schema) + assert(declaredOther == otherModel.transform(dfMeta).schema) + assert(!declaredOther("f1").nullable) + } + + test("estimator and model copy preserve parameters and learned state") { + val est = new LumpFeatures().setLumpRules(Map("f1" -> 1)).setOtherValue("X").setHandleNull("other") + val estCopy = est.copy(new ParamMap()) + assert(estCopy.getLumpRules == Map("f1" -> 1)) + assert(estCopy.getOtherValue == "X") + assert(estCopy.getHandleNull == "other") + val model = est.fit(df) + val modelCopy = model.copy(new ParamMap()) + assert(modelCopy.getKeptValues == model.getKeptValues) + assert(modelCopy.getOtherValue == "X") + assert(modelCopy.getHandleNull == "other") + assert(modelCopy.getLumpRules == Map("f1" -> 1)) + } + + test("legacy JSON-string and Java HashMap setters populate lumpRules") { + val fromJson = new LumpFeatures().setLumpRules("{\"f1\":2,\"f2\":1}") + assert(fromJson.getLumpRules == Map("f1" -> 2, "f2" -> 1)) + val jmap = new java.util.HashMap[String, Int]() + jmap.put("f1", 3) + val fromJava = new LumpFeatures().setLumpRules(jmap) + assert(fromJava.getLumpRules == Map("f1" -> 3)) + } + + test("persisted learned values survive arbitrary strings round-trip") { + val weird = Seq( + ("a\"b", "x"), + ("[bracket]", "x"), + ("\u00fcn\u00eecod\u00e9", "x"), + ("with,comma", "x"), + ("back\\slash", "x"), + ("a\"b", "y") + ).toDF("f1", "f2") + val model = new LumpFeatures().setLumpRules(Map("f1" -> 10)).fit(weird) + val path = new File(tmpDir.toFile, "lump-weird").toString + model.write.overwrite().save(path) + val loaded = LumpFeaturesModel.load(path) + assert(loaded.getKeptValues("f1").toSet == + Set("a\"b", "[bracket]", "\u00fcn\u00eecod\u00e9", "with,comma", "back\\slash")) + assertDFEq(model.transform(weird), loaded.transform(weird)) + } + + private def bq(name: String): Column = col("`" + name.replace("`", "``") + "`") + + private def dumpQ(data: DataFrame, cols: Seq[String]): List[String] = { + val projected = data.select(cols.map(bq): _*) + projected.collect().map { r => + cols.indices.map(i => if (r.isNullAt(i)) "" else r.get(i).toString).mkString("|") + }.sorted.toList + } + + test("rule columns named count, dotted, and embedded-backtick fit and transform correctly") { + val specialDf = Seq( + ("apple", "red", "x"), + ("apple", "red", "x"), + ("apple", "blue", "y"), + ("banana", "red", "x") + ).toDF("count", "a.b", "a`b") + val rules = Map("count" -> 1, "a.b" -> 1, "a`b" -> 1) + val model = new LumpFeatures().setLumpRules(rules).fit(specialDf) + assert(model.getKeptValues("count") == Seq("apple")) + assert(model.getKeptValues("a.b") == Seq("red")) + assert(model.getKeptValues("a`b") == Seq("x")) + val out = model.transform(specialDf) + assert(out.columns.toSeq == Seq("count", "a.b", "a`b")) + assert(dumpQ(out, Seq("count")) == List(other, "apple", "apple", "apple")) + assert(dumpQ(out, Seq("a.b")) == List(other, "red", "red", "red")) + assert(dumpQ(out, Seq("a`b")) == List(other, "x", "x", "x")) + val collideDf = Seq(("apple", "red", "x"), (other, "blue", "y")).toDF("count", "a.b", "a`b") + val ex = intercept[IllegalArgumentException] { + new LumpFeatures().setLumpRules(Map("count" -> 1)).fit(collideDf) + } + assert(ex.getMessage.toLowerCase.contains("othervalue")) + } + + test("fitted model rejects increasing or decreasing K even when distinct values are fewer than K") { + val twoDistinct = Seq(("apple", "x"), ("apple", "x"), ("banana", "x")).toDF("f1", "f2") + val model = new LumpFeatures().setLumpRules(Map("f1" -> 3)).fit(twoDistinct) + assert(model.getKeptValues("f1") == Seq("apple", "banana")) + intercept[IllegalArgumentException] { model.setLumpRules(Map("f1" -> 5)) } + intercept[IllegalArgumentException] { model.setLumpRules(Map("f1" -> 2)) } + assert(model.getLumpRules == Map("f1" -> 3)) + assert(model.getKeptValues("f1") == Seq("apple", "banana")) + } + + test("setting the exact fitted lumpRules map on a fitted model is an allowed no-op") { + val model = new LumpFeatures().setLumpRules(Map("f1" -> 1, "f2" -> 1)).fit(df) + val before = dump(model.transform(df), Seq("f1", "f2")) + val returned = model.setLumpRules(Map("f1" -> 1, "f2" -> 1)) + assert(returned.getLumpRules == Map("f1" -> 1, "f2" -> 1)) + assert(dump(model.transform(df), Seq("f1", "f2")) == before) + } + + test("loaded model and its copy retain immutable fitted rules and reject incompatible changes") { + val model = new LumpFeatures().setLumpRules(Map("f1" -> 1, "f2" -> 1)).fit(df) + val path = new File(tmpDir.toFile, "lump-immutable").toString + model.write.overwrite().save(path) + val loaded = LumpFeaturesModel.load(path) + assert(loaded.getLumpRules == Map("f1" -> 1, "f2" -> 1)) + assert(loaded.getKeptValues == model.getKeptValues) + assertDFEq(loaded.transform(df), model.transform(df)) + intercept[IllegalArgumentException] { loaded.setLumpRules(Map("f1" -> 2, "f2" -> 1)) } + val copied = loaded.copy(new ParamMap()) + assert(copied.getKeptValues == model.getKeptValues) + assert(copied.getLumpRules == Map("f1" -> 1, "f2" -> 1)) + intercept[IllegalArgumentException] { copied.setLumpRules(Map("f1" -> 3, "f2" -> 1)) } + } + + test("copy with an incompatible lumpRules ParamMap override is rejected") { + val model = new LumpFeatures().setLumpRules(Map("f1" -> 1)).fit(df) + val incompatible = new ParamMap().put(model.lumpRules, Map("f1" -> 5)) + intercept[IllegalArgumentException] { model.copy(incompatible) } + val same = new ParamMap().put(model.lumpRules, Map("f1" -> 1)) + val copied = model.copy(same) + assert(copied.getLumpRules == Map("f1" -> 1)) + assert(copied.getKeptValues == model.getKeptValues) + } + + test("copy rejects forged learned state, otherValue, and rules atomically but accepts identical state") { + val model = new LumpFeatures().setLumpRules(Map("f1" -> 1)).fit(df) + val learned = model.getKeptValuesJson + val before = dump(model.transform(df), Seq("f1")) + + val incompatibleOther = new ParamMap().put(model.otherValue, "banana") + intercept[IllegalArgumentException] { model.copy(incompatibleOther) } + + val incompatibleRules = new ParamMap().put(model.lumpRules, Map("f1" -> 2)) + intercept[IllegalArgumentException] { model.copy(incompatibleRules) } + + val forged = learned.updated("f1", LumpFeaturesModel.encodeKept(1, Seq("banana"), other)) + val forgedState = new ParamMap().put(model.keptValuesJson, forged) + intercept[IllegalArgumentException] { model.copy(forgedState) } + + val missingState = new ParamMap().put(model.keptValuesJson, Map.empty[String, String]) + intercept[IllegalArgumentException] { model.copy(missingState) } + + val identicalState = new ParamMap().put(model.keptValuesJson, learned) + val copied = model.copy(identicalState) + assert(copied.getKeptValuesJson == learned) + assertDFEq(copied.transform(df), model.transform(df)) + + assert(model.getOtherValue == other) + assert(model.getLumpRules == Map("f1" -> 1)) + assert(model.getKeptValuesJson == learned) + assert(dump(model.transform(df), Seq("f1")) == before) + } + + test("model-state validation rejects lumpRules mutated through the generic Param path") { + val model = new LumpFeatures().setLumpRules(Map("f1" -> 1)).fit(df) + model.set(model.lumpRules, Map("f1" -> 5)) + val ex = intercept[IllegalArgumentException] { model.transform(df).collect() } + assert(ex.getMessage.toLowerCase.contains("lumprules") || ex.getMessage.toLowerCase.contains("top-k")) + } + + test("fitted model setLumpRules overloads (Scala Map, Java HashMap, JSON string) all reject changes") { + def fitted(): LumpFeaturesModel = new LumpFeatures().setLumpRules(Map("f1" -> 1)).fit(df) + intercept[IllegalArgumentException] { fitted().setLumpRules(Map("f1" -> 2)) } + val jmap = new java.util.HashMap[String, Int]() + jmap.put("f1", 2) + intercept[IllegalArgumentException] { fitted().setLumpRules(jmap) } + intercept[IllegalArgumentException] { fitted().setLumpRules("{\"f1\":2}") } + } + + test("minCount lumps values below the count threshold before the top-K cap applies") { + val model = new LumpFeatures().setLumpRules(Map("f1" -> 3)).setMinCount(2).fit(df) + assert(model.getKeptValues("f1") == Seq("apple")) + assert(dump(model.transform(df), Seq("f1")) == List(other, other, "apple", "apple", "apple")) + } + + test("minFreq lumps values below the frequency share before the top-K cap applies") { + val model = new LumpFeatures().setLumpRules(Map("f1" -> 3)).setMinFreq(0.25).fit(df) + assert(model.getKeptValues("f1") == Seq("apple")) + val permissive = new LumpFeatures().setLumpRules(Map("f1" -> 3)).setMinFreq(0.15).fit(df) + assert(permissive.getKeptValues("f1") == Seq("apple", "banana", "cherry")) + } + + test("top-K still caps values that clear the frequency filters") { + val model = new LumpFeatures().setLumpRules(Map("f1" -> 1)).setMinCount(1).setMinFreq(0.1).fit(df) + assert(model.getKeptValues("f1") == Seq("apple")) + } + + test("frequency filters are per column and are computed against non-null rows only") { + val dfN = Seq(("apple", "red"), ("apple", "red"), (null, "red"), ("banana", "blue")) + .toDF("f1", "f2") + val model = new LumpFeatures().setLumpRules(Map("f1" -> 3, "f2" -> 3)).setMinFreq(0.5).fit(dfN) + // f1 has 3 non-null rows, so apple (2/3) clears 0.5 and banana (1/3) does not. + assert(model.getKeptValues("f1") == Seq("apple")) + // f2 has 4 non-null rows, so red (3/4) clears 0.5 and blue (1/4) does not. + assert(model.getKeptValues("f2") == Seq("red")) + } + + test("minCount and minFreq default to no-ops and reject out-of-range values") { + val est = new LumpFeatures().setLumpRules(Map("f1" -> 3)) + assert(est.getMinCount == 1) + assert(est.getMinFreq == 0.0) + intercept[IllegalArgumentException] { est.setMinCount(0) } + intercept[IllegalArgumentException] { est.setMinFreq(-0.1) } + intercept[IllegalArgumentException] { est.setMinFreq(1.1) } + assert(est.fit(df).getKeptValues("f1") == Seq("apple", "banana", "cherry")) + } + + test("multi-column fit agrees with fitting each column on its own") { + val rules = Map("f1" -> 2, "f2" -> 2) + val together = new LumpFeatures().setLumpRules(rules).fit(df).getKeptValues + rules.foreach { case (name, k) => + val alone = new LumpFeatures().setLumpRules(Map(name -> k)).fit(df).getKeptValues(name) + assert(together(name) == alone, s"column $name disagreed between joint and single-column fits") + } + } + + test("outputCols appends lumped columns and leaves the raw columns untouched") { + val model = new LumpFeatures() + .setLumpRules(Map("f1" -> 1)) + .setOutputCols(Map("f1" -> "f1_lumped")) + .fit(df) + val out = model.transform(df) + assert(out.columns.toSeq == Seq("f1", "f2", "f1_lumped")) + assert(dump(out, Seq("f1", "f1_lumped")) == List( + "apple|apple", "apple|apple", "apple|apple", s"banana|$other", s"cherry|$other")) + } + + test("outputCols mixes appended and in-place columns with a schema that matches transform") { + val model = new LumpFeatures() + .setLumpRules(Map("f1" -> 1, "f2" -> 1)) + .setOutputCols(Map("f2" -> "f2_lumped")) + .fit(df) + val declared = model.transformSchema(df.schema) + assert(declared == model.transform(df).schema) + assert(declared.fieldNames.toSeq == Seq("f1", "f2", "f2_lumped")) + assert(dump(model.transform(df), Seq("f2", "f2_lumped")) == List( + s"blue|$other", s"green|$other", "red|red", "red|red", "red|red")) + assert(dump(model.transform(df), Seq("f1")) == List(other, other, "apple", "apple", "apple")) + } + + test("outputCols rejects unknown sources, empty, duplicate, and pre-existing destinations") { + intercept[IllegalArgumentException] { + new LumpFeatures().setLumpRules(Map("f1" -> 1)).setOutputCols(Map("nope" -> "x")).fit(df) + } + intercept[IllegalArgumentException] { + new LumpFeatures().setLumpRules(Map("f1" -> 1)).setOutputCols(Map("f1" -> "")).fit(df) + } + intercept[IllegalArgumentException] { + new LumpFeatures().setLumpRules(Map("f1" -> 1, "f2" -> 1)) + .setOutputCols(Map("f1" -> "z", "f2" -> "z")).fit(df) + } + intercept[IllegalArgumentException] { + new LumpFeatures().setLumpRules(Map("f1" -> 1)).setOutputCols(Map("f1" -> "f2")).fit(df) + } + } + + test("outputCols survives fit, copy, and a save-load round-trip") { + val model = new LumpFeatures() + .setLumpRules(Map("f1" -> 1)) + .setOutputCols(Map("f1" -> "f1_lumped")) + .fit(df) + assert(model.getOutputCols == Map("f1" -> "f1_lumped")) + assert(model.copy(new ParamMap()).getOutputCols == Map("f1" -> "f1_lumped")) + val path = new File(tmpDir.toFile, "lump-outputcols").toString + model.write.overwrite().save(path) + val loaded = LumpFeaturesModel.load(path) + assert(loaded.getOutputCols == Map("f1" -> "f1_lumped")) + assertDFEq(loaded.transform(df), model.transform(df)) + } + + test("outputCols accepts the Java HashMap and JSON-string setter overloads") { + val jmap = new java.util.HashMap[String, String]() + jmap.put("f1", "f1_lumped") + assert(new LumpFeatures().setOutputCols(jmap).getOutputCols == Map("f1" -> "f1_lumped")) + val fromJson = new LumpFeatures().setOutputCols("{\"f1\":\"f1_lumped\"}") + assert(fromJson.getOutputCols == Map("f1" -> "f1_lumped")) + } + + test("estimator and model nullability follow handleNull even for a non-nullable input column") { + val schema = StructType(Seq(StructField("f1", StringType, nullable = false))) + val rows = Seq(Row("apple"), Row("apple"), Row("banana")) + val dfNN = spark.createDataFrame(rows.asJava, schema) + assert(!dfNN.schema("f1").nullable) + + val keepEstimator = new LumpFeatures().setLumpRules(Map("f1" -> 1)).setHandleNull("keep") + assert(keepEstimator.transformSchema(dfNN.schema)("f1").nullable) + val keepModel = keepEstimator.fit(dfNN) + val declaredKeep = keepModel.transformSchema(dfNN.schema) + assert(declaredKeep == keepModel.transform(dfNN).schema) + assert(declaredKeep("f1").nullable) + assert(dump(keepModel.transform(dfNN), Seq("f1")) == List(other, "apple", "apple")) + + val otherEstimator = new LumpFeatures().setLumpRules(Map("f1" -> 1)).setHandleNull("other") + assert(!otherEstimator.transformSchema(dfNN.schema)("f1").nullable) + val otherModel = otherEstimator.fit(dfNN) + val declaredOther = otherModel.transformSchema(dfNN.schema) + assert(declaredOther == otherModel.transform(dfNN).schema) + assert(!declaredOther("f1").nullable) + assert(dump(otherModel.transform(dfNN), Seq("f1")) == List(other, "apple", "apple")) + } + + test("fit-time otherValue survives save-load and protects observed non-retained categories") { + val fallback = "fallback" + val model = new LumpFeatures().setLumpRules(Map("f1" -> 1)).setOtherValue(fallback).fit(df) + val path = new File(tmpDir.toFile, "lump-fitted-other").toString + model.write.overwrite().save(path) + val loaded = LumpFeaturesModel.load(path) + val state = loaded.getKeptValuesJson("f1").parseJson.asJsObject.fields + assert(state("otherValue") == JsString(fallback)) + assert(!loaded.hasParam("fittedOtherValue")) + assert(loaded.getKeptValues("f1") == Seq("apple")) + intercept[IllegalArgumentException] { loaded.setOtherValue("banana") } + intercept[IllegalArgumentException] { loaded.set(loaded.otherValue, "banana") } + assert(loaded.getOtherValue == fallback) + assertDFEq(loaded.transform(df), model.transform(df)) + } + + test("loading legacy model state initializes and enforces the fitted otherValue") { + val legacy = new LumpFeaturesModel() + .setLumpRules(Map("f1" -> 1)) + .setOtherValue("fallback") + .setKeptValuesJson(Map("f1" -> "{\"topK\":1,\"values\":[\"apple\"]}")) + assert(!legacy.getKeptValuesJson("f1").parseJson.asJsObject.fields.contains("otherValue")) + val path = new File(tmpDir.toFile, "lump-legacy-other").toString + legacy.write.overwrite().save(path) + val loaded = LumpFeaturesModel.load(path) + val upgraded = loaded.getKeptValuesJson("f1").parseJson.asJsObject.fields + assert(upgraded("otherValue") == JsString("fallback")) + intercept[IllegalArgumentException] { loaded.setOtherValue("banana") } + intercept[IllegalArgumentException] { loaded.clear(loaded.getParam("keptValuesJson")) } + assert(loaded.getOtherValue == "fallback") + } + + test("getKeptValuesAsJson gives language wrappers one document with the learned values") { + val model = new LumpFeatures().setLumpRules(Map("f1" -> 2, "f2" -> 1)).fit(df) + val expected = """{"f1":["apple","banana"],"f2":["red"]}""" + assert(model.getKeptValuesAsJson.parseJson == expected.parseJson) + val dfAllNull = Seq((null.asInstanceOf[String], "red")).toDF("f1", "f2") + val empty = new LumpFeatures().setLumpRules(Map("f1" -> 2)).fit(dfAllNull) + assert(empty.getKeptValuesAsJson.parseJson == """{"f1":[]}""".parseJson) + } + + override def testObjects(): Seq[TestObject[LumpFeatures]] = Seq( + new TestObject(new LumpFeatures().setLumpRules(Map("f1" -> 1, "f2" -> 1)), df)) + + override def reader: MLReadable[_] = LumpFeatures + + override def modelReader: MLReadable[_] = LumpFeaturesModel + +} + +/** Dedicated fuzzing suite for [[LumpFeaturesModel]]. LumpFeatures serializes fitted models, so the + * model needs its own Experiment/Serialization/Python/R fuzzer coverage. The estimator suite only + * registers a fuzzer for the estimator type, which left the model without any discovered fuzzer. + * The test object is a valid, manually constructed persisted model (learned levels supplied directly) + * paired with a deterministic input DataFrame. + */ +class LumpFeaturesModelSuite extends TransformerFuzzing[LumpFeaturesModel] with LumpFeaturesTestData { + + private def persistedModel: LumpFeaturesModel = new LumpFeaturesModel() + .setLumpRules(Map("f1" -> 1, "f2" -> 1)) + .setOtherValue(other) + .setKeptValuesJson(Map( + "f1" -> LumpFeaturesModel.encodeKept(1, Seq("apple"), other), + "f2" -> LumpFeaturesModel.encodeKept(1, Seq("red"), other))) + .setHandleNull("keep") + + test("manually constructed model retains learned values and lumps the rest deterministically") { + val out = persistedModel.transform(df) + assert(out.columns.toSeq == Seq("f1", "f2")) + assert(dump(out, Seq("f1", "f2")) == List( + s"$other|$other", s"$other|red", s"apple|$other", "apple|red", "apple|red")) + } + + override def testObjects(): Seq[TestObject[LumpFeaturesModel]] = + Seq(new TestObject(persistedModel, df)) + + override def reader: MLReadable[_] = LumpFeaturesModel + +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/stages/UnicodeNormalizeSuite.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/stages/UnicodeNormalizeSuite.scala index e7d098ca6fb..664140c5c6b 100644 --- a/core/src/test/scala/com/microsoft/azure/synapse/ml/stages/UnicodeNormalizeSuite.scala +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/stages/UnicodeNormalizeSuite.scala @@ -5,6 +5,7 @@ package com.microsoft.azure.synapse.ml.stages import com.microsoft.azure.synapse.ml.core.test.base.TestBase import com.microsoft.azure.synapse.ml.core.test.fuzzing.{TestObject, TransformerFuzzing} +import org.apache.spark.ml.param.Param import org.apache.spark.ml.util.MLReadable import org.apache.spark.sql.DataFrame @@ -49,9 +50,25 @@ class UnicodeNormalizeSuite extends TestBase with TransformerFuzzing[UnicodeNorm test("Check for NFKD forms") { testForm("NFKD", expectedResultDecomposed) } + test("setForm rejects an invalid form at set time") { + // An unvalidated form used to be accepted here and only fail later inside the UDF, + // on the executors, where the cause is far harder to attribute. + assertThrows[IllegalArgumentException](new UnicodeNormalize().setForm("NOT_A_FORM")) + } + + test("setForm accepts every supported form") { + java.text.Normalizer.Form.values().foreach { f => + assert(new UnicodeNormalize().setForm(f.name).getForm === f.name) + } + } + def testObjects(): Seq[TestObject[UnicodeNormalize]] = List(new TestObject( new UnicodeNormalize().setInputCol("words").setOutputCol("out"), makeBasicDF())) + // The generic fuzzer probes String setters with "foo"; form only accepts Normalizer.Form names. + override def getterSetterParamExamples(pipelineStage: UnicodeNormalize): Map[Param[_], Any] = + Map[Param[_], Any]((pipelineStage.form, "NFC")) + override def reader: MLReadable[_] = UnicodeNormalize } diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/stages/VerifyCacher.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/stages/VerifyCacher.scala new file mode 100644 index 00000000000..572ed7b8b60 --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/stages/VerifyCacher.scala @@ -0,0 +1,49 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.stages + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.spark.storage.StorageLevel + +class VerifyCacher extends TestBase { + import spark.implicits._ + + test("transform with disable=false caches the dataframe") { + val df = Seq(1, 2, 3).toDF("x") + val cacher = new Cacher().setDisable(false) + val result = cacher.transform(df) + try { + result.count() + assert(result.storageLevel !== StorageLevel.NONE) + } finally { + // TestBase shares one SparkSession across every suite in the leg, so a failed assertion + // must not leave this DataFrame cached for whatever runs next. + result.unpersist() + } + } + + test("transform with disable=true does not cache") { + val df = Seq(4, 5, 6).toDF("y") + val cacher = new Cacher().setDisable(true) + val result = cacher.transform(df) + assert(result.storageLevel === StorageLevel.NONE) + } + + test("default disable value is false") { + val cacher = new Cacher() + assert(!cacher.getDisable) + } + + test("copy preserves params") { + val cacher = new Cacher().setDisable(true) + val copied = cacher.copy(new org.apache.spark.ml.param.ParamMap()) + assert(copied.asInstanceOf[Cacher].getDisable) + } + + test("transformSchema returns same schema") { + val df = Seq(1, 2, 3).toDF("x") + val cacher = new Cacher() + assert(cacher.transformSchema(df.schema) === df.schema) + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/stages/VerifyTextPreprocessor.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/stages/VerifyTextPreprocessor.scala new file mode 100644 index 00000000000..ec788d314cc --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/stages/VerifyTextPreprocessor.scala @@ -0,0 +1,51 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.stages + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.spark.sql.types.{StringType, StructField} + +class VerifyTextPreprocessor extends TestBase { + import spark.implicits._ + + test("replaces matched substrings in DataFrame column") { + val df = Seq("hello world").toDF("text") + val tp = new TextPreprocessor() + .setInputCol("text") + .setOutputCol("out") + .setMap(Map("hello" -> "hi")) + val result = tp.transform(df).select("out").collect() + assert(result.head.getString(0) === "hi world") + } + + test("no matches returns original text") { + val df = Seq("hello world").toDF("text") + val tp = new TextPreprocessor() + .setInputCol("text") + .setOutputCol("out") + .setMap(Map("xyz" -> "abc")) + val result = tp.transform(df).select("out").collect() + assert(result.head.getString(0) === "hello world") + } + + test("multiple replacements in same text") { + val df = Seq("hello world.").toDF("text") + val tp = new TextPreprocessor() + .setInputCol("text") + .setOutputCol("out") + .setMap(Map("hello" -> "hi", "world" -> "earth")) + val result = tp.transform(df).select("out").collect() + assert(result.head.getString(0) === "hi earth.") + } + + test("transformSchema adds output column") { + val df = Seq("a").toDF("text") + val tp = new TextPreprocessor() + .setInputCol("text") + .setOutputCol("out") + val schema = tp.transformSchema(df.schema) + assert(schema.fieldNames.contains("out")) + assert(schema(schema.fieldIndex("out")) === StructField("out", StringType)) + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/stages/VerifyTrie.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/stages/VerifyTrie.scala new file mode 100644 index 00000000000..afaa0b1dccb --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/stages/VerifyTrie.scala @@ -0,0 +1,50 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.stages + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase + +class VerifyTrie extends TestBase { + + test("Trie.apply creates from map") { + val t = Trie(Map("hello" -> "hi")) + assert(t.get('h').isDefined) + } + + test("put and get for single characters") { + val t = new Trie().put("a", "b") + assert(t.get('a').isDefined) + } + + test("mapText replaces matching substrings") { + val t = Trie(Map("hello" -> "hi")) + assert(t.mapText("hello there") === "hi there") + } + + test("mapText with no matches returns original text") { + val t = Trie(Map("xyz" -> "abc")) + assert(t.mapText("hello world") === "hello world") + } + + test("mapText with multiple replacements") { + val t = Trie(Map("hello" -> "hi", "world" -> "earth")) + assert(t.mapText("hello world.") === "hi earth.") + } + + test("putAll adds all entries") { + val t = new Trie().putAll(Map("a" -> "1", "b" -> "2")) + assert(t.get('a').isDefined) + assert(t.get('b').isDefined) + } + + test("get returns None for missing key") { + val t = new Trie() + assert(t.get('z').isEmpty) + } + + test("mapText with overlapping keys longer key wins") { + val t = Trie(Map("he" -> "X", "hello" -> "Y")) + assert(t.mapText("hello.") === "Y.") + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/stages/VerifyUnicodeNormalize.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/stages/VerifyUnicodeNormalize.scala new file mode 100644 index 00000000000..c79a44f2b0b --- /dev/null +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/stages/VerifyUnicodeNormalize.scala @@ -0,0 +1,59 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.stages + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.spark.sql.types.{StringType, StructField} + +class VerifyUnicodeNormalize extends TestBase { + import spark.implicits._ + + test("normalizes unicode text") { + // e + combining acute accent (decomposed) vs precomposed e-acute + val decomposed = "cafe\u0301" + val df = Seq(decomposed).toDF("text") + val normalizer = new UnicodeNormalize() + .setInputCol("text") + .setOutputCol("normalized") + .setForm("NFC") + .setLower(false) + val result = normalizer.transform(df).select("normalized").collect() + assert(result.head.getString(0) === "caf\u00e9") + } + + test("lower=true lowercases output") { + val df = Seq("HELLO").toDF("text") + val normalizer = new UnicodeNormalize() + .setInputCol("text") + .setOutputCol("out") + .setLower(true) + val result = normalizer.transform(df).select("out").collect() + assert(result.head.getString(0) === "hello") + } + + test("lower=false preserves case") { + val df = Seq("Hello").toDF("text") + val normalizer = new UnicodeNormalize() + .setInputCol("text") + .setOutputCol("out") + .setLower(false) + val result = normalizer.transform(df).select("out").collect() + assert(result.head.getString(0).contains("H")) + } + + test("default form is NFKD") { + val normalizer = new UnicodeNormalize() + assert(normalizer.getForm === "NFKD") + } + + test("transformSchema adds output column") { + val df = Seq("a").toDF("text") + val normalizer = new UnicodeNormalize() + .setInputCol("text") + .setOutputCol("out") + val schema = normalizer.transformSchema(df.schema) + assert(schema.fieldNames.contains("out")) + assert(schema(schema.fieldIndex("out")) === StructField("out", StringType)) + } +} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/train/VerifyComputeModelStatistics.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/train/VerifyComputeModelStatistics.scala index 2dcab90bece..21a67bbf592 100644 --- a/core/src/test/scala/com/microsoft/azure/synapse/ml/train/VerifyComputeModelStatistics.scala +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/train/VerifyComputeModelStatistics.scala @@ -11,15 +11,17 @@ import com.microsoft.azure.synapse.ml.core.test.benchmarks.DatasetUtils import com.microsoft.azure.synapse.ml.core.test.fuzzing.{TestObject, TransformerFuzzing} import com.microsoft.azure.synapse.ml.train.TrainClassifierTestUtilities._ import com.microsoft.azure.synapse.ml.train.TrainRegressorTestUtilities._ +import org.apache.spark.ml.Pipeline import org.apache.spark.ml.classification.LogisticRegression import org.apache.spark.ml.evaluation.BinaryClassificationEvaluator import org.apache.spark.ml.feature.FastVectorAssembler -import org.apache.spark.ml.linalg.Vector +import org.apache.spark.ml.linalg.{Vector, Vectors} +import org.apache.spark.ml.param.ParamMap import org.apache.spark.ml.regression.GeneralizedLinearRegression import org.apache.spark.ml.util.MLReadable import org.apache.spark.sql._ import org.apache.spark.sql.functions._ -import org.apache.spark.sql.types.{DoubleType, StructField, StructType} +import org.apache.spark.sql.types.{DoubleType, MetadataBuilder, StructField, StructType} import scala.util.Random @@ -41,6 +43,479 @@ class VerifyComputeModelStatistics extends TransformerFuzzing[ComputeModelStatis (1, 4, 0.12, 0.34, 3) )).toDF(labelColumn, "col1", "col2", "col3", "col4") + private lazy val rankedBinaryDataset: DataFrame = { + import spark.implicits._ + val data = (1 to 100).map { rank => + val label = if (rank == 1 || rank == 11) 1.0 else 0.0 + val prediction = if (rank <= 10) 1.0 else 0.0 + val rawPrediction = Vectors.dense(0.0, (101 - rank).toDouble) + (label, prediction, rawPrediction) + }.toDF("label", SchemaConstants.SparkPredictionColumn, SchemaConstants.SparkRawPredictionColumn) + val modelName = SchemaConstants.ScoreModelPrefix + "_ranked binary" + val withLabel = SparkSchema.setLabelColumnName( + data, modelName, "label", SchemaConstants.ClassificationKind) + val withPrediction = SparkSchema.updateColumnMetadata( + withLabel, modelName, SchemaConstants.SparkPredictionColumn, SchemaConstants.ClassificationKind) + SparkSchema.updateColumnMetadata( + withPrediction, modelName, SchemaConstants.SparkRawPredictionColumn, SchemaConstants.ClassificationKind) + } + + private def rankedBinaryStatistics(metric: String): ComputeModelStatistics = + new ComputeModelStatistics() + .setLabelCol("label") + .setScoredLabelsCol(SchemaConstants.SparkPredictionColumn) + .setScoresCol(SchemaConstants.SparkRawPredictionColumn) + .setEvaluationMetric(metric) + + private def assertBinaryOnlyMetricsRejected(schema: StructType): Unit = { + Seq( + MetricConstants.AucSparkMetric -> "Error: AUC is not available for multiclass case", + MetricConstants.AreaUnderROCMetric -> "Error: AUC is not available for multiclass case", + MetricConstants.AreaUnderPRMetric -> "Error: areaUnderPR is not available for multiclass case") + .foreach { case (metric, expectedMessage) => + val error = intercept[IllegalArgumentException] { + new ComputeModelStatistics() + .setLabelCol("label") + .setEvaluationMetric(metric) + .transformSchema(schema) + } + + assert(error.getMessage === expectedMessage) + } + } + + test("areaUnderPR uses Spark trapezoidal precision-recall AUC") { + val evaluator = rankedBinaryStatistics(MetricConstants.AreaUnderPRMetric) + val result = evaluator.transform(rankedBinaryDataset) + val areaUnderPR = result.first().getAs[Double](MetricConstants.AreaUnderPRColumnName) + + assert(result.columns.last === MetricConstants.AreaUnderPRColumnName) + assert(result.columns.contains(MetricConstants.AreaUnderPRColumnName)) + assert(evaluator.transformSchema(rankedBinaryDataset.schema) === + StructType(Array(StructField(MetricConstants.AreaUnderPRColumnName, DoubleType)))) + assert(math.abs(areaUnderPR - 251.0 / 440.0) < 1e-8) + assert(math.abs(areaUnderPR - 13.0 / 22.0) > 0.01) + } + + test("areaUnderROC remains an AUC output alias") { + val aucEvaluator = rankedBinaryStatistics(MetricConstants.AucSparkMetric) + val auc = aucEvaluator + .transform(rankedBinaryDataset) + .first() + .getAs[Double](MetricConstants.AucColumnName) + val rocEvaluator = rankedBinaryStatistics(MetricConstants.AreaUnderROCMetric) + val areaUnderROCAlias = rocEvaluator + .transform(rankedBinaryDataset) + .first() + .getAs[Double](MetricConstants.AucColumnName) + val aucSchema = StructType(Array(StructField(MetricConstants.AucColumnName, DoubleType))) + + assert(aucEvaluator.transformSchema(rankedBinaryDataset.schema) === aucSchema) + assert(rocEvaluator.transformSchema(rankedBinaryDataset.schema) === aucSchema) + assert(math.abs(auc - 187.0 / 196.0) < 1e-8) + assert(math.abs(areaUnderROCAlias - 187.0 / 196.0) < 1e-8) + } + + test("all and classification metrics append binary metrics in runtime output order") { + val binaryDataset = CategoricalUtilities.setLevels( + rankedBinaryDataset, + "label", + Array(0.0, 1.0)) + val expectedColumns = List(MetricConstants.EvaluationType, MetricConstants.ConfusionMatrix) ++ + MetricConstants.BinaryClassificationColumns + Seq(MetricConstants.AllSparkMetrics, MetricConstants.ClassificationMetricsName).foreach { metric => + val evaluator = rankedBinaryStatistics(metric) + val result = evaluator.transform(binaryDataset) + val row = result.first() + val transformedSchema = evaluator.transformSchema(binaryDataset.schema) + + assert(result.columns.toList === expectedColumns) + assert(transformedSchema === + StructType(MetricConstants.BinaryClassificationColumns.map(StructField(_, DoubleType)))) + assert(math.abs(row.getAs[Double](MetricConstants.AucColumnName) - 187.0 / 196.0) < 1e-8) + assert(math.abs(row.getAs[Double](MetricConstants.AreaUnderPRColumnName) - 251.0 / 440.0) < 1e-8) + } + } + + test("multiclass classification schema retains the legacy common metrics") { + val multiclassData = spark.createDataFrame(Seq( + (0.0, 0.0), + (1.0, 1.0), + (2.0, 2.0))).toDF("label", "prediction") + val modelName = SchemaConstants.ScoreModelPrefix + "_multiclass" + val withLabel = SparkSchema.setLabelColumnName( + multiclassData, modelName, "label", SchemaConstants.ClassificationKind) + val withPrediction = SparkSchema.updateColumnMetadata( + withLabel, modelName, "prediction", SchemaConstants.ClassificationKind) + val multiclass = CategoricalUtilities.setLevels( + withPrediction, + "label", + Array(0.0, 1.0, 2.0)) + val expectedRuntimeColumns = + List(MetricConstants.EvaluationType, MetricConstants.ConfusionMatrix) ++ + MetricConstants.ClassificationColumns ++ + List(MetricConstants.AverageAccuracy, + MetricConstants.MacroAveragedPrecision, + MetricConstants.MacroAveragedRecall) + + Seq(MetricConstants.AllSparkMetrics, MetricConstants.ClassificationMetricsName).foreach { metric => + val evaluator = new ComputeModelStatistics().setEvaluationMetric(metric) + val schema = evaluator.transformSchema(multiclass.schema) + val result = evaluator.transform(multiclass) + + assert(schema.fieldNames.toList === MetricConstants.ClassificationColumns) + assert(!schema.fieldNames.contains(MetricConstants.AucColumnName)) + assert(!schema.fieldNames.contains(MetricConstants.AreaUnderPRColumnName)) + assert(result.columns.toList === expectedRuntimeColumns) + } + } + + test("classification schema without cardinality metadata retains the legacy common metrics") { + Seq(MetricConstants.AllSparkMetrics, MetricConstants.ClassificationMetricsName).foreach { metric => + val schema = rankedBinaryStatistics(metric).transformSchema(rankedBinaryDataset.schema) + + assert(schema.fieldNames.toList === MetricConstants.ClassificationColumns) + } + } + + test("transformSchema rejects binary-only metrics for multiclass MML categorical labels") { + val schema = CategoricalUtilities.setLevels( + spark.createDataFrame(Seq( + (0.0, 0.0), + (1.0, 1.0), + (2.0, 2.0))).toDF("label", "prediction"), + "label", + Array(0.0, 1.0, 2.0)).schema + + assertBinaryOnlyMetricsRejected(schema) + } + + test("transformSchema preserves binary-only metric schema when configured labelCol is absent") { + val schema = spark.createDataFrame(Seq( + (0.0, 0.9), + (1.0, 0.8))).toDF("prediction", "rawPrediction").schema + val evaluator = new ComputeModelStatistics() + .setLabelCol("label") + .setEvaluationMetric(MetricConstants.AreaUnderPRMetric) + + assert(evaluator.transformSchema(schema) === + StructType(Array(StructField(MetricConstants.AreaUnderPRColumnName, DoubleType)))) + } + + test("areaUnderPR rejects multiclass and unsupported metric inputs") { + val multiclass = spark.createDataFrame(Seq( + (0.0, 0.0, 0.9), + (1.0, 1.0, 0.8), + (2.0, 2.0, 0.7))).toDF("label", "prediction", "rawPrediction") + val multiclassError = intercept[Exception] { + new ComputeModelStatistics() + .setLabelCol("label") + .setScoredLabelsCol("prediction") + .setScoresCol("rawPrediction") + .setEvaluationMetric(MetricConstants.AreaUnderPRMetric) + .transform(multiclass) + } + assert(multiclassError.getMessage === "Error: areaUnderPR is not available for multiclass case") + + assertThrows[Exception] { + rankedBinaryStatistics("averagePrecision").transform(rankedBinaryDataset) + } + } + + private def addScoreColumnMetadata(dataset: DataFrame, + modelName: String, + columnName: String, + columnKind: String, + scoreValueKind: String): DataFrame = { + val existingMetadata = dataset.schema(columnName).metadata + val mmlBuilder = new MetadataBuilder() + if (existingMetadata.contains(SchemaConstants.MMLTag)) { + mmlBuilder.withMetadata(existingMetadata.getMetadata(SchemaConstants.MMLTag)) + } + val modelMetadata = new MetadataBuilder() + .putString(SchemaConstants.ScoreColumnKind, columnKind) + .putString(SchemaConstants.ScoreValueKind, scoreValueKind) + .build() + val updatedMetadata = new MetadataBuilder() + .withMetadata(existingMetadata) + .putMetadata( + SchemaConstants.MMLTag, + mmlBuilder.putMetadata(modelName, modelMetadata).build()) + .build() + dataset.withColumn(columnName, col(columnName).as(columnName, updatedMetadata)) + } + + private def addScoredModelMetadata(dataset: DataFrame, + modelName: String, + labelCol: String, + scoreValueKind: String): DataFrame = { + val withLabel = SparkSchema.setLabelColumnName(dataset, modelName, labelCol, scoreValueKind) + SparkSchema.updateColumnMetadata( + withLabel, modelName, SchemaConstants.SparkPredictionColumn, scoreValueKind) + } + + test("Explicit settings select classification after a stale regression score is dropped") { + val regressionLabel = "regressionLabel" + val input = dataset + .withColumn(regressionLabel, col("col2") + col("col3")) + .select(col(regressionLabel), col(labelColumn), col("col1"), col("col2"), col("col3"), col("col4")) + val regressionScored = createLinearRegressor(regressionLabel).fit(input).transform(input) + + val regressionEvaluation = new ComputeModelStatistics().transform(regressionScored) + assert(regressionEvaluation.columns.contains(MetricConstants.MseColumnName)) + + val classifierInput = regressionScored.drop(SchemaConstants.SparkPredictionColumn) + assert(classifierInput.schema(regressionLabel).metadata.contains(SchemaConstants.MMLTag)) + val classificationScored = createLR.setLabelCol(labelColumn).fit(classifierInput).transform(classifierInput) + val classificationEvaluation = new ComputeModelStatistics() + .setLabelCol(labelColumn) + .setScoredLabelsCol(SchemaConstants.SparkPredictionColumn) + .setEvaluationMetric(MetricConstants.ClassificationMetricsName) + .transform(classificationScored) + + assert(classificationEvaluation.columns.contains(MetricConstants.AccuracyColumnName)) + assert(!classificationEvaluation.columns.contains(MetricConstants.MseColumnName)) + } + + test("Explicit columns and metric beat unrelated scored-model metadata") { + val unrelatedLabel = "unrelatedLabel" + val selectedLabel = "selectedLabel" + val selectedPrediction = "selectedPrediction" + val unrelatedModel = SchemaConstants.ScoreModelPrefix + "_unrelated" + val wrongKindModel = SchemaConstants.ScoreModelPrefix + "_wrong_kind" + val input = spark.createDataFrame(Seq( + (1.0, 0.0, 1.0, 0.0), + (0.0, 1.0, 0.0, 1.0), + (1.0, 0.0, 1.0, 0.0), + (0.0, 1.0, 0.0, 1.0))) + .toDF(unrelatedLabel, selectedLabel, SchemaConstants.SparkPredictionColumn, selectedPrediction) + val withUnrelatedModel = addScoredModelMetadata( + input, unrelatedModel, unrelatedLabel, SchemaConstants.ClassificationKind) + val scored = addScoredModelMetadata( + withUnrelatedModel, wrongKindModel, selectedLabel, SchemaConstants.RegressionKind) + + val result = new ComputeModelStatistics() + .setLabelCol(selectedLabel) + .setScoredLabelsCol(selectedPrediction) + .setEvaluationMetric(MetricConstants.AccuracySparkMetric) + .transform(scored) + + assert(result.first().getAs[Double](MetricConstants.AccuracyColumnName) === 1.0) + } + + test("Multiple complete scored-model metadata candidates fail deterministically") { + val modelA = SchemaConstants.ScoreModelPrefix + "_a" + val modelB = SchemaConstants.ScoreModelPrefix + "_b" + val labelA = "labelA" + val labelB = "labelB" + val input = spark.createDataFrame(Seq((0.0, 1.0, 0.0))) + .toDF(labelA, labelB, SchemaConstants.SparkPredictionColumn) + val withModelB = addScoredModelMetadata( + input, modelB, labelB, SchemaConstants.ClassificationKind) + val withBothModels = addScoredModelMetadata( + withModelB, modelA, labelA, SchemaConstants.RegressionKind) + + val error = intercept[IllegalArgumentException] { + new ComputeModelStatistics().transformSchema(withBothModels.schema) + } + val expectedCandidates = + s"[$modelA (label=$labelA, kind=${SchemaConstants.RegressionKind}, " + + s"prediction=${SchemaConstants.SparkPredictionColumn}), " + + s"$modelB (label=$labelB, kind=${SchemaConstants.ClassificationKind}, " + + s"prediction=${SchemaConstants.SparkPredictionColumn})]" + assert(error.getMessage.contains(expectedCandidates)) + assert(error.getMessage.contains("Set labelCol and evaluationMetric")) + } + test("Explicit evaluation metric omits irrelevant all-metrics hint when labelCol is missing") { + val input = spark.createDataFrame(Seq((0.0, 1.0))).toDF("label", "feature") + val error = intercept[IllegalArgumentException] { + new ComputeModelStatistics() + .setEvaluationMetric(MetricConstants.RegressionMetricsName) + .transformSchema(input.schema) + } + assert(error.getMessage.contains("Set labelCol, or score the dataset")) + assert(!error.getMessage.contains( + s"evaluationMetric must not be '${MetricConstants.AllSparkMetrics}'")) + } + + test("Missing default score column produces an actionable error") { + val label = "label" + val input = spark.createDataFrame(Seq((0.0, 1.0), (1.0, 2.0))).toDF(label, "feature") + val error = intercept[IllegalArgumentException] { + new ComputeModelStatistics() + .setLabelCol(label) + .setEvaluationMetric(MetricConstants.RegressionMetricsName) + .transform(input) + } + assert(error.getMessage.contains("regression prediction/score column ")) + assert(error.getMessage.contains("setScoresCol")) + assert(error.getMessage.contains("Available columns: [feature, label]")) + } + + test("Invalid explicit scores column fails only for score-consuming metrics") { + val label = "label" + val prediction = "selectedPrediction" + val input = spark.createDataFrame(Seq((0.0, 0.0), (1.0, 1.0))).toDF(label, prediction) + val statistics = new ComputeModelStatistics() + .setLabelCol(label) + .setScoredLabelsCol(prediction) + .setScoresCol("missingScore") + val accuracy = statistics + .setEvaluationMetric(MetricConstants.AccuracySparkMetric) + .transform(input) + .first() + .getAs[Double](MetricConstants.AccuracyColumnName) + assert(accuracy === 1.0) + val error = intercept[IllegalArgumentException] { + statistics + .setEvaluationMetric(MetricConstants.AucSparkMetric) + .transform(input) + } + assert(error.getMessage.contains("classification score column 'missingScore'")) + assert(error.getMessage.contains("setScoresCol")) + assert(error.getMessage.contains("Available columns: [label, selectedPrediction]")) + } + test("Single complete scored-model metadata remains supported") { + val modelName = SchemaConstants.ScoreModelPrefix + "_single" + val label = "label" + val input = spark.createDataFrame(Seq((0.0, 0.0), (1.0, 1.0))) + .toDF(label, SchemaConstants.SparkPredictionColumn) + val scored = addScoredModelMetadata(input, modelName, label, SchemaConstants.RegressionKind) + val evaluator = new ComputeModelStatistics() + .setLabelCol(label.toUpperCase) + .setEvaluationMetric(MetricConstants.RegressionMetricsName) + val activeSession = SparkSession.getActiveSession + val defaultSession = SparkSession.getDefaultSession + SparkSession.clearActiveSession() + SparkSession.setDefaultSession(spark) + try assert(evaluator.transformSchema(scored.schema).fieldNames.contains(MetricConstants.MseColumnName)) + finally { + activeSession.fold(SparkSession.clearActiveSession())(SparkSession.setActiveSession) + defaultSession.fold(SparkSession.clearDefaultSession())(SparkSession.setDefaultSession) + } + val result = evaluator.transform(scored) + assert(result.first().getAs[Double](MetricConstants.MseColumnName) === 0.0) + } + + test("Equivalent complete model metadata is de-duplicated deterministically") { + val modelA = SchemaConstants.ScoreModelPrefix + "_duplicate_a" + val modelB = SchemaConstants.ScoreModelPrefix + "_duplicate_b" + val label = "label" + val input = spark.createDataFrame(Seq((0.0, 0.0), (1.0, 1.0))) + .toDF(label, SchemaConstants.SparkPredictionColumn) + val withModelB = addScoredModelMetadata(input, modelB, label, SchemaConstants.RegressionKind) + val withDuplicates = addScoredModelMetadata(withModelB, modelA, label, SchemaConstants.RegressionKind) + val result = new ComputeModelStatistics().transform(withDuplicates) + assert(result.first().getAs[Double](MetricConstants.MseColumnName) === 0.0) + } + + test("Conflicting prediction metadata fails independently of schema column order") { + val model = SchemaConstants.ScoreModelPrefix + "_conflicting" + val label = "label" + val predictionA = "predictionA" + val predictionB = "predictionB" + val input = spark.createDataFrame(Seq((0.0, 0.0, 0.0), (1.0, 1.0, 1.0))) + .toDF(label, predictionA, predictionB) + val withLabel = SparkSchema.setLabelColumnName(input, model, label, SchemaConstants.RegressionKind) + val withPredictionA = addScoreColumnMetadata( + withLabel, model, predictionA, SchemaConstants.SparkPredictionColumn, SchemaConstants.RegressionKind) + val conflicting = addScoreColumnMetadata( + withPredictionA, model, predictionB, SchemaConstants.SparkPredictionColumn, SchemaConstants.RegressionKind) + val schemas = Seq( + conflicting.schema, + conflicting.select(col(predictionB), col(label), col(predictionA)).schema) + + val messages = schemas.map { schema => + intercept[IllegalArgumentException] { + new ComputeModelStatistics().transformSchema(schema) + }.getMessage + } + + assert(messages.distinct.size === 1) + assert(messages.head.indexOf(s"prediction=$predictionA") < messages.head.indexOf(s"prediction=$predictionB")) + } + + test("Complete explicit settings override ambiguous scored-model metadata") { + val label = "label" + val selectedPrediction = "selectedPrediction" + val metadataPredictionA = "metadataPredictionA" + val metadataPredictionB = "metadataPredictionB" + val input = spark.createDataFrame(Seq( + (0.0, 0.0, 1.0, 1.0), + (1.0, 1.0, 0.0, 0.0))) + .toDF(label, selectedPrediction, metadataPredictionA, metadataPredictionB) + val modelA = SchemaConstants.ScoreModelPrefix + "_explicit_a" + val modelB = SchemaConstants.ScoreModelPrefix + "_explicit_b" + val withModelALabel = SparkSchema.setLabelColumnName( + input, modelA, label, SchemaConstants.ClassificationKind) + val withModelA = addScoreColumnMetadata( + withModelALabel, modelA, metadataPredictionA, + SchemaConstants.SparkPredictionColumn, SchemaConstants.ClassificationKind) + val withModelBLabel = SparkSchema.setLabelColumnName( + withModelA, modelB, label, SchemaConstants.ClassificationKind) + val ambiguous = addScoreColumnMetadata( + withModelBLabel, modelB, metadataPredictionB, + SchemaConstants.SparkPredictionColumn, SchemaConstants.ClassificationKind) + + val evaluator = new ComputeModelStatistics() + .setLabelCol(label.toUpperCase) + .setScoredLabelsCol(selectedPrediction) + .setEvaluationMetric(MetricConstants.AccuracySparkMetric) + val result = evaluator.transform(ambiguous) + assert(evaluator.transformSchema(ambiguous.schema).fieldNames.contains(MetricConstants.AccuracyColumnName)) + assert(result.first().getAs[Double](MetricConstants.AccuracyColumnName) === 1.0) + } + + test("Duplicate raw-score metadata is rejected only when the metric consumes it") { + val model = SchemaConstants.ScoreModelPrefix + "_duplicate_raw" + val input = spark.createDataFrame(Seq( + (0.0, 0.0, 0.2, 0.3), + (1.0, 1.0, 0.8, 0.7))).toDF("label", "prediction", "rawA", "rawB") + val withLabel = SparkSchema.setLabelColumnName( + input, model, "label", SchemaConstants.ClassificationKind) + val withPrediction = addScoreColumnMetadata( + withLabel, model, "prediction", + SchemaConstants.SparkPredictionColumn, SchemaConstants.ClassificationKind) + val withRawA = addScoreColumnMetadata( + withPrediction, model, "rawA", + SchemaConstants.SparkRawPredictionColumn, SchemaConstants.ClassificationKind) + val scored = addScoreColumnMetadata( + withRawA, model, "rawB", + SchemaConstants.SparkRawPredictionColumn, SchemaConstants.ClassificationKind) + + val accuracy = new ComputeModelStatistics() + .setEvaluationMetric(MetricConstants.AccuracySparkMetric) + .transform(scored) + assert(accuracy.first().getAs[Double](MetricConstants.AccuracyColumnName) === 1.0) + + val error = intercept[IllegalArgumentException] { + new ComputeModelStatistics().setEvaluationMetric(MetricConstants.AucSparkMetric).transform(scored) + } + assert(error.getMessage.contains("rawPrediction columns [rawA:Classification, rawB:Classification]")) + } + + test("Copy and pipeline preserve explicit statistics parameters") { + val label = "label" + val prediction = "selectedPrediction" + val input = spark.createDataFrame(Seq((0.0, 0.0), (1.0, 1.0))).toDF(label, prediction) + val configured = new ComputeModelStatistics() + .setLabelCol(label.toUpperCase) + .setScoresCol(prediction.toUpperCase) + .setEvaluationMetric(MetricConstants.MseSparkMetric) + val copied = configured.copy(ParamMap.empty).asInstanceOf[ComputeModelStatistics] + + assert(copied.uid === configured.uid) + assert(copied.getLabelCol === label.toUpperCase) + assert(copied.getScoresCol === prediction.toUpperCase) + assert(copied.getEvaluationMetric === MetricConstants.MseSparkMetric) + + val pipelineResult = new Pipeline() + .setStages(Array(configured)) + .fit(input) + .transform(input) + assert(pipelineResult.first().getAs[Double](MetricConstants.MseColumnName) === 0.0) + } + test("Verify multiclass evaluation is not slow for large number of labels") { val numRows = 4096 import spark.implicits._ @@ -187,7 +662,8 @@ class VerifyComputeModelStatistics extends TransformerFuzzing[ComputeModelStatis val _ = new ComputeModelStatistics().transform(scoredDataset) val evaluatedSchema = new ComputeModelStatistics().transformSchema(scoredDataset.schema) - assert(evaluatedSchema == StructType(MetricConstants.ClassificationColumns.map(StructField(_, DoubleType)))) + assert(evaluatedSchema == + StructType(MetricConstants.BinaryClassificationColumns.map(StructField(_, DoubleType)))) } test("Verify computing statistics on generic spark ML estimators is supported") { diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/train/VerifyComputePerInstanceStatistics.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/train/VerifyComputePerInstanceStatistics.scala index 932661c8978..3fc34e2fa7b 100644 --- a/core/src/test/scala/com/microsoft/azure/synapse/ml/train/VerifyComputePerInstanceStatistics.scala +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/train/VerifyComputePerInstanceStatistics.scala @@ -10,6 +10,7 @@ import com.microsoft.azure.synapse.ml.train.TrainClassifierTestUtilities._ import com.microsoft.azure.synapse.ml.train.TrainRegressorTestUtilities._ import org.apache.spark.ml.classification.LogisticRegression import org.apache.spark.ml.feature.FastVectorAssembler +import org.apache.spark.ml.linalg.{Vector, Vectors} import org.apache.spark.sql._ /** Tests to validate the functionality of Compute Per Instance Statistics module. */ @@ -121,6 +122,41 @@ class VerifyComputePerInstanceStatistics extends TestBase { validatePerInstanceClassificationStatistics(evaluatedData) } + test("Verify log loss uses the distinct label count when levels metadata is absent") { + // numLevels is derived from a distinct count over the label column. Pin that here: the label + // has three distinct values, and the final row is scored with a label equal to that count, so + // it must fall into the "no label seen in training" branch instead of indexing the vector. + // An off-by-one in the distinct count shows up either as an index error or as a wrong loss. + val probs: Vector = Vectors.dense(0.7, 0.2, 0.1) + val scoredData = spark.createDataFrame(Seq( + (0.0, probs, probs, 0.0), + (1.0, probs, probs, 1.0), + (2.0, probs, probs, 2.0), + (0.0, probs, probs, 0.0), + (1.0, probs, probs, 1.0), + (2.0, probs, probs, 3.0))) + .toDF(labelColumn, "scoresCol", "probabilitiesCol", "scoredLabelsCol") + + val evaluatedData = new ComputePerInstanceStatistics() + .setLabelCol(labelColumn) + .setScoredLabelsCol("scoredLabelsCol") + .setScoresCol("scoresCol") + .setScoredProbabilitiesCol("probabilitiesCol") + .setEvaluationMetric(MetricConstants.ClassificationMetricsName) + .transform(scoredData) + + val penalized = -Math.log(ComputePerInstanceStatistics.Epsilon) + val rows = evaluatedData.select("scoredLabelsCol", MetricConstants.LogLossMetric).collect() + assert(rows.length === 6) + assert(rows.count(r => r.getDouble(1) === penalized) === 1) + rows.foreach { row => + val scoredLabel = row.getDouble(0).toInt + val logLoss = row.getDouble(1) + val expected = if (scoredLabel < 3) -Math.log(probs(scoredLabel)) else penalized + assert(logLoss === expected) + } + } + private def validatePerInstanceRegressionStatistics(evaluatedData: DataFrame): Unit = { // Validate the per instance statistics evaluatedData.collect().foreach(row => { diff --git a/deep-learning/src/main/scala/com/microsoft/azure/synapse/ml/onnx/ONNXModel.scala b/deep-learning/src/main/scala/com/microsoft/azure/synapse/ml/onnx/ONNXModel.scala index bee36327655..ec4c56e03f1 100644 --- a/deep-learning/src/main/scala/com/microsoft/azure/synapse/ml/onnx/ONNXModel.scala +++ b/deep-learning/src/main/scala/com/microsoft/azure/synapse/ml/onnx/ONNXModel.scala @@ -86,6 +86,15 @@ trait ONNXModelParams extends Params with HasMiniBatcher with HasFeedFetchDicts def setDeviceType(value: String): this.type = set(deviceType, value) + /** + * Returns the configured deviceType normalized to upper-case ("CPU"/"CUDA"), if set. The deviceType + * Param validator accepts any case (validated via `x.toUpperCase()`), so all internal deviceType + * comparisons (GPU auto-detection, explicit-CUDA fail-fast) must read this normalized accessor + * instead of the raw Param value -- otherwise a validator-accepted but differently-cased value like + * "cuda" or "Cuda" would silently bypass CUDA handling entirely. + */ + def getNormalizedDeviceType: Option[String] = get(deviceType).map(_.toUpperCase) + val optimizationLevel: Param[String] = new Param[String]( this, "optimizationLevel", @@ -237,7 +246,8 @@ class ONNXModel(override val uid: String) val batchedDF = getMiniBatcher.transform(dataset) val (coerced, feedDict) = coerceBatchedDf(batchedDF) val modelBc = broadcastedModelPayload.getOrElse(rebroadcastModelPayload(dataset.sparkSession)) - val (fetchDicts, devType, optLevel) = (getFetchDict, get(deviceType), OptLevel.valueOf(getOptimizationLevel)) + val (fetchDicts, devType, optLevel) = + (getFetchDict, getNormalizedDeviceType, OptLevel.valueOf(getOptimizationLevel)) val outputDf = coerced.mapPartitions { rows => @@ -246,7 +256,8 @@ class ONNXModel(override val uid: String) val gpuDeviceId = selectGpuDevice(devType) val env = OrtEnvironment.getEnvironment logInfo(s"Task:$taskId;DeviceType=$devType;DeviceId=$gpuDeviceId;OptimizationLevel=$optLevel") - val session = createOrtSession(payload, env, optLevel, gpuDeviceId) + val session = createOrtSession( + payload, env, optLevel, gpuDeviceId, explicitCudaRequested = devType.contains("CUDA")) applyModel(session, env, feedDict, fetchDicts, inputSchema)(rows) } diff --git a/deep-learning/src/main/scala/com/microsoft/azure/synapse/ml/onnx/ONNXRuntime.scala b/deep-learning/src/main/scala/com/microsoft/azure/synapse/ml/onnx/ONNXRuntime.scala index 77768a7cbeb..56a7a09d035 100644 --- a/deep-learning/src/main/scala/com/microsoft/azure/synapse/ml/onnx/ONNXRuntime.scala +++ b/deep-learning/src/main/scala/com/microsoft/azure/synapse/ml/onnx/ONNXRuntime.scala @@ -3,7 +3,6 @@ package com.microsoft.azure.synapse.ml.onnx -import ai.onnxruntime.OrtException.OrtErrorCode import ai.onnxruntime.OrtSession.SessionOptions import ai.onnxruntime.OrtSession.SessionOptions.OptLevel import ai.onnxruntime._ @@ -22,25 +21,76 @@ import scala.jdk.CollectionConverters._ * ONNXRuntime: A wrapper around the ONNX Runtime (ORT) */ object ONNXRuntime extends Logging { + // Extracted so createOrtSession's control flow fits scalastyle's method-length limit; the message + // text/rationale is unchanged from what was previously inlined at each throw/log call site. + private def noGpuResourceAssignedMessage: String = + "deviceType=CUDA was explicitly requested, but no Spark \"gpu\" resource is assigned to this " + + "executor/task, so there is no GPU device id to use. Configure Spark's GPU resource " + + "allocation (for example spark.executor.resource.gpu.amount and " + + "spark.task.resource.gpu.amount) so a gpu resource is assigned to this task, or set " + + "deviceType=CPU to run on CPU intentionally." + + private def explicitCudaProviderFailedMessage(gpuDeviceId: Option[Int], exp: OrtException): String = + s"deviceType=CUDA was explicitly requested and a GPU device (id ${gpuDeviceId.get}) was " + + s"found on this executor, but adding CUDA support failed with error code ${exp.getCode}. " + + s"Most likely the ONNX runtime supplied to the cluster is the default " + + s"com.microsoft.onnxruntime:onnxruntime (CPU-only) artifact, or CUDA/cuDNN aren't " + + s"installed on this node. Add com.microsoft.onnxruntime:onnxruntime_gpu:{version} " + + s"(excluding the transitive onnxruntime CPU artifact) and a matching CUDA/cuDNN runtime " + + s"for GPU acceleration, or set deviceType=CPU to run on CPU intentionally." + + private def autoFallbackProviderFailedMessage(gpuDeviceId: Option[Int], exp: OrtException): String = + s"GPU device is found on executor nodes with id ${gpuDeviceId.get}, " + + s"but adding CUDA support failed with error code ${exp.getCode}. Most likely the ONNX " + + s"runtime supplied to the cluster is the default com.microsoft.onnxruntime:onnxruntime " + + s"(CPU-only) artifact, or CUDA/cuDNN aren't installed on this node. Add " + + s"com.microsoft.onnxruntime:onnxruntime_gpu:{version} (excluding the transitive onnxruntime " + + s"CPU artifact) and a matching CUDA/cuDNN runtime for GPU acceleration. Falling back to CPU. " + + s"Exception details: ${exp.toString}" + private[onnx] def createOrtSession(modelContent: Array[Byte], ortEnv: OrtEnvironment, optLevel: OptLevel = OptLevel.ALL_OPT, - gpuDeviceId: Option[Int] = None): OrtSession = { - val options = new SessionOptions() - - try { - gpuDeviceId.foreach(options.addCUDA) - } catch { - case exp: OrtException if exp.getCode == OrtErrorCode.ORT_INVALID_ARGUMENT => - val err = s"GPU device is found on executor nodes with id ${gpuDeviceId.get}, " + - s"but adding CUDA support failed. Most likely the ONNX runtime supplied to the cluster " + - s"does not support GPU. Please install com.microsoft.onnxruntime:onnxruntime_gpu:{version} " + - s"instead for optimal performance. Exception details: ${exp.toString}" - logError(err) + gpuDeviceId: Option[Int] = None, + explicitCudaRequested: Boolean = false): OrtSession = { + // deviceType=CUDA is an explicit request for GPU acceleration. If Spark never assigned a "gpu" + // resource to this executor/task, gpuDeviceId is None and addCUDA below would simply never be + // attempted -- silently handing back a working CPU session with no error at all. That is the same + // "silently broken GPU" failure mode this change must not reintroduce, so fail before creating any + // session rather than let CPU inference proceed unannounced. + if (explicitCudaRequested && gpuDeviceId.isEmpty) { + throw new IllegalStateException(noGpuResourceAssignedMessage) } - options.setOptimizationLevel(optLevel) - ortEnv.createSession(modelContent, options) + // SessionOptions owns a native handle that ONNX Runtime's own examples close via try-with-resources + // right after createSession returns: the session copies what it needs from options during + // construction and never needs the options object again, so closing it afterward is safe on every + // path. Use this file's established using(...) resource-cleanup pattern (see applyModel below) so + // SessionOptions is closed whether we fall through to a normal/auto-fallback session, or throw the + // explicit-CUDA fail-fast error -- but never before createSession has actually consumed it. + using(new SessionOptions()) { options => + try { + gpuDeviceId.foreach(options.addCUDA) + } catch { + // A "gpu" resource was assigned (gpuDeviceId is defined) but adding CUDA support still failed. + // Silently continuing on CPU here would produce a success-shaped result while hiding a severe, + // hard-to-notice performance regression -- exactly the "silently broken GPU" failure mode this + // dependency change must not reintroduce. Fail fast with an actionable error instead. There is + // currently no parameter to opt in to a graceful CPU fallback for an explicit CUDA request; add + // one deliberately before relaxing this if that behavior is ever needed -- do not silently + // reinstate it here. + case exp: OrtException if explicitCudaRequested => + throw new IllegalStateException(explicitCudaProviderFailedMessage(gpuDeviceId, exp), exp) + // deviceType was left unset (auto-detection): a "gpu" Spark resource was found, but CUDA isn't + // usable here. This wasn't an explicit ask for GPU, so log a clear, actionable error and + // continue on CPU rather than failing an otherwise-working job. + case exp: OrtException => + logError(autoFallbackProviderFailedMessage(gpuDeviceId, exp)) + } + + options.setOptimizationLevel(optLevel) + ortEnv.createSession(modelContent, options) + }.get } private[onnx] def selectGpuDevice(deviceType: Option[String]): Option[Int] = { @@ -86,7 +136,10 @@ object ONNXRuntime extends Logging { case (_, outputName) => val i = session.getOutputInfo.asScala.keysIterator.indexOf(outputName) val outputValue: OnnxValue = result.get(i) - mapOnnxValueToArray(outputValue) + outputValue.getInfo match { + case _: SequenceInfo => ONNXValueConverter.mapSequenceToArray(outputValue) + case _ => mapOnnxValueToArray(outputValue) + } }.toSeq }.get diff --git a/deep-learning/src/main/scala/com/microsoft/azure/synapse/ml/onnx/ONNXValueConverter.scala b/deep-learning/src/main/scala/com/microsoft/azure/synapse/ml/onnx/ONNXValueConverter.scala new file mode 100644 index 00000000000..5b3ab8578f6 --- /dev/null +++ b/deep-learning/src/main/scala/com/microsoft/azure/synapse/ml/onnx/ONNXValueConverter.scala @@ -0,0 +1,68 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.onnx + +import ai.onnxruntime.OnnxValue + +private[onnx] object ONNXValueConverter { + + /** + * Eagerly copies a sequence to JVM values and closes every child handle returned by ONNX Runtime. + * The caller remains responsible for closing the outer value through its owning OrtSession.Result. + */ + def mapSequenceToArray(value: OnnxValue): Seq[Any] = { + value.getValue match { + case values: java.util.List[_] => unwrapList(values) + case other => + val valueType = Option(other).map(_.getClass.getName).getOrElse("null") + throw new IllegalArgumentException(s"Expected an ONNX sequence value, but found $valueType") + } + } + + private def unwrapValue(value: Any): Any = value match { + case values: java.util.List[_] => unwrapList(values) + case map: java.util.Map[_, _] => unwrapMap(map) + case other => other + } + + private def unwrapOnnxValue(value: OnnxValue): Any = unwrapValue(value.getValue) + + private def unwrapList(values: java.util.List[_]): Vector[Any] = { + val elements = values.toArray + val closeables = elements.collect { case value: OnnxValue => value } + + try { + elements.iterator.map { + case value: OnnxValue => unwrapOnnxValue(value) + case other => unwrapValue(other) + }.toVector + } finally { + closeables.foreach(_.close()) + } + } + + private def unwrapMap(map: java.util.Map[_, _]): Map[Any, Any] = { + val entries = map.entrySet().toArray.iterator.map { + case entry: java.util.Map.Entry[_, _] => entry.getKey -> entry.getValue + }.toVector + val closeables = entries.iterator.flatMap { + case (key, value) => Iterator(key, value).collect { case onnxValue: OnnxValue => onnxValue } + }.toVector + + try { + entries.iterator.map { + case (key: OnnxValue, value: OnnxValue) => + unwrapOnnxValue(key) -> unwrapOnnxValue(value) + case (key: OnnxValue, value) => + unwrapOnnxValue(key) -> unwrapValue(value) + case (key, value: OnnxValue) => + unwrapValue(key) -> unwrapOnnxValue(value) + case (key, value) => + unwrapValue(key) -> unwrapValue(value) + }.toMap + } finally { + closeables.foreach(_.close()) + } + } +} diff --git a/deep-learning/src/test/scala/com/microsoft/azure/synapse/ml/onnx/ONNXModelSuite.scala b/deep-learning/src/test/scala/com/microsoft/azure/synapse/ml/onnx/ONNXModelSuite.scala index 5cd28f57192..f1ffe98ea65 100644 --- a/deep-learning/src/test/scala/com/microsoft/azure/synapse/ml/onnx/ONNXModelSuite.scala +++ b/deep-learning/src/test/scala/com/microsoft/azure/synapse/ml/onnx/ONNXModelSuite.scala @@ -3,9 +3,10 @@ package com.microsoft.azure.synapse.ml.onnx +import ai.onnxruntime.OrtEnvironment import breeze.linalg.{argmax, argtopk} import com.microsoft.azure.synapse.ml.build.BuildInfo -import com.microsoft.azure.synapse.ml.core.env.FileUtilities +import com.microsoft.azure.synapse.ml.core.env.{FileUtilities, StreamUtilities} import com.microsoft.azure.synapse.ml.core.test.base.TestBase import com.microsoft.azure.synapse.ml.core.test.fuzzing.{TestObject, TransformerFuzzing} import com.microsoft.azure.synapse.ml.core.utils.BreezeUtils._ @@ -16,7 +17,7 @@ import org.apache.spark.SparkException import org.apache.spark.injections.UDFUtils import org.apache.spark.ml.image.ImageSchema import org.apache.spark.ml.linalg.{DenseVector, Vector, Vectors} -import org.apache.spark.ml.param.Param +import org.apache.spark.ml.param.{Param, ParamMap} import org.apache.spark.ml.util.MLReadable import org.apache.spark.sql.expressions.UserDefinedFunction import org.apache.spark.sql.functions._ @@ -50,8 +51,9 @@ class ONNXModelSuite extends TestBase private implicit val eqFloat: Equality[Float] = TolerantNumerics.tolerantFloatEquality(1E-5f) private implicit val eqMap: Equality[Map[Long, Float]] = mapEq[Long, Float] private implicit val eqSeqDouble: Equality[Seq[Double]] = (a: Seq[Double], b: Any) => { - b match { - case sd: Seq[Double] => a.zip(sd).forall(x => x._1 === x._2) + // Using @unchecked because Seq[Double] type parameter is erased at runtime + (b: @unchecked) match { + case sd: Seq[Double @unchecked] => a.zip(sd).forall(x => x._1 === x._2) case _ => false } } @@ -127,6 +129,160 @@ class ONNXModelSuite extends TestBase assert(caught.getMessage.contains("IllegalArgumentException")) } + // Minimal dotted-version comparator (e.g. "1.17.3" vs "1.16.3"). Avoids hard-coding an exact ONNX + // Runtime version so the GH2417 regression check keeps passing across future confirmed upgrades. + private def versionAtLeast(actual: String, minimum: String): Boolean = { + def parts(v: String): Seq[Int] = v.split("\\.").map(_.takeWhile(_.isDigit)).map(s => if (s.isEmpty) 0 else s.toInt) + val (a, m) = (parts(actual), parts(minimum)) + Ordering.Iterable[Int].gteq(a.padTo(m.length, 0), m.padTo(a.length, 0)) + } + + test("ONNXModel loads upgraded runtime for local CPU inference (GH2417)") { + val runtimeVersion = StreamUtilities.using(OrtEnvironment.getEnvironment)(_.getVersion).get + // GH2417: ONNX Runtime 1.8.1 fails to load its natives for local Spark 3.5 inference (macOS/Linux/ + // Windows). 1.16.3 was the smallest version confirmed to fix it; guard against ever regressing below it. + assert(versionAtLeast(runtimeVersion, "1.16.3"), + s"Expected ONNX Runtime >= 1.16.3 to keep GH2417 fixed, but found $runtimeVersion") + + val predictions = onnxIris.copy(ParamMap.empty) + .setDeviceType("CPU") + .transform(testDfIrisFloat) + .select("prediction", "rawProbability") + .as[(Long, Map[Long, Float])] + .collect() + + assert(predictions.map(_._1).toSet == Set(0L, 1L, 2L)) + assert(predictions.forall(_._2.keySet == Set(0L, 1L, 2L))) + } + + test("ONNXRuntime.createOrtSession falls back to CPU with a clear log when CUDA is unavailable and was " + + "not explicitly requested (GH2417)") { + // Auto-detection (deviceType left unset): the default packaged dependency is the CPU-only, + // cross-platform `onnxruntime` artifact (no CUDA execution provider). A "gpu" resource was found, + // but this wasn't an explicit ask for GPU, so ONNXRuntime.createOrtSession should log a clear, + // actionable error naming onnxruntime_gpu and gracefully continue on CPU rather than fail the task. + val env = OrtEnvironment.getEnvironment + val session = ONNXRuntime.createOrtSession( + onnxIris.getModelPayload, env, gpuDeviceId = Some(0), explicitCudaRequested = false) + try { + assert(!session.getInputInfo.isEmpty) + } finally { + session.close() + } + } + + test("ONNXRuntime.createOrtSession fails fast before touching the provider when CUDA is explicitly " + + "requested but no Spark gpu resource was assigned (GH2417)") { + // gpuDeviceId=None here models a Spark task with no "gpu" resource assigned (the common case on a + // CPU-only cluster). Without this check, createOrtSession would never call addCUDA at all (there's + // no device id to add) and would silently hand back a working CPU session -- the same + // "silently broken GPU" failure mode this dependency change must not reintroduce. + val env = OrtEnvironment.getEnvironment + val caught = intercept[IllegalStateException] { + ONNXRuntime.createOrtSession( + onnxIris.getModelPayload, env, gpuDeviceId = None, explicitCudaRequested = true) + } + assert(caught.getMessage.contains("deviceType=CUDA was explicitly requested")) + assert(caught.getMessage.contains("no Spark \"gpu\" resource is assigned")) + } + + test("ONNXRuntime.createOrtSession fails fast with an actionable error when CUDA is explicitly " + + "requested but the provider is unavailable (GH2417)") { + // deviceType=CUDA is an explicit request for GPU acceleration. Silently continuing on CPU would be + // a success-shaped result that hides a severe, hard-to-notice performance regression -- the same + // "silently broken GPU" failure mode this dependency change must not reintroduce. There is no + // opt-in parameter to request a graceful CPU fallback for an explicit CUDA request, so this must + // throw a clear, actionable error rather than swallow it and return CPU results. + val env = OrtEnvironment.getEnvironment + val caught = intercept[IllegalStateException] { + ONNXRuntime.createOrtSession( + onnxIris.getModelPayload, env, gpuDeviceId = Some(0), explicitCudaRequested = true) + } + assert(caught.getMessage.contains("deviceType=CUDA was explicitly requested")) + assert(caught.getMessage.contains("onnxruntime_gpu")) + assert(caught.getCause != null) + } + + test("ONNXRuntime.createOrtSession closes SessionOptions without invalidating the returned session " + + "(GH2417)") { + // SessionOptions.close() runs (via the using(...) resource-cleanup pattern) right after + // ortEnv.createSession(...) returns. If that close happened too early -- or corrupted the session + // it had just been used to build -- the returned OrtSession would fail to actually run inference. + // Running real inference through it is the most direct observable proof the cleanup is correct. + val env = OrtEnvironment.getEnvironment + val session = ONNXRuntime.createOrtSession(onnxIris.getModelPayload, env, gpuDeviceId = None) + try { + val input = java.nio.FloatBuffer.wrap(Array(6.7f, 3.1f, 4.7f, 1.5f)) + val tensor = ai.onnxruntime.OnnxTensor.createTensor(env, input, Array(1L, 4L)) + try { + val result = session.run(java.util.Collections.singletonMap("float_input", tensor)) + try { + assert(result.size() > 0) + } finally { + result.close() + } + } finally { + tensor.close() + } + } finally { + session.close() + } + } + + test("ONNXRuntime.createOrtSession does not leak across repeated create/close cycles on any path " + + "(GH2417)") { + // Exercises all three createOrtSession outcomes (plain CPU, auto-fallback log-and-continue, and + // the explicit-CUDA fail-fast throw) repeatedly. SessionOptions is a local variable inside + // createOrtSession with no public isClosed() accessor to assert against directly (unlike OnnxValue + // in ORT 1.17+), so this loop is the practical, observable regression check available: if the + // using(...) refactor ever re-leaked SessionOptions, mishandled double-closing, or broke the + // fail-fast throw's cleanup ordering, repeated cycles are the most likely way to surface it. + val env = OrtEnvironment.getEnvironment + val payload = onnxIris.getModelPayload + for (_ <- 1 to 25) { + val cpuSession = ONNXRuntime.createOrtSession(payload, env, gpuDeviceId = None) + cpuSession.close() + + val fallbackSession = ONNXRuntime.createOrtSession( + payload, env, gpuDeviceId = Some(0), explicitCudaRequested = false) + fallbackSession.close() + + intercept[IllegalStateException] { + ONNXRuntime.createOrtSession(payload, env, gpuDeviceId = Some(0), explicitCudaRequested = true) + } + } + } + + // End-to-end: local test Spark sessions never assign a "gpu" TaskContext resource, so any explicit + // CUDA request here must hit the new "no GPU resource assigned" fail-fast path in + // ONNXRuntime.createOrtSession, exercised through the public setDeviceType API and normalization. + private def assertExplicitCudaFailsFastEndToEnd(requestedDeviceType: String): Unit = { + val caught = intercept[SparkException] { + onnxIris.copy(ParamMap.empty) + .setDeviceType(requestedDeviceType) + .transform(testDfIrisFloat) + .collect() + } + assert(caught.getMessage.contains("IllegalStateException")) + assert(caught.getMessage.contains("deviceType=CUDA was explicitly requested")) + assert(caught.getMessage.contains("no Spark \"gpu\" resource is assigned")) + } + + test("ONNXModel fails fast end-to-end when deviceType=CUDA is explicitly requested but no Spark gpu " + + "resource is assigned (GH2417)") { + assertExplicitCudaFailsFastEndToEnd("CUDA") + } + + test("ONNXModel fails fast end-to-end for a lower-case deviceType=\"cuda\" request (GH2417)") { + // The deviceType Param validator accepts any case (`x.toUpperCase()`), so a differently-cased but + // still-valid value must not silently bypass CUDA handling and fall through to CPU. + assertExplicitCudaFailsFastEndToEnd("cuda") + } + + test("ONNXModel fails fast end-to-end for a mixed-case deviceType=\"CuDa\" request (GH2417)") { + assertExplicitCudaFailsFastEndToEnd("CuDa") + } + test("ONNXModel can infer observations of matching input types") { val predicted = onnxIris.transform(testDfIrisFloat).as[(Seq[Float], Long, Map[Long, Float])].collect() diff --git a/deep-learning/src/test/scala/com/microsoft/azure/synapse/ml/onnx/ONNXRuntimeDependencySuite.scala b/deep-learning/src/test/scala/com/microsoft/azure/synapse/ml/onnx/ONNXRuntimeDependencySuite.scala new file mode 100644 index 00000000000..5acb27ac219 --- /dev/null +++ b/deep-learning/src/test/scala/com/microsoft/azure/synapse/ml/onnx/ONNXRuntimeDependencySuite.scala @@ -0,0 +1,96 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.onnx + +import com.microsoft.azure.synapse.ml.build.BuildInfo +import com.microsoft.azure.synapse.ml.core.env.FileUtilities +import org.scalatest.funsuite.AnyFunSuite + +/** + * GH2417: verifies the published dependency strategy for ONNX Runtime, independent of whatever version + * happens to be resolved on the test classpath (that runtime behavior is covered by ONNXModelSuite and + * ONNXRuntime's own tests). This reads the build definitions directly so a future edit that re-adds + * onnxruntime_gpu as a default/compile dependency -- reintroducing the ~300+MB, macOS-incompatible + * artifact for every user -- fails a fast, local test instead of only being caught by a live macOS run. + * + * A live "does this resolve from Maven Central" integration test isn't practical here: it would need + * network access and a version of SynapseML already published under the coordinate being tested, which + * doesn't exist for an unmerged change (the same reason VerifyPackageUtils.scala only checks coordinate + * *format*, not live resolution). Instead, the tests below regression-guard the documented opt-in + * coordinate/exclusion syntax in ONNX.md itself, and a separate scratch sbt project (not committed) was + * used to empirically confirm resolution behavior before writing that guidance -- see the PR description. + */ +class ONNXRuntimeDependencySuite extends AnyFunSuite { + + private val repoRoot: String = BuildInfo.baseDirectory.getParent + private val buildSbt = FileUtilities.readFile(FileUtilities.join(repoRoot, "build.sbt")) + private val onnxRuntimeDependencyScala = FileUtilities.readFile( + FileUtilities.join(repoRoot, "project", "OnnxRuntimeDependency.scala")) + private val onnxDoc = FileUtilities.readFile( + FileUtilities.join(repoRoot, "docs", "Explore Algorithms", "Deep Learning", "ONNX.md")) + + private val deepLearningBlock: String = { + val start = buildSbt.indexOf("lazy val deepLearning") + require(start >= 0, "Could not find the deepLearning project definition in build.sbt") + val nextProject = buildSbt.indexOf("\nlazy val ", start + 1) + buildSbt.substring(start, if (nextProject >= 0) nextProject else buildSbt.length) + } + + test("deep-learning declares the CPU-only, cross-platform onnxruntime artifact as its default dependency") { + assert(deepLearningBlock.contains(""""com.microsoft.onnxruntime" % "onnxruntime""""), + "build.sbt must declare com.microsoft.onnxruntime:onnxruntime (CPU-only, cross-platform, " + + "including macOS) as the default deep-learning dependency so ONNXModel works out of the box " + + "on macOS/Linux/Windows.") + } + + test("deep-learning does not force the GPU-only onnxruntime_gpu artifact on every user by default") { + assert(!deepLearningBlock.contains(""""onnxruntime_gpu""""), + "onnxruntime_gpu (Linux/Windows-only CUDA build, ~300+MB) must not be a default/compile " + + "dependency of synapseml-deep-learning; it should remain an explicit, documented opt-in " + + "(see docs/Explore Algorithms/Deep Learning/ONNX.md) so it is never forced on macOS or " + + "CPU-only users.") + } + + test("the shared ONNX Runtime version is pinned at or above the confirmed GH2417 fix version") { + val versionPattern = """val Version = "([\d.]+)"""".r + val version = versionPattern.findFirstMatchIn(onnxRuntimeDependencyScala).map(_.group(1)) + .getOrElse(fail("Could not find `val Version = \"...\"` in project/OnnxRuntimeDependency.scala")) + + def parts(v: String): Seq[Int] = v.split("\\.").map(_.toInt) + assert(Ordering.Iterable[Int].gteq(parts(version), parts("1.16.3")), + s"Expected the ONNX Runtime version pinned in project/OnnxRuntimeDependency.scala ($version) " + + s"to stay at or above 1.16.3, the smallest version confirmed to fix GH2417.") + } + + test("ONNX.md documents the Spark --packages/--exclude-packages exclusion syntax for the GPU opt-in") { + assert(onnxDoc.contains("--exclude-packages") && onnxDoc.contains("spark.jars.excludes"), + "ONNX.md must document both the spark-submit CLI form (--packages/--exclude-packages) and the " + + "equivalent Spark conf keys (spark.jars.packages/spark.jars.excludes) for excluding the " + + "transitive CPU-only onnxruntime artifact when opting in to onnxruntime_gpu.") + assert(onnxDoc.contains("com.microsoft.onnxruntime:onnxruntime_gpu"), + "ONNX.md must name the exact onnxruntime_gpu Maven coordinate for the --packages/spark.jars." + + "packages example.") + } + + test("ONNX.md documents the Databricks Maven library Exclusions field on the correct entry") { + assert(onnxDoc.contains("\"exclusions\""), + "ONNX.md must show the Databricks Maven library JSON \"exclusions\" field for the GPU opt-in.") + assert(onnxDoc.contains("Exclusions"), + "ONNX.md must name the Databricks UI \"Exclusions\" field for the GPU opt-in.") + // The exclusion must be documented on the SynapseML entry (the one that actually depends + // transitively on the CPU-only onnxruntime), not on the onnxruntime_gpu entry -- an exclusion on + // onnxruntime_gpu would be a no-op since it never depends on the plain onnxruntime artifact. + assert(onnxDoc.contains("not on the onnxruntime_gpu entry") || + onnxDoc.contains("not on `onnxruntime_gpu`"), + "ONNX.md must call out that the Databricks Exclusions field belongs on the SynapseML library " + + "entry, not the onnxruntime_gpu entry, since Databricks resolves each Maven library's " + + "dependency tree independently.") + } + + test("ONNX.md states the exactly-one-ai.onnxruntime-jar invariant for the GPU opt-in") { + assert(onnxDoc.contains("exactly one `ai.onnxruntime` jar"), + "ONNX.md must state that exactly one ai.onnxruntime jar must be present after opting in to " + + "onnxruntime_gpu, so users have a concrete way to verify their exclusion actually worked.") + } +} diff --git a/deep-learning/src/test/scala/com/microsoft/azure/synapse/ml/onnx/ONNXValueConverterSuite.scala b/deep-learning/src/test/scala/com/microsoft/azure/synapse/ml/onnx/ONNXValueConverterSuite.scala new file mode 100644 index 00000000000..f559166bf68 --- /dev/null +++ b/deep-learning/src/test/scala/com/microsoft/azure/synapse/ml/onnx/ONNXValueConverterSuite.scala @@ -0,0 +1,46 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.onnx + +import ai.onnxruntime.{OnnxValue, ValueInfo} +import org.scalatest.funsuite.AnyFunSuite + +class ONNXValueConverterSuite extends AnyFunSuite { + + test("ONNX sequence conversion recursively materializes and closes nested values") { + val leaf = new StubOnnxValue(java.lang.Long.valueOf(42L)) + val nestedValues = new java.util.ArrayList[OnnxValue]() + nestedValues.add(leaf) + val nestedSequence = new StubOnnxValue(nestedValues) + val outerValues = new java.util.ArrayList[OnnxValue]() + outerValues.add(nestedSequence) + val outerSequence = new StubOnnxValue(outerValues) + + val converted = ONNXValueConverter.mapSequenceToArray(outerSequence) + + assert(converted == Vector(Vector(42L))) + assert(nestedSequence.closed) + assert(leaf.closed) + assert(!outerSequence.closed) + } + + private object StubValueInfo extends ValueInfo + + private class StubOnnxValue(rawValue: AnyRef) extends OnnxValue { + var closed: Boolean = false + + override def getValue: AnyRef = rawValue + + override def getInfo: ValueInfo = StubValueInfo + + override def getType: OnnxValue.OnnxValueType = OnnxValue.OnnxValueType.ONNX_TYPE_SEQUENCE + + // ONNX Runtime 1.17 added OnnxValue.isClosed as an abstract method (GH2417 runtime upgrade). + override def isClosed: Boolean = closed + + override def close(): Unit = { + closed = true + } + } +} diff --git a/docs/Explore Algorithms/AI Services/Overview.ipynb b/docs/Explore Algorithms/AI Services/Overview.ipynb index 978312c222d..368d623fb01 100644 --- a/docs/Explore Algorithms/AI Services/Overview.ipynb +++ b/docs/Explore Algorithms/AI Services/Overview.ipynb @@ -125,7 +125,7 @@ "- List Custom Models: Get information about all custom models. ([Scala](https://mmlspark.blob.core.windows.net/docs/1.0.15/scala/com/microsoft/azure/synapse/ml/services/form/ListCustomModels.html), [Python](https://mmlspark.blob.core.windows.net/docs/1.0.15/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.ListCustomModels))\n", "\n", "### Search\n", - "- [**Azure Cognitive search**](https://docs.microsoft.com/azure/search/search-what-is-azure-search) ([Scala](https://mmlspark.blob.core.windows.net/docs/1.0.15/scala/com/microsoft/azure/synapse/ml/services/search/AzureSearchWriter$.html), [Python](https://mmlspark.blob.core.windows.net/docs/1.0.15/pyspark/synapse.ml.services.search.html#module-synapse.ml.services.search.AzureSearchWriter))" + "- [**Azure AI Search**](https://docs.microsoft.com/azure/search/search-what-is-azure-search) ([Scala](https://mmlspark.blob.core.windows.net/docs/1.0.15/scala/com/microsoft/azure/synapse/ml/services/search/AzureSearchWriter$.html), [Python](https://mmlspark.blob.core.windows.net/docs/1.0.15/pyspark/synapse.ml.services.search.html#module-synapse.ml.services.search.AzureSearchWriter))" ] }, { @@ -179,7 +179,7 @@ ") # Replace the call to find_secret with your key as a python string.\n", "translator_loc = \"eastus\"\n", "\n", - "# An Azure search key\n", + "# An Azure AI Search key\n", "search_key = find_secret(\n", " secret_name=\"azure-search-key\", keyvault=\"mmlspark-build-keys\"\n", ") # Replace the call to find_secret with your key as a python string." @@ -556,7 +556,7 @@ "source": [ "## Azure AI search sample\n", "\n", - "In this example, we show how you can enrich data using Cognitive Skills and write to an Azure Search Index using SynapseML." + "In this example, we show how you can enrich data using Cognitive Skills and write to an Azure AI Search Index using SynapseML." ] }, { diff --git a/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine.ipynb b/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine.ipynb index 2326836eb3c..33aa42b6eaa 100644 --- a/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine.ipynb +++ b/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine.ipynb @@ -10,7 +10,7 @@ { "cell_type": "markdown", "source": [ - "In this example, we show how you can enrich data using Cognitive Skills and write to an Azure Search Index using SynapseML. We use a subset of The MET's open-access collection and enrich it by passing it through 'Describe Image' and a custom 'Image Similarity' skill. The results are then written to a searchable index." + "In this example, we show how you can enrich data using Cognitive Skills and write to an Azure AI Search Index using SynapseML. We use a subset of The MET's open-access collection and enrich it by passing it through 'Describe Image' and a custom 'Image Similarity' skill. The results are then written to a searchable index." ], "metadata": {} }, @@ -105,7 +105,7 @@ { "cell_type": "markdown", "source": [ - "Before writing the results to a Search Index, you must define a schema which must specify the name, type, and attributes of each field in your index. Refer [Create a basic index in Azure Search](https://docs.microsoft.com/azure/search/search-what-is-an-index) for more information." + "Before writing the results to a Search Index, you must define a schema which must specify the name, type, and attributes of each field in your index. Refer [Create a basic index in Azure AI Search](https://docs.microsoft.com/azure/search/search-what-is-an-index) for more information." ], "metadata": {} }, @@ -131,7 +131,7 @@ { "cell_type": "markdown", "source": [ - "The Search Index can be queried using the [Azure Search REST API](https://docs.microsoft.com/rest/api/searchservice/) by sending GET or POST requests and specifying query parameters that give the criteria for selecting matching documents. For more information on querying refer [Query your Azure Search index using the REST API](https://docs.microsoft.com/rest/api/searchservice/Search-Documents)" + "The Search Index can be queried using the [Azure AI Search REST API](https://docs.microsoft.com/rest/api/searchservice/) by sending GET or POST requests and specifying query parameters that give the criteria for selecting matching documents. For more information on querying refer [Query your Azure AI Search index using the REST API](https://docs.microsoft.com/rest/api/searchservice/Search-Documents)" ], "metadata": {} }, @@ -139,7 +139,7 @@ "cell_type": "code", "execution_count": 12, "source": [ - "url = \"https://{}.search.windows.net/indexes/{}/docs/search?api-version=2019-05-06\".format(\n", + "url = \"https://{}.search.windows.net/indexes/{}/docs/search?api-version=2026-04-01\".format(\n", " search_service, search_index\n", ")\n", "requests.post(\n", diff --git a/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs.ipynb b/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs.ipynb index 2bf0ebd434b..20f9b6f6d8e 100644 --- a/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs.ipynb +++ b/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs.ipynb @@ -51,7 +51,7 @@ "We\u2019ll cover the following key steps:\n", "\n", "1. Preprocessing PDF Documents: Learn how to load the PDF documents into a Spark DataFrame, read the documents using the [Azure AI Document Intelligence](https://azure.microsoft.com/products/ai-services/ai-document-intelligence) in Azure AI Services, and use SynapseML to split the documents into chunks.\n", - "2. Embedding Generation and Storage: Learn how to generate embeddings for the chunks using SynapseML and [Azure OpenAI Services](https://azure.microsoft.com/products/ai-services/openai-service), store the embeddings in a vector store using [Azure Cognitive Search](https://azure.microsoft.com/products/search), and search the vector store to answer the user\u2019s question.\n", + "2. Embedding Generation and Storage: Learn how to generate embeddings for the chunks using SynapseML and [Azure OpenAI Services](https://azure.microsoft.com/products/ai-services/openai-service), store the embeddings in a vector store using [Azure AI Search](https://azure.microsoft.com/products/search), and search the vector store to answer the user\u2019s question.\n", "3. Question Answering Pipeline: Learn how to retrieve relevant document based on the user\u2019s question and provide the answer using [Langchain](https://python.langchain.com/en/latest/index.html#)." ] }, @@ -155,7 +155,7 @@ "aoai_deployment_name_query = \"gpt-5.1\"\n", "aoai_model_name_query = \"gpt-5.1\"\n", "\n", - "# Azure Cognitive Search\n", + "# Azure AI Search\n", "cogsearch_name = \"mmlspark-azure-search\"\n", "cogsearch_index_name = \"examplevectorindex\"\n", "cogsearch_api_key = find_secret(\n", @@ -608,7 +608,7 @@ } }, "source": [ - "### Step 6: Store the embeddings in Azure Cognitive Search Vector Store." + "### Step 6: Store the embeddings in Azure AI Search Vector Store." ] }, { @@ -627,7 +627,7 @@ } }, "source": [ - "[Azure Cognitive Search](https://learn.microsoft.com/azure/search/search-what-is-azure-search) offers a user-friendly interface for creating a vector database, as well as storing and retrieving data using vector search. If you're interested in learning more about vector search, you can look [here](https://github.com/Azure/cognitive-search-vector-pr/tree/main).\n", + "[Azure AI Search](https://learn.microsoft.com/azure/search/search-what-is-azure-search) offers a user-friendly interface for creating a vector database, as well as storing and retrieving data using vector search. If you're interested in learning more about vector search, you can look [here](https://github.com/Azure/cognitive-search-vector-pr/tree/main).\n", "\n", "\n", "Storing data in the AzureCogSearch vector database involves two main steps:\n", @@ -674,6 +674,8 @@ " serviceName=cogsearch_name,\n", " indexName=cogsearch_index_name,\n", " keyCol=\"idx\",\n", + " # This shared index uses the legacy vector schema until it is explicitly migrated.\n", + " apiVersion=\"2023-07-01-Preview\",\n", " vectorCols=json.dumps([{\"name\": \"embeddings\", \"dimension\": 1536}]),\n", ")" ] @@ -771,10 +773,19 @@ "\n", "def retrieve_k_chunk(k, question_embedding):\n", " # Retrieve the top K entries\n", - " url = f\"https://{cogsearch_name}.search.windows.net/indexes/{cogsearch_index_name}/docs/search?api-version=2023-07-01-Preview\"\n", + " url = f\"https://{cogsearch_name}.search.windows.net/indexes/{cogsearch_index_name}/docs/search?api-version=2026-04-01\"\n", "\n", " payload = json.dumps(\n", - " {\"vector\": {\"value\": question_embedding, \"fields\": \"embeddings\", \"k\": k}}\n", + " {\n", + " \"vectorQueries\": [\n", + " {\n", + " \"kind\": \"vector\",\n", + " \"vector\": question_embedding,\n", + " \"fields\": \"embeddings\",\n", + " \"k\": k,\n", + " }\n", + " ]\n", + " }\n", " )\n", " headers = {\n", " \"Content-Type\": \"application/json\",\n", diff --git a/docs/Explore Algorithms/Deep Learning/ONNX.md b/docs/Explore Algorithms/Deep Learning/ONNX.md index 5d45e386794..d89aae11be0 100644 --- a/docs/Explore Algorithms/Deep Learning/ONNX.md +++ b/docs/Explore Algorithms/Deep Learning/ONNX.md @@ -13,6 +13,136 @@ description: Learn how to use the ONNX model transformer to run inference for an SynapseML now includes a Spark transformer to bring a trained ONNX model to Apache Spark, so you can run inference on your data with Spark's large-scale data processing power. +## Runtime selection: CPU by default, CUDA as an explicit opt-in + +SynapseML depends on `com.microsoft.onnxruntime:onnxruntime` (the CPU-only ONNX Runtime artifact) by +default. It bundles native libraries for Windows x64, Linux x64/aarch64, and **macOS x64/aarch64**, so +`ONNXModel` works out of the box for CPU inference on every platform SynapseML supports, including local +Spark on a Mac. + +CUDA/GPU acceleration is **not** bundled by default: + +- NVIDIA's CUDA execution provider only exists for Linux and Windows -- it has no macOS build, so + shipping it by default would still leave macOS broken. +- The artifact that adds it, `com.microsoft.onnxruntime:onnxruntime_gpu`, is roughly 300+ MB (mostly the + embedded CUDA/TensorRT provider binaries and Windows debug symbols), and forcing that onto every + CPU-only and macOS user just to support Linux/Windows GPU clusters isn't reasonable. + +To opt in to GPU acceleration on a Linux or Windows cluster with a matching NVIDIA GPU and CUDA/cuDNN +installed, add `com.microsoft.onnxruntime:onnxruntime_gpu` at the same version SynapseML pins in +`project/OnnxRuntimeDependency.scala`, and exclude the transitive CPU-only `onnxruntime` artifact that +comes from `synapseml-deep-learning` (or the aggregate `synapseml` package) so you don't end up with two +copies of the `ai.onnxruntime` classes on your classpath. **The exclusion must be attached to the +SynapseML dependency itself** (the one that transitively depends on `onnxruntime`), not to the +`onnxruntime_gpu` dependency (which never depends on the CPU-only artifact, so excluding anything from it +would be a no-op). After installing, verify that exactly one `ai.onnxruntime` jar is present -- if you +ever see both `onnxruntime-.jar` and `onnxruntime_gpu-.jar` on the same classpath, the +exclusion is missing or attached to the wrong dependency. + +- **sbt** (per-dependency exclusion, attached to the SynapseML dependency): + + ```scala + libraryDependencies ++= Seq( + ("com.microsoft.azure" %% "synapseml-deep-learning" % "") + .exclude("com.microsoft.onnxruntime", "onnxruntime"), + "com.microsoft.onnxruntime" % "onnxruntime_gpu" % "" + ) + ``` + + (A project-wide `excludeDependencies += ExclusionRule("com.microsoft.onnxruntime", "onnxruntime")` + works too and is simpler if nothing else in your build needs the CPU-only artifact.) + +- **Maven**: add an `` for `com.microsoft.onnxruntime:onnxruntime` to your + `synapseml-deep-learning` (or `synapseml`) ``, and add `onnxruntime_gpu` as a separate, + unexcluded dependency: + + ```xml + + com.microsoft.azure + synapseml-deep-learning_2.12 + ... + + + com.microsoft.onnxruntime + onnxruntime + + + + + com.microsoft.onnxruntime + onnxruntime_gpu + 1.17.3 + + ``` + +- **spark-submit / spark-shell / pyspark (`--packages`)**: pass the SynapseML and `onnxruntime_gpu` + coordinates via `--packages`, and exclude the CPU-only artifact via the separate `--exclude-packages` + flag (format `groupId:artifactId`, no version): + + ```bash + spark-submit \ + --packages com.microsoft.azure:synapseml_2.12:,com.microsoft.onnxruntime:onnxruntime_gpu:1.17.3 \ + --exclude-packages com.microsoft.onnxruntime:onnxruntime \ + your_script.py + ``` + + The equivalent Spark configuration keys (for a `SparkConf`/notebook `%%configure` cell instead of CLI + flags) are `spark.jars.packages` and `spark.jars.excludes`: + + ``` + spark.jars.packages=com.microsoft.azure:synapseml_2.12:,com.microsoft.onnxruntime:onnxruntime_gpu:1.17.3 + spark.jars.excludes=com.microsoft.onnxruntime:onnxruntime + ``` + +- **Databricks cluster library UI/API**: install the SynapseML package and `onnxruntime_gpu` as **two + separate Maven libraries**, and set the library's **Exclusions** field (format `groupId:artifactId`) on + the **SynapseML library entry**, not on `onnxruntime_gpu`: + + ```json + [ + { + "maven": { + "coordinates": "com.microsoft.azure:synapseml-deep-learning_2.12:", + "exclusions": ["com.microsoft.onnxruntime:onnxruntime"] + } + }, + { "maven": { "coordinates": "com.microsoft.onnxruntime:onnxruntime_gpu:1.17.3" } } + ] + ``` + + Each Databricks Maven library entry resolves its own dependency tree independently, so an exclusion + set on the `onnxruntime_gpu` entry has no effect on what the SynapseML entry pulls in -- it must be on + the entry that actually depends on the CPU-only artifact. + +### What happens if CUDA is requested but unavailable + +`deviceType` is matched case-insensitively (`"CUDA"`, `"cuda"`, and `"CuDa"` are all treated the same), so +the behavior below cannot be bypassed by casing. It differs depending on whether you explicitly asked for +GPU acceleration: + +- **`deviceType` explicitly set to `CUDA`:** `ONNXModel` **fails the task** with a clear, actionable error + rather than silently falling back to CPU, in either of these cases: + - No Spark `gpu` resource is assigned to the executor/task (the common case on a CPU-only cluster) -- + there is no GPU device id to use at all, so the error explains how to configure Spark's GPU resource + allocation (or to set `deviceType` to `CPU` instead). + - A `gpu` resource *is* assigned, but the CUDA execution provider still isn't usable (for example, only + the default CPU-only artifact is installed, or CUDA/cuDNN aren't installed on the node) -- the error + names the `onnxruntime_gpu` artifact to install. + + In both cases, silently continuing on CPU would produce a success-shaped result that quietly hides a + severe performance regression, which is the same class of problem this dependency change is meant to + fix, not reintroduce. There is currently no parameter to opt in to a graceful CPU fallback for an + explicit CUDA request; set `deviceType` to `CPU` if you intend to run on CPU. +- **`deviceType` left unset (auto-detection):** if a `gpu` resource happens to be present but CUDA isn't + usable, `ONNXModel` logs a clear, actionable error (again naming `onnxruntime_gpu`) and continues on + CPU, since GPU was never explicitly requested in this case. If no `gpu` resource is present, CPU is + used with no error, since auto-detection found nothing to use. + +> **Note:** Real GPU acceleration (the `onnxruntime_gpu` artifact actually engaging the CUDA execution +> provider end-to-end on hardware such as an NVIDIA T4) is not covered by an automated CI test in this +> repository; only the CPU default, the explicit-CUDA fail-fast errors, and the auto-detect fallback are. +> Validate GPU throughput on real GPU hardware before relying on it in production. + ## ONNXHub Although you can use your own local model, many popular existing models are provided through the ONNXHub. You can use a model's ONNXHub name (for example "MNIST") and download the bytes of the model, and some metadata about the model. You can also list @@ -70,7 +200,7 @@ available models, optionally filtering by name or tags. | miniBatcher | Specify the MiniBatcher to use. | `FixedMiniBatchTransformer` with batch size 10 | | softMaxDict | A map between output DataFrame columns, where the value column will be computed from taking the softmax of the key column. If the 'rawPrediction' column contains logits outputs, then one can set softMaxDict to `Map("rawPrediction" -> "probability")` to obtain the probability outputs. | None | | argMaxDict | A map between output DataFrame columns, where the value column will be computed from taking the argmax of the key column. This parameter can be used to convert probability or logits output to the predicted label. | None | - | deviceType | Specify a device type the model inference runs on. Supported types are: CPU or CUDA. If not specified, auto detection will be used. | None | + | deviceType | Specify a device type the model inference runs on. Supported types are: CPU or CUDA. If not specified, auto detection will be used. CUDA requires the opt-in `onnxruntime_gpu` artifact; if explicitly set to CUDA and the CUDA provider is unavailable, transform **fails fast** rather than silently running on CPU -- see [Runtime selection](#runtime-selection-cpu-by-default-cuda-as-an-explicit-opt-in). | None | | optimizationLevel | Specify the [optimization level](https://onnxruntime.ai/docs/performance/model-optimizations/graph-optimizations.html#graph-optimization-levels) for the ONNX graph optimizations. Supported values are: `NO_OPT`, `BASIC_OPT`, `EXTENDED_OPT`, `ALL_OPT`. | `ALL_OPT` | 4. Call `transform` method to run inference on the input DataFrame. diff --git a/docs/Explore Algorithms/LightGBM/Overview.md b/docs/Explore Algorithms/LightGBM/Overview.md index 9106da7171c..9ef88079bed 100644 --- a/docs/Explore Algorithms/LightGBM/Overview.md +++ b/docs/Explore Algorithms/LightGBM/Overview.md @@ -102,6 +102,58 @@ You can mix *passThroughArgs* and explicit args, as shown in the example. Synaps merges them to create one argument string to send to LightGBM. If you set a parameter in both places, *passThroughArgs* takes precedence. +#### GPU training with a custom OpenCL native library + +SynapseML's published `lightgbmlib` artifact contains CPU-only native libraries. Only +`deviceType="gpu"` selects an accelerator: it selects LightGBM's OpenCL learner and +requires a compatible custom native library. All `cuda` requests are rejected before +native training because LightGBM 3.3.510 CUDA is incompatible with SynapseML streaming +Datasets. + +Accelerator training is intended for users who provide their own compatible LightGBM +native build. Put both `lib_lightgbm` and `lib_lightgbm_swig` on `java.library.path` for +the Spark driver and every executor before LightGBM is initialized. `NativeLoader` checks +that path first and falls back to the CPU-only libraries packaged in the SynapseML JAR. +For example, a Spark deployment can set both `spark.driver.extraLibraryPath` and +`spark.executor.extraLibraryPath` to the directory containing the custom libraries. +The custom SWIG library must be ABI-compatible with the Java classes shipped by the +SynapseML version in use; supplying only one library can accidentally mix incompatible +custom and bundled binaries. + +SynapseML does not support `deviceType="cuda"` with `lightgbmlib` 3.3.510. Its CUDA +objective expects CUDA metadata that is not created by the serialized streaming Dataset +path and can segfault the Spark executor during booster creation. SynapseML rejects CUDA +before native training. Use `deviceType="gpu"` with an OpenCL-enabled native build; this +path supports classifier, regressor, and ranker training on NVIDIA GPUs such as T4. + +`deviceType` exposes only the accelerator backends implemented by LightGBM: + +| Value | Backend | Hardware | +| --- | --- | --- | +| `cpu` | Native CPU learner | Supported by the bundled SynapseML native library | +| `gpu` | OpenCL learner (`USE_GPU=1`) | AMD, Intel, or NVIDIA devices with a working OpenCL runtime | +| `cuda` | Unsupported with SynapseML's LightGBM 3.3.510 streaming Dataset path | Do not use | + +Apple Metal Performance Shaders (`mps`) and Habana HPU are not LightGBM tree-learning +backends and are therefore not accepted values. Apple Silicon can only be evaluated +through LightGBM's OpenCL learner and a custom macOS ARM64 native build; this is not MPS +support, Apple has deprecated OpenCL, and LightGBM documents a macOS Boost.Compute cache +workaround. Do not claim Apple Silicon GPU support without testing that exact native build, +Spark/JVM architecture, dataset correctness, and performance on supported macOS hardware. + +After installing the custom native libraries, select the learner explicitly: + +```python +model = LightGBMClassifier(deviceType="gpu").fit(train) +``` + +The default is `cpu` and does not add a `device_type` native parameter. If +*passThroughArgs* contains `device_type`, that canonical value takes precedence over both +the `device` alias and `deviceType`, regardless of argument order. If only `device` is +present, it takes precedence over `deviceType`. A native-effective value of `cuda` is +always rejected. If a requested OpenCL accelerator is unavailable, SynapseML reports that +the bundled native is CPU-only and identifies the custom-library configuration required. + ### Architecture LightGBM on Spark uses the Simple Wrapper and Interface Generator (SWIG) @@ -260,3 +312,86 @@ To use it in scala, you can call setUseBarrierExecutionMode(true), for example: ... Note: barrier execution mode can also cause complicated issues, so use it only if needed. + +Barrier execution mode is also the only mode that can recover from a task failure that happens after the +network topology has been negotiated. A LightGBM network is negotiated once and then fixed, so an individual +Spark task retry can never rejoin it. Spark restarts a barrier stage in its entirety, and the driver serves a +fresh topology round for each stage attempt, so training can survive a failure that would otherwise abort the +job. Failures that happen before a task joins the network are still retried normally in either mode. + +### Diagnosing "Connection refused" during training + +Distributed training first exchanges `host:port` information with the driver, which serves that exchange +once per stage attempt. If a task fails after that exchange, the regular (non-barrier) retry of that task +reconnects to a driver endpoint that is no longer accepting connections. Because Spark only reports the +most recent attempt, this can hide the failure that actually caused the retry. + +When this happens, the reported error explains that it's a retry that could not rejoin the network, and +names the partition to investigate. Look for the **first** failed attempt of that partition in the executor +logs — that attempt holds the real cause. + +### IPv6 clusters + +Distributed training works on clusters whose executors only have IPv6 addresses. + +The native LightGBM library is IPv4-only in every released version: it splits each machine list entry on +`:` and keeps the entry only when that yields exactly two parts, and it builds every socket with `AF_INET` +and `inet_pton(AF_INET, ...)`. An IPv6 endpoint is therefore dropped or misread by the native parser, and +could not be dialed or accepted even if it survived parsing. + +SynapseML handles this itself. The topology exchange publishes IPv6 endpoints in the unambiguous +`[address]:port` form, and each task then bridges the transport for the native library: + +- the port a task advertised to its peers is owned by SynapseML, which accepts peer connections over + either address family and forwards them to the native listener over IPv4 loopback; +- each IPv6 peer gets an IPv4 loopback relay that forwards what the native library sends to that peer's + real IPv6 address; +- the machine list handed to the native library has the same entries in the same order, with every bridged + endpoint rewritten to `127.0.0.1:port` and an explicit rank, so LightGBM ranks are unchanged. + +An IPv4 machine list is passed to the native library exactly as before, with no relay and no rewriting, so +IPv4 clusters see no behavior or performance change. On an IPv6 cluster, peer traffic takes one extra +loopback hop on each side, and IPv4 loopback must be available on the executors. Measured on a 16 core +developer machine over loopback, a bridged link sustains roughly a third of the throughput of a direct one +(about 0.6 GB/s per direction) for about four times the CPU per transferred byte, and adds roughly 200 +microseconds to a small message round trip per hop. Links faster than a few Gb/s can therefore be limited +by the bridge rather than by the network. The relays never buffer in the JVM heap: a reader that stops +reading stops the sender, with only the socket buffers in between. + +The advertised port is the one address peers know, so SynapseML binds it on every interface, exactly as the +native listener did before this change, and performs the LightGBM link handshake there itself. A machine +opens a LightGBM link by sending its rank, so a connection has to produce a valid, unused, lower rank within +a timeout before any of its bytes reach the native library; connections that stall, repeat a rank, or claim +a rank the topology does not have are closed by the bridge. That matters because the native accept loop has +no timeout and treats the first four bytes of any connection as a rank. + +The native listener itself is the one thing this library cannot rebind: `TcpSocket::Bind` hardcodes +`0.0.0.0`, so while it is open it is reachable on every IPv4 interface. The bridge therefore claims each of +that listener's link slots itself, over IPv4 loopback, as soon as the port is bound — one slot per lower +rank, which is exactly how many the native library accepts before closing the listener. In practice the +listener is open for milliseconds on an unadvertised ephemeral port rather than for the whole handshake +phase, and every byte the native library reads from it comes from the bridge. Closing that window entirely +requires an upstream change to LightGBM: `TcpSocket::Bind` would have to take a bind address so that +`Linkers::TryBind` can pass a loopback one. Until then, a LightGBM training port must only be reachable +from the cluster's own executors, which was already true before this change. + +The per peer relays are bound to IPv4 loopback and are not reachable from outside the machine, the number of +links a bridge will relay is capped at twice the machine count, and each lower rank may link only once. One +event loop thread serves every listener and every link, so a bridge runs exactly one thread whatever the +machine count is, with two fixed size buffers per link. + +A connection the topology cannot need — an unsolicited one, one past the cap, or one that fails its +handshake — is refused without consuming any of that budget, so it cannot starve the links the native +library itself opens. Transport failures are retried while they can be: a dial that fails, immediately or +later, backs off and retries until its deadline, and a failure while handling one connection never closes +the listener that accepted it. A failure that cannot be retried away is recorded and fails the Spark task +with that cause, both before and after the native initialization call, rather than leaving the task waiting +on a transport that will not recover. + +Link-local addresses (`fe80::/10`) are supported only when a peer advertises a zone identifier +(`fe80::1%eth0`) that names an interface on every other machine, since a link-local address is scoped to a +single interface. A task normalizes a numeric scope (an interface index, which only means something on the +machine that produced it) to the interface name before publishing its endpoint, and a peer that still +advertises a numeric scope is rejected. A link-local peer without a zone, or with a zone this machine does +not have, fails immediately with an error naming the address instead of hanging. Use a globally routable or +unique-local IPv6 address for distributed training. diff --git a/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms.ipynb b/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms.ipynb index 7d292948c46..5f0809c7dc4 100644 --- a/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms.ipynb +++ b/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms.ipynb @@ -23,7 +23,7 @@ "> + Load various forms (invoices) into a data frame in an Apache Spark session\n", "> + Analyze them to determine their features\n", "> + Assemble the resulting output into a tabular data structure\n", - "> + Write the output to a search index hosted in Azure Cognitive Search\n", + "> + Write the output to a search index hosted in Azure AI Search\n", "> + Explore and query over the content you created" ] }, @@ -595,7 +595,7 @@ } }, "source": [ - "## 8 - Create an Azure Search Index for the Forms" + "## 8 - Create an Azure AI Search Index for the Forms" ] }, { @@ -669,7 +669,7 @@ "source": [ "import requests\n", "\n", - "search_url = \"https://{}.search.windows.net/indexes/{}/docs/search?api-version=2019-05-06\".format(\n", + "search_url = \"https://{}.search.windows.net/indexes/{}/docs/search?api-version=2026-04-01\".format(\n", " search_service, search_index\n", ")\n", "requests.post(\n", @@ -693,7 +693,7 @@ } }, "source": [ - "## 10 - Build a chatbot that can use Azure Search as a tool 🧠🔧" + "## 10 - Build a chatbot that can use Azure AI Search as a tool 🧠🔧" ] }, { diff --git a/docs/Explore Algorithms/Responsible AI/Data Balance Analysis.md b/docs/Explore Algorithms/Responsible AI/Data Balance Analysis.md index c8437cc65b7..d80363c9b70 100644 --- a/docs/Explore Algorithms/Responsible AI/Data Balance Analysis.md +++ b/docs/Explore Algorithms/Responsible AI/Data Balance Analysis.md @@ -163,7 +163,7 @@ We can use distance measures to find out how far our observed and reference dist | Measure | Description | Interpretation | Reference | |--------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------| | KL Divergence | Measure of how one probability distribution is different from a second, reference probability distribution. Measure of the information gained when one revises one's beliefs from the prior probability distribution Q to the posterior probability distribution P. In other words, it is the amount of information lost when Q is used to approximate P. | Non-negative. 0 means P = Q. | [Link](https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence) | -| JS Distance | Measuring the similarity between two probability distributions. Symmetrized and smoothed version of the Kullback–Leibler (KL) divergence. Square root of JS Divergence. | Range [0, 1]. 0 means perfectly same to balanced distribution. | [Link](https://en.wikipedia.org/wiki/Jensen%E2%80%93Shannon_divergence) | +| JS Distance | Square root of the average base-2 KL divergence from each distribution to their midpoint mixture. The mixture handles zero-probability support without additive smoothing. | For finite, non-negative distributions with unit sum, the range is [0, 1]. 0 means identical distributions; 1 means disjoint support. | [Link](https://en.wikipedia.org/wiki/Jensen%E2%80%93Shannon_divergence) | | Wasserstein Distance | This distance is also known as the earth mover’s distance, since it can be seen as the minimum amount of “work” required to transform u into v, where “work” is measured as the amount of distribution weight that must be moved multiplied by the distance it has to be moved. | Non-negative. 0 means P = Q. | [Link](https://en.wikipedia.org/wiki/Wasserstein_metric) | | Infinity Norm Distance | Distance between two vectors is the greatest of their differences along any coordinate dimension. Also called Chebyshev distance or chessboard distance. | Non-negative. 0 means same distribution. | [Link](https://en.wikipedia.org/wiki/Chebyshev_distance) | | Total Variation Distance | It is equal to half the L1 (Manhattan) distance between the two distributions. Take the difference between the two proportions in each category, add up the absolute values of all the differences, and then divide the sum by 2. | Non-negative. 0 means same distribution. | [Link](https://en.wikipedia.org/wiki/Total_variation_distance_of_probability_measures) | diff --git a/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis.ipynb b/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis.ipynb index 517cde68caf..b4015270f1a 100644 --- a/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis.ipynb +++ b/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis.ipynb @@ -491,7 +491,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "![Distribution Balance Measures of Sex and Race in Adult Dataset](https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_AdultCensusIncome_DistributionMeasures.png)" + "Run the cell above to render the distribution balance chart with the normalized JS distance values." ] }, { @@ -507,12 +507,12 @@ "source": [ "#### Interpret Distribution Balance Measures\n", "\n", - "Race has a JS Distance of 0.5104 while Sex has a JS Distance of 0.1217.\n", + "Race has a JS Distance of 0.6131 while Sex has a JS Distance of 0.1462.\n", "\n", - "Knowing that JS Distance is between [0, 1] where 0 means perfectly balanced distribution, we can tell that:\n", - "* There is a larger disparity between various races than various sexes in our dataset.\n", - "* Race is nowhere close to a perfectly balanced distribution (i.e. some races are seen ALOT more than others in our dataset).\n", - "* Sex is fairly close to a perfectly balanced distribution." + "JS Distance uses base-2 normalization and is in [0, 1], where 0 means the observed and reference distributions match and 1 means they have disjoint support. We can tell that:\n", + "* Race differs substantially more from its uniform reference distribution than sex.\n", + "* Race is far from its uniform reference distribution (some races appear much more often than others).\n", + "* Sex is comparatively close to its uniform reference distribution." ] }, { diff --git a/docs/Quick Examples/estimators/core/_Stages.md b/docs/Quick Examples/estimators/core/_Stages.md index 853a2a35d14..86e43893ee3 100644 --- a/docs/Quick Examples/estimators/core/_Stages.md +++ b/docs/Quick Examples/estimators/core/_Stages.md @@ -73,6 +73,80 @@ csharp="classSynapse_1_1ML_1_1Stages_1_1ClassBalancer.html" sourceLink="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/ClassBalancer.scala" /> +### LumpFeatures + + + + + + +```python +from synapse.ml.stages import * + +df = (spark.createDataFrame([ + ("apple", "red"), + ("apple", "red"), + ("apple", "blue"), + ("banana", "red"), + ("banana", "green"), + ("cherry", "green") + ], ["fruit", "color"])) + +lumper = (LumpFeatures() + .setLumpRules({"fruit": 3, "color": 3}) + .setMinCount(2) + .setOutputCols({"fruit": "fruit_lumped", "color": "color_lumped"})) + +model = lumper.fit(df) + +# cherry and blue occur once, so minCount drops them before the top-K cap is reached +print(model.getKeptValues()) + +model.transform(df).show() +``` + + + + +```scala +import com.microsoft.azure.synapse.ml.stages._ + +val df = Seq( + ("apple", "red"), + ("apple", "red"), + ("apple", "blue"), + ("banana", "red"), + ("banana", "green"), + ("cherry", "green")).toDF("fruit", "color") + +val lumper = (new LumpFeatures() + .setLumpRules(Map("fruit" -> 3, "color" -> 3)) + .setMinCount(2) + .setOutputCols(Map("fruit" -> "fruit_lumped", "color" -> "color_lumped"))) + +val model = lumper.fit(df) + +// cherry and blue occur once, so minCount drops them before the top-K cap is reached +println(model.getKeptValues) + +model.transform(df).show() +``` + + + + + + + ### MultiColumnAdapter =4.25 + # Intentionally unpinned: master pins numpy==1.26.4, which has no Python 3.13 wheels. + # Do not adopt master's pin when syncing. - numpy # CPU-only wheels avoid unused CUDA downloads in local/CI; DBR GPU notebooks use runtime-provided CUDA. - "https://download.pytorch.org/whl/cpu/torch-2.9.1%2Bcpu-cp313-cp313-manylinux_2_28_x86_64.whl#sha256=39b3dff6d8fba240ae0d1bede4ca11c2531ae3b47329206512d99e17907ff74b" diff --git a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/BasePartitionTask.scala b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/BasePartitionTask.scala index 64cfd5102fc..f68e2572e2c 100644 --- a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/BasePartitionTask.scala +++ b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/BasePartitionTask.scala @@ -131,7 +131,7 @@ abstract class BasePartitionTask extends Serializable with Logging { // Start with initialization val taskCtx = initialize(ctx, inputRows) - NetworkManager.withCleanupPreservingPrimary(taskCtx.networkTopologyInfo.releasePortReservation()) { + NetworkManager.withCleanupPreservingPrimary(taskCtx.networkTopologyInfo.releaseNetworkResources()) { if (taskCtx.isEmptyPartition) { log.warn("LightGBM task encountered empty partition, for best performance ensure no partitions are empty") Array { PartitionResult(None, taskCtx.measures) }.toIterator @@ -199,7 +199,7 @@ abstract class BasePartitionTask extends Serializable with Logging { shouldExecuteTraining, taskMeasures) - NetworkManager.withCleanupOnFailurePreservingPrimary(networkInfo.releasePortReservation()) { + NetworkManager.withCleanupOnFailurePreservingPrimary(networkInfo.releaseNetworkResources()) { // Return booster only from main worker to reduce network communication overhead val shouldReturnBooster = if (isEmptyPartition) false else if (!shouldExecuteTraining) false diff --git a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMBase.scala b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMBase.scala index 441787c8cf9..0148c56d838 100644 --- a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMBase.scala +++ b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMBase.scala @@ -199,7 +199,7 @@ trait LightGBMBase[TrainedModel <: Model[TrainedModel] with LightGBMModelParams] private def getSlotNamesWithMetadata(featuresSchema: StructField): Option[Array[String]] = { if (getSlotNames.nonEmpty) { - Some(getSlotNames) + Some(ensureUniqueFeatureNames(getSlotNames)) } else { AttributeGroup.fromStructField(featuresSchema).attributes.flatMap(attributes => if (attributes.isEmpty) { @@ -208,28 +208,82 @@ trait LightGBMBase[TrainedModel <: Model[TrainedModel] with LightGBMModelParams] val colNames = attributes.indices.map(_.toString).toArray attributes.foreach(attr => attr.index.foreach(index => colNames(index) = attr.name.getOrElse(index.toString))) - Some(colNames) + // Ensure unique feature names to avoid LightGBM error: + // "Feature (Column_) appears more than one time" + // This can occur in Spark 3.5+ due to changes in AttributeGroup metadata handling + Some(ensureUniqueFeatureNames(colNames)) } ) } } - private def validateSlotNames(featuresSchema: StructField): Unit = { - val metadata = AttributeGroup.fromStructField(featuresSchema) - if (metadata.attributes.isDefined) { - val slotNamesOpt = getSlotNamesWithMetadata(featuresSchema) - val pattern = new Regex("[\",:\\[\\]{}]") - slotNamesOpt.foreach(slotNames => { - val badSlotNames = slotNames.flatMap(slotName => - if (pattern.findFirstIn(slotName).isEmpty) None else Option(slotName)) - if (!badSlotNames.isEmpty) { - throw new IllegalArgumentException( - s"Invalid slot names detected in features column: ${badSlotNames.mkString(",")}" + - " \n Special characters \" , : \\ [ ] { } will cause unexpected behavior in LGBM unless changed." + - " This error can be fixed by renaming the problematic columns prior to vector assembly.") - } - }) + /** + * Maps a feature name to the form LightGBM compares internally. LightGBM replaces every space + * with an underscore before checking for duplicates, so "a b" and "a_b" are the same feature + * as far as the native library is concerned even though they differ in Scala. + */ + private def normalizeFeatureName(name: String): String = name.replace(' ', '_') + + /** + * Ensures all feature names are unique by appending a numeric suffix to repeated names. + * LightGBM rejects a Dataset whose feature names repeat, failing the native + * LGBM_DatasetSetFeatureNames call with "Feature (X) appears more than one time", and Spark + * can surface repeated names through AttributeGroup metadata on the features column. + * + * Uniqueness is decided on the normalized form (see normalizeFeatureName), because that is + * what LightGBM compares. The original names are what get emitted, so this only affects which + * names are considered to collide. + * + * Every original name is reserved up front, so a generated name can never collide with an + * original that appears later in the array. Generated names are reserved as they are handed + * out, so they cannot collide with each other either. Order is preserved, which matters + * because feature names are positional in LightGBM. + * + * @param names The array of feature names that may contain duplicates. + * @return An array of unique feature names, in the original order. + */ + private def ensureUniqueFeatureNames(names: Array[String]): Array[String] = { + val reserved = scala.collection.mutable.HashSet[String](names.map(normalizeFeatureName): _*) + val emitted = scala.collection.mutable.HashSet[String]() + val renamed = scala.collection.mutable.ArrayBuffer[String]() + + val uniqueNames = names.map { name => + if (emitted.add(normalizeFeatureName(name))) { + name + } else { + // Terminates because only finitely many names are reserved. + val uniqueName = Iterator.from(1) + .map(suffix => s"${name}_$suffix") + .find(candidate => !reserved.contains(normalizeFeatureName(candidate))) + .get + reserved.add(normalizeFeatureName(uniqueName)) + emitted.add(normalizeFeatureName(uniqueName)) + renamed += name + uniqueName + } + } + + if (renamed.nonEmpty) { + log.warn(s"Duplicate feature names detected and renamed: ${renamed.distinct.mkString(", ")}. " + + "Set the 'slotNames' parameter explicitly to control feature naming.") } + + uniqueNames + } + + private def validateSlotNames(featuresSchema: StructField): Unit = { + val slotNamesOpt = getSlotNamesWithMetadata(featuresSchema) + val pattern = new Regex("[\",:\\[\\]{}]") + slotNamesOpt.foreach(slotNames => { + val badSlotNames = slotNames.flatMap(slotName => + if (pattern.findFirstIn(slotName).isEmpty) None else Option(slotName)) + if (!badSlotNames.isEmpty) { + throw new IllegalArgumentException( + s"Invalid slot names detected in features column: ${badSlotNames.mkString(",")}" + + " \n Special characters \" , : [ ] { } will cause unexpected behavior in LGBM unless changed." + + " This error can be fixed by renaming the problematic columns prior to vector assembly.") + } + }) } /** @@ -320,6 +374,28 @@ trait LightGBMBase[TrainedModel <: Model[TrainedModel] with LightGBMModelParams] getMaxStreamingOMPThreads) } + /** Adds an explicitly requested accelerator to the native parameter string. + * + * The default CPU path returns the caller's pass-through arguments unchanged. This keeps the + * existing CPU parameter string and ExecutionParams API intact. LightGBM also accepts "device" + * as an alias for "device_type", so either spelling in passThroughArgs takes precedence. + */ + protected def getEffectivePassThroughArgs: Option[String] = { + val configuredArgs = get(passThroughArgs) + if (getDeviceType == LightGBMConstants.CPUDeviceType || + configuredArgs.exists(LightGBMUtils.hasDeviceParameter)) { + configuredArgs + } else { + val deviceArg = s"device_type=$getDeviceType" + configuredArgs.filter(_.trim.nonEmpty).map(args => s"$args $deviceArg").orElse(Some(deviceArg)) + } + } + + protected def getEffectiveDeviceType: String = { + getEffectivePassThroughArgs.flatMap(LightGBMUtils.effectiveDeviceType) + .getOrElse(getDeviceType) + } + /** * Constructs the ColumnParams. * @@ -372,8 +448,11 @@ trait LightGBMBase[TrainedModel <: Model[TrainedModel] with LightGBMModelParams] } protected def getDatasetCreationParams(categoricalIndexes: Array[Int], numThreads: Int): String = { + val effectiveDeviceType = getEffectiveDeviceType new ParamsStringBuilder(prefix = "", delimiter = "=") .appendParamValueIfNotThere("is_pre_partition", Option("True")) + .appendParamValueIfNotThere("device_type", + if (effectiveDeviceType == LightGBMConstants.CPUDeviceType) None else Option(effectiveDeviceType)) .appendParamValueIfNotThere("max_bin", Option(getMaxBin)) .appendParamValueIfNotThere("bin_construct_sample_cnt", Option(getBinSampleCount)) .appendParamValueIfNotThere("min_data_in_leaf", Option(getMinDataInLeaf)) @@ -399,6 +478,12 @@ trait LightGBMBase[TrainedModel <: Model[TrainedModel] with LightGBMModelParams] val numTasksPerExecutor = ClusterUtil.getNumTasksPerExecutor(dataset.sparkSession, log) val numTasks = determineNumTasks(dataset, getNumTasks, numTasksPerExecutor) + if (getEffectiveDeviceType == LightGBMConstants.CUDADeviceType) { + throw new IllegalArgumentException( + "deviceType=cuda is not supported by SynapseML's LightGBM 3.3.510 integration. " + + "The upstream CUDA objective dereferences missing CUDA metadata for SynapseML streaming Datasets and " + + "can crash the Spark executor. Use deviceType=gpu with an OpenCL-enabled custom native library.") + } val sc = dataset.sparkSession.sparkContext val df = prepareDataframe(dataset, numTasks) @@ -414,6 +499,10 @@ trait LightGBMBase[TrainedModel <: Model[TrainedModel] with LightGBMModelParams] val (numCols, numInitScoreClasses) = calculateColumnStatistics(preprocessedDF, measures) val featuresSchema = dataset.schema(getFeaturesCol) + // Validate before any native LightGBM call that consumes feature names (e.g. reference + // Dataset creation below), so an invalid name surfaces as an actionable error naming the + // offending columns rather than an opaque native failure. + validateSlotNames(featuresSchema) val generalTrainParams: BaseTrainParams = getTrainParams(numTasks, featuresSchema, numTasksPerExecutor) val trainParams = addCustomTrainParams(generalTrainParams, dataset) log.info(s"LightGBM batch $batchIndex of $batchCount, parameters: ${trainParams.toString()}") @@ -422,7 +511,7 @@ trait LightGBMBase[TrainedModel <: Model[TrainedModel] with LightGBMModelParams] val (serializedReferenceDataset: Option[Array[Byte]], partitionCounts: Option[Array[Long]]) = if (isStreamingMode) { val (referenceDataset, partitionCounts) = - calculateRowStatistics(trainingData, trainParams, numCols, measures) + calculateRowStatistics(trainingData, trainParams, numCols, featuresSchema, measures) // Save the reference Dataset so it's available to client and other batches if (getReferenceDataset.isEmpty) { @@ -432,7 +521,6 @@ trait LightGBMBase[TrainedModel <: Model[TrainedModel] with LightGBMModelParams] (Some(referenceDataset), Some(partitionCounts)) } else (None, None) - validateSlotNames(featuresSchema) executeTraining(preprocessedDF, validationData, serializedReferenceDataset, @@ -503,12 +591,14 @@ trait LightGBMBase[TrainedModel <: Model[TrainedModel] with LightGBMModelParams] * @param dataframe The dataset to train on. * @param trainingParams The training parameters. * @param numCols The number of feature columns. + * @param featuresSchema The schema of the features column. * @param measures Instrumentation measures. * @return The serialized Dataset reference and an array of partition counts. */ private def calculateRowStatistics(dataframe: DataFrame, trainingParams: BaseTrainParams, numCols: Int, + featuresSchema: StructField, measures: InstrumentationMeasures): (Array[Byte], Array[Long]) = { measures.markRowStatisticsStart() @@ -523,6 +613,9 @@ trait LightGBMBase[TrainedModel <: Model[TrainedModel] with LightGBMModelParams] trainingParams.generalParams.categoricalFeatures, trainingParams.executionParams.numThreads) + // Get feature names to set on the reference dataset (ensures unique names for Spark 3.5+) + val featureNames = getSlotNamesWithMetadata(featuresSchema) + // Either get a reference dataset (as bytes) from params, or calculate it val precalculatedDataset = getReferenceDataset val serializedReference = if (precalculatedDataset.nonEmpty) { @@ -541,6 +634,7 @@ trait LightGBMBase[TrainedModel <: Model[TrainedModel] with LightGBMModelParams] totalNumRows, numCols, collectedSampleData, + featureNames, measures, log) } @@ -595,10 +689,17 @@ trait LightGBMBase[TrainedModel <: Model[TrainedModel] with LightGBMModelParams] networkManager) // Execute the Tasks on workers - val lightGBMBooster = executePartitionTasks(ctx, dataframe, measures) - - // Wait for network to complete (should be done by now) - networkManager.waitForNetworkCommunicationsDone() + val lightGBMBooster = try { + val booster = executePartitionTasks(ctx, dataframe, measures) + + // Wait for network to complete (should be done by now) + networkManager.waitForNetworkCommunicationsDone() + booster + } finally { + // If training failed, the network thread may still be blocked accepting connections that will + // never arrive. Closing the driver sockets releases it and the port for the next attempt. + networkManager.closeConnections() + } measures.markTrainingStop() val model = getModel(trainParams, lightGBMBooster) measures.markExecutionEnd() diff --git a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMClassifier.scala b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMClassifier.scala index 819a74d0d68..5695dc6eab2 100644 --- a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMClassifier.scala +++ b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMClassifier.scala @@ -42,7 +42,7 @@ class LightGBMClassifier(override val uid: String) def getTrainParams(numTasks: Int, featuresSchema: StructField, numTasksPerExec: Int): BaseTrainParams = { ClassifierTrainParams( - get(passThroughArgs), + getEffectivePassThroughArgs, getIsUnbalance, getBoostFromAverage, get(isProvideTrainingMetric), diff --git a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMConstants.scala b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMConstants.scala index bd7f4c1499f..ebda0966525 100644 --- a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMConstants.scala +++ b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMConstants.scala @@ -46,6 +46,15 @@ object LightGBMConstants { /** Sampling mode take first n rows. */ val SubsetSamplingModeFixed: String = "fixed" + /** Tree learning on the CPU. This is LightGBM's own default. + */ + val CPUDeviceType: String = "cpu" + /** Tree learning on an OpenCL GPU. + */ + val GPUDeviceType: String = "gpu" + /** Tree learning on a CUDA GPU. + */ + val CUDADeviceType: String = "cuda" /** Enabled task, used to indicate task that creates lightgbm dataset and runs training. */ val EnabledTask: String = "enabledTask" diff --git a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMNetworkBridge.scala b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMNetworkBridge.scala new file mode 100644 index 00000000000..90eb0bc1fec --- /dev/null +++ b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMNetworkBridge.scala @@ -0,0 +1,346 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.lightgbm + +import org.slf4j.Logger + +import java.io.Closeable +import java.net.{InetAddress, InetSocketAddress, NetworkInterface, SocketException, UnknownHostException} +import java.nio.channels.ServerSocketChannel +import java.util.concurrent.atomic.AtomicInteger +import scala.collection.JavaConverters._ +import scala.collection.mutable +import scala.util.control.NonFatal + +/** The machine list, listen port, and machine count to hand to the native LGBM_NetworkInit call. */ +private[lightgbm] final case class BridgedNetwork(machineList: String, + localListenPort: Int, + machineCount: Int, + rank: Int) + +/** Bridges an IPv6 LightGBM training network onto the IPv4-only native transport. + * + * The native LightGBM socket layer is IPv4-only in every released version, including the + * lightgbmlib artifact this project depends on. `Linkers::ParseMachineList` splits each machine + * entry on ':' and keeps it only when it has exactly two parts, so `[2001:db8::1]:12400` is + * discarded and `2001:db8::1:12400` is misread as the host `1`; `TcpSocket` then builds every + * address with `socket(AF_INET, ...)`, `sockaddr_in`, and `inet_pton(AF_INET, ...)`, so even a + * parsed IPv6 literal could neither be dialed nor accepted. + * + * Rather than fail, this bridge keeps the native library on the transport it understands and + * carries the traffic itself: + * + * - an inbound relay owns the port this task advertised to its peers, accepts their connections + * on either address family, checks the LightGBM rank handshake, and forwards each link to the + * native listener over IPv4 loopback; + * - one outbound relay per IPv6 peer listens on IPv4 loopback and forwards whatever the native + * library sends to that peer's real IPv6 endpoint; + * - the machine list handed to the native library is rewritten so every bridged endpoint becomes + * a `127.0.0.1:port` entry, prefixed with an explicit `rank=` so the native library never has + * to infer its own position from a loopback address. + * + * Entry order is preserved because a LightGBM rank is an index into the machine list, and an IPv4 + * peer keeps its original entry so it stays a direct native connection. A machine list without any + * IPv6 entry never reaches this class: `requiresBridge` is false and the native call is made + * exactly as it always was. + * + * The native listener is the one thing this library cannot rebind: `TcpSocket::Bind` hardcodes + * `0.0.0.0`, so it is reachable on every IPv4 interface for as long as it is open, and the native + * accept loop has no timeout and trusts the first four bytes of every connection as a rank. The + * bridge therefore claims each of that listener's link slots itself, from loopback, as soon as the + * port is bound, which closes the listener within milliseconds of `LGBM_NetworkInit` binding it, + * and it performs the rank handshake with peers on its own port instead, where a stalled or + * invalid handshake is rejected rather than left to block the native accept loop. Removing the + * remaining window needs an upstream change: `TcpSocket::Bind` taking a bind address so that + * `Linkers::TryBind` can pass a loopback address. + */ +private[lightgbm] object LightGBMNetworkBridge { + /** Native LightGBM resolves this with inet_pton(AF_INET), so it has to stay a numeric IPv4 literal. */ + private[lightgbm] val LoopbackHost: String = "127.0.0.1" + + /** LightGBM reads this prefix from the machine list and skips its own local-address lookup. */ + private[lightgbm] val RankPrefix: String = "rank=" + + /** Covers the native connect-retry budget (20 attempts with a 1.3x backoff from 200ms). */ + private[lightgbm] val DefaultConnectTimeoutMillis: Long = 180000L + + /** A peer that has opened a connection has to identify itself well within the native timeout. */ + private[lightgbm] val DefaultHandshakeTimeoutMillis: Long = 30000L + + /** Headroom over the links a topology can actually need, which is one per other machine. */ + private val RelayedConnectionsPerMachine: Int = 2 + + private val BridgeCount = new AtomicInteger(0) + + private def nextBridgeId(): Int = BridgeCount.incrementAndGet() + + /** Parse a LightGBM machine list into its endpoints, preserving order. */ + def parseMachineList(machineList: String): Seq[WorkerEndpoint] = { + val entries = splitEntries(machineList) + if (entries.isEmpty) { + throw new IllegalArgumentException( + s"LightGBM machine list ${WorkerEndpoint.preview(machineList)} does not contain any endpoint") + } + entries.map(WorkerEndpoint.parse) + } + + /** Whether the native library would have to speak IPv6 to establish this network. + * + * This deliberately classifies without parsing, so an IPv4 machine list keeps reaching the + * native call unchanged even if it holds an entry this library would reject. + */ + def requiresBridge(machineList: String): Boolean = splitEntries(machineList).exists(isIpv6Entry) + + private def splitEntries(machineList: String): Seq[String] = + Option(machineList).getOrElse("").split(",").map(_.trim).filter(_.nonEmpty).toSeq + + /** An IPv6 entry is either bracketed or has more colons than the single host:port separator. */ + private def isIpv6Entry(entry: String): Boolean = entry.startsWith("[") || entry.count(_ == ':') > 1 + + /** Build the machine list handed to the native library. */ + private[lightgbm] def formatMachineList(rank: Int, entries: Seq[String]): String = + (Seq(s"$RankPrefix$rank") ++ entries).mkString(",") + + /** Locate this task's own entry, which is also its LightGBM rank. + * + * The driver echoes back the exact host string the task reported, so an exact match is the + * normal path. The fallbacks only matter when a host string is rewritten on the way (a + * differently compressed IPv6 literal, for example), and each one is anchored on the advertised + * port so it can never select another machine's entry. + */ + private[lightgbm] def findSelf(entries: Seq[WorkerEndpoint], advertisedHost: String, advertisedPort: Int): Int = { + val candidates = entries.zipWithIndex.filter { case (entry, _) => entry.port == advertisedPort } + val self = candidates.find { case (entry, _) => entry.host == advertisedHost } + .orElse(candidates.find { case (entry, _) => sameAddress(entry.host, advertisedHost) }) + .orElse(if (candidates.lengthCompare(1) == 0) candidates.headOption else None) + .orElse(candidates.find { case (entry, _) => isLocalAddress(entry.host) }) + self.map { case (_, index) => index }.getOrElse { + throw new IllegalArgumentException( + s"LightGBM machine list ${WorkerEndpoint.preview(entries.map(_.wireString).mkString(","))} does not " + + "contain this task's own endpoint " + + s"${WorkerEndpoint.preview(s"${WorkerEndpoint.wireHost(advertisedHost)}:$advertisedPort")}. " + + "The endpoint a task advertises to the driver has to appear in the machine list the driver sends back.") + } + } + + /** Resolve a peer host, failing with an actionable message instead of a bare UnknownHostException. */ + private[lightgbm] def resolvePeer(entry: WorkerEndpoint, log: Logger): InetAddress = { + if (entry.hasNumericZone) { + throw new IllegalArgumentException(s"Cannot reach LightGBM peer ${entry.wireString}: its IPv6 zone " + + s"identifier '${entry.zoneId.getOrElse("")}' is a numeric interface index, which only means " + + "anything on the machine that produced it. A peer has to advertise an interface name, such as " + + "fe80::1%eth0.") + } + val address = try { + InetAddress.getByName(entry.address) + } catch { + case failure: UnknownHostException => + throw new IllegalArgumentException(s"Cannot reach LightGBM peer ${entry.wireString}: " + + "the host could not be resolved on this machine.", failure) + } + if (address.isLinkLocalAddress) scopedLinkLocalAddress(entry, log) else address + } + + /** Attach a link-local peer's zone to a locally meaningful interface, or explain why it cannot be. */ + private def scopedLinkLocalAddress(entry: WorkerEndpoint, log: Logger): InetAddress = { + log.warn(s"LightGBM peer ${entry.wireString} is an IPv6 link-local address. A link-local address is only " + + "meaningful within one interface's scope, so its zone identifier is resolved against this machine's " + + "interfaces. Prefer a globally routable or unique-local IPv6 address for distributed training.") + entry.zoneId match { + case None => + throw new IllegalArgumentException(s"Cannot reach LightGBM peer ${entry.wireString}: an IPv6 " + + "link-local peer has to advertise a zone identifier (for example fe80::1%eth0), because a " + + "link-local address alone does not say which interface to send from.") + case Some(zone) => + try { + InetAddress.getByName(entry.host) + } catch { + case failure: UnknownHostException => + throw new IllegalArgumentException(s"Cannot reach LightGBM peer ${entry.wireString}: its IPv6 " + + s"zone identifier '$zone' does not name an interface on this machine, so the link-local " + + "address cannot be reached from here.", failure) + } + } + } + + private def sameAddress(host: String, otherHost: String): Boolean = { + if (host.isEmpty || otherHost.isEmpty) { + false + } else { + try { + InetAddress.getByName(host) == InetAddress.getByName(otherHost) + } catch { + case _: UnknownHostException => false + } + } + } + + private def isLocalAddress(host: String): Boolean = { + try { + val address = InetAddress.getByName(host) + address.isAnyLocalAddress || address.isLoopbackAddress || + NetworkInterface.getNetworkInterfaces.asScala.exists(_.getInetAddresses.asScala.contains(address)) + } catch { + case _: UnknownHostException => false + case _: SocketException => false + } + } + + /** Start the relays for a machine list and return a running bridge. + * + * The caller owns the returned bridge and has to close it once the native network is done with + * it, including when the native initialization it wraps fails. + */ + def open(machineList: String, + advertisedHost: String, + advertisedPort: Int, + log: Logger, + connectTimeoutMillis: Long = DefaultConnectTimeoutMillis, + handshakeTimeoutMillis: Long = DefaultHandshakeTimeoutMillis, + connectAttempt: LightGBMNetworkRelay.ConnectAttempt = + LightGBMNetworkRelay.DefaultConnect): LightGBMNetworkBridge = { + requireIpv6CapableJvm() + val entries = parseMachineList(machineList) + val rank = findSelf(entries, advertisedHost, advertisedPort) + val bridge = new LightGBMNetworkBridge(entries, rank, advertisedPort, log, connectTimeoutMillis, + handshakeTimeoutMillis, connectAttempt) + NetworkManagerSocketSupport.withCleanupOnFailurePreservingPrimary(bridge.close())(bridge.start()) + bridge + } + + /** A JVM forced onto the IPv4 stack cannot open an IPv6 socket at all, whatever the bridge does. */ + private def requireIpv6CapableJvm(): Unit = { + if (java.lang.Boolean.getBoolean("java.net.preferIPv4Stack")) { + throw new IllegalStateException("This LightGBM training network has IPv6 endpoints, but this JVM was " + + "started with -Djava.net.preferIPv4Stack=true, which prevents it from opening any IPv6 socket. " + + "Remove that option from the Spark executor JVM options, or give the executors IPv4 addresses.") + } + } + + private[lightgbm] def maxLinksFor(machineCount: Int): Int = machineCount * RelayedConnectionsPerMachine +} + +/** Owns the sockets and the single relay loop for one task. Created through its companion's `open`. */ +private[lightgbm] final class LightGBMNetworkBridge private(entries: Seq[WorkerEndpoint], + rank: Int, + advertisedPort: Int, + log: Logger, + connectTimeoutMillis: Long, + handshakeTimeoutMillis: Long, + connectAttempt: LightGBMNetworkRelay.ConnectAttempt) + extends Closeable { + import LightGBMNetworkBridge._ + + /** The name the relay thread of this bridge carries, so a thread dump attributes it. */ + private[lightgbm] val threadNamePrefix: String = s"lightgbm-network-bridge-${nextBridgeId()}-rank$rank" + + private val relay = new LightGBMNetworkRelay(threadNamePrefix, log, maxLinksFor(entries.length), + connectTimeoutMillis, handshakeTimeoutMillis, connectAttempt) + private val listeners = mutable.ListBuffer.empty[ServerSocketChannel] + private var nativeListenPort: Int = -1 + private var bridgedMachineList: String = "" + + /** The values to pass to the native LGBM_NetworkInit call. */ + def bridgedNetwork: BridgedNetwork = synchronized { + require(nativeListenPort > 0, "The LightGBM network bridge has not been started") + BridgedNetwork(bridgedMachineList, nativeListenPort, entries.length, rank) + } + + /** A failure the relay could not retry away, which leaves this task's transport unusable. */ + def terminalFailure: Option[Throwable] = relay.failure + + private[lightgbm] def relayLinkCount: Int = relay.linkCount + + private[lightgbm] def isRunning: Boolean = relay.isLoopAlive + + private[lightgbm] def start(): Unit = synchronized { + // Resolve every peer before binding anything, so an unreachable address fails immediately. + val peerAddresses = entries.zipWithIndex.map { case (entry, index) => + if (index == rank || !entry.isIpv6Literal) None else Some(resolvePeer(entry, log)) + } + + // Own the advertised port first: it is the only port peers know, and taking it before the + // native port is chosen guarantees the two cannot collide. + val inbound = bindWildcard(advertisedPort) + nativeListenPort = findFreePort() + relay.expectInboundRanks((0 until rank).toSet) + relay.start() + + val bridgedEntries = entries.zipWithIndex.map { case (entry, index) => + if (index == rank) { + WorkerEndpoint.wireString(LoopbackHost, nativeListenPort) + } else { + peerAddresses(index) + .map(address => WorkerEndpoint.wireString(LoopbackHost, startOutboundRelay(index, entry, address))) + .getOrElse(entry.wireString) + } + } + bridgedMachineList = formatMachineList(rank, bridgedEntries) + + relay.addInboundListener(inbound) + // Claim every slot of the native listener from loopback, so it closes as soon as it opens. + val nativeAddress = new InetSocketAddress(InetAddress.getByName(LoopbackHost), nativeListenPort) + (0 until rank).foreach(peerRank => relay.primeNativeLink(nativeAddress, peerRank)) + + log.info(s"LightGBM IPv6 network bridge is rank $rank of ${entries.length}, relaying peer traffic from " + + s"advertised port $advertisedPort to native listen port $nativeListenPort with machine list " + + s"$bridgedMachineList") + } + + private def startOutboundRelay(index: Int, entry: WorkerEndpoint, address: InetAddress): Int = { + val listener = bindLoopback() + relay.addOutboundListener(listener, new InetSocketAddress(address, entry.port), + s"LightGBM machine $index at ${entry.wireString}") + listener.socket().getLocalPort + } + + private def bindWildcard(port: Int): ServerSocketChannel = { + val listener = ServerSocketChannel.open() + NetworkManagerSocketSupport.withCleanupOnFailurePreservingPrimary(closeQuietly(listener)) { + // A wildcard bind is dual stack, so peers reach this port over either address family. + listener.bind(new InetSocketAddress(port)) + listeners += listener + listener + } + } + + private def bindLoopback(): ServerSocketChannel = { + val listener = ServerSocketChannel.open() + NetworkManagerSocketSupport.withCleanupOnFailurePreservingPrimary(closeQuietly(listener)) { + listener.bind(new InetSocketAddress(InetAddress.getByName(LoopbackHost), 0)) + listeners += listener + listener + } + } + + /** Pick a port for the native listener. Only this bridge ever dials it, over loopback. */ + private def findFreePort(): Int = { + val probe = ServerSocketChannel.open() + try { + probe.bind(new InetSocketAddress(0)) + probe.socket().getLocalPort + } finally { + closeQuietly(probe) + } + } + + private def closeQuietly(resource: Closeable): Unit = { + try { + resource.close() + } catch { + case NonFatal(failure) => log.debug("LightGBM network bridge could not close a channel", failure) + } + } + + /** Release every relay socket and the loop thread. Safe to call more than once and from any thread. */ + override def close(): Unit = { + relay.close() + val current = synchronized { + val snapshot = listeners.toSeq + listeners.clear() + snapshot + } + current.foreach(closeQuietly) + } +} diff --git a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMNetworkRelay.scala b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMNetworkRelay.scala new file mode 100644 index 00000000000..01ecb9ae7bb --- /dev/null +++ b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMNetworkRelay.scala @@ -0,0 +1,668 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.lightgbm + +import org.slf4j.Logger + +import java.io.{Closeable, IOException} +import java.net.{InetSocketAddress, StandardSocketOptions} +import java.nio.channels.{SelectionKey, Selector, ServerSocketChannel, SocketChannel} +import java.nio.{ByteBuffer, ByteOrder} +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger, AtomicReference} +import scala.annotation.tailrec +import scala.collection.JavaConverters._ +import scala.collection.mutable +import scala.util.control.NonFatal + +private[lightgbm] object LightGBMNetworkRelay { + /** LightGBM opens every link by sending its own rank as a host endian int. */ + val RankBytes: Int = 4 + + /** How a dial reaches its address. Injected so tests can fail a connect the way a route can. */ + type ConnectAttempt = (SocketChannel, InetSocketAddress) => Boolean + + val DefaultConnect: ConnectAttempt = (channel, address) => channel.connect(address) + + private val BufferSize: Int = 65536 + private val SocketBufferSize: Int = 100000 + private val SelectTimeoutMillis: Long = 100L + private val ShutdownWaitMillis: Long = 2000L + private val RetryIntervalMillis: Long = 5L + private val MaxRetryIntervalMillis: Long = 200L + + private final case class Timer(deadline: Long, action: () => Unit) + + private val TimerOrder: Ordering[Timer] = Ordering.by[Timer, Long](-_.deadline) + + private[lightgbm] def rankBuffer(rank: Int): ByteBuffer = + ByteBuffer.allocate(RankBytes).order(ByteOrder.nativeOrder()).putInt(rank) +} + +/** The event loop that moves LightGBM traffic between peers and the native library. + * + * One selector thread serves every listener and every relayed link, so the thread count of a + * bridge is one no matter how many machines the training network has. Memory is bounded the same + * way: two fixed buffers per relayed link, and links are capped, so nothing scales with peers + * except the sockets LightGBM itself would have opened. + * + * The loop also owns the LightGBM link handshake. A machine opens a link by sending its rank, so + * an inbound connection has to produce a valid, unused, lower rank within a deadline before any of + * its bytes reach the native library. Connections that stall, repeat a rank, or claim a rank the + * topology does not have are closed here instead of stalling the native listener thread, which has + * no timeout of its own. + * + * Every dial goes through one retry policy, whether the failure arrives synchronously (a connect + * to an address with no route fails on the calling thread) or asynchronously, and a failure while + * handling an accepted connection never reaches the listener that accepted it. What cannot be + * retried is recorded as a terminal failure, which the caller surfaces instead of leaving a task + * waiting on a transport that will not recover. + */ +private[lightgbm] final class LightGBMNetworkRelay(threadName: String, + log: Logger, + maxLinks: Int, + connectTimeoutMillis: Long, + handshakeTimeoutMillis: Long, + connectAttempt: LightGBMNetworkRelay.ConnectAttempt = + LightGBMNetworkRelay.DefaultConnect) extends Closeable { + import LightGBMNetworkRelay._ + + private val selector: Selector = Selector.open() + private val tasks = new ConcurrentLinkedQueue[() => Unit]() + private val timers = mutable.PriorityQueue.empty[Timer](TimerOrder) + private val closed = new AtomicBoolean(false) + private val started = new AtomicBoolean(false) + private val liveLinks = new AtomicInteger(0) + private val terminalFailure = new AtomicReference[Option[Throwable]](None) + private val resources = mutable.Set.empty[Closeable] + + // Only the loop thread touches these. + private var expectedInboundRanks: Set[Int] = Set.empty + private val primedNativeLinks = mutable.Map.empty[Int, SocketChannel] + private val peersAwaitingNativeLink = mutable.Map.empty[Int, (SocketChannel, LinkSlot)] + private val linkedRanks = mutable.Set.empty[Int] + + private val loopThread: Thread = { + val thread = new Thread(new Runnable { + override def run(): Unit = runLoop() + }, threadName) + thread.setDaemon(true) + thread + } + + /** The ranks allowed to open a link to this task, which LightGBM defines as the lower ranks. */ + def expectInboundRanks(ranks: Set[Int]): Unit = submit(() => expectedInboundRanks = ranks) + + def start(): Unit = if (started.compareAndSet(false, true)) loopThread.start() + + def linkCount: Int = liveLinks.get() + + /** A failure the relay cannot retry away, which leaves this task's transport unusable. */ + def failure: Option[Throwable] = terminalFailure.get() + + private[lightgbm] def isLoopAlive: Boolean = loopThread.isAlive + + /** Accept peer connections, which must pass the rank handshake before they are relayed. */ + def addInboundListener(listener: ServerSocketChannel): Unit = + register(listener, SelectionKey.OP_ACCEPT, new InboundAcceptor) + + /** Accept the native library's link to one peer and forward it to that peer's real address. */ + def addOutboundListener(listener: ServerSocketChannel, + target: InetSocketAddress, + description: String): Unit = + register(listener, SelectionKey.OP_ACCEPT, new OutboundAcceptor(target, description)) + + /** Take one of the native listener's link slots before anyone else can. + * + * The native listener accepts exactly one link per lower rank and then closes, so filling those + * slots from loopback as soon as the port is bound keeps every other caller out of a listener + * that the native library binds on all interfaces and that this library cannot rebind. + */ + def primeNativeLink(nativeAddress: InetSocketAddress, peerRank: Int): Unit = + submit(() => dialNative(nativeAddress, peerRank, System.currentTimeMillis() + connectTimeoutMillis, + RetryIntervalMillis)) + + override def close(): Unit = { + if (closed.compareAndSet(false, true)) { + selector.wakeup() + // Cleanup runs on a cancelled Spark task too, where the thread is already interrupted, so the + // wait for the loop has to survive that and hand the interrupt back to the caller. + if (started.get()) { + try { + loopThread.join(ShutdownWaitMillis) + } catch { + case _: InterruptedException => Thread.currentThread().interrupt() + } + } + closeEverything() + closeQuietly(selector) + log.info(s"$threadName closed") + } + } + + private def recordTerminalFailure(reason: String, failure: Throwable): Unit = { + log.error(s"$threadName $reason", failure) + terminalFailure.compareAndSet(None, Some(new IOException(s"$threadName $reason", failure))) + } + + private def submit(task: () => Unit): Unit = { + tasks.add(task) + selector.wakeup() + } + + private def register(channel: java.nio.channels.SelectableChannel, + ops: Int, + handler: Handler): Unit = { + track(channel) + submit(() => { + channel.configureBlocking(false) + channel.register(selector, ops, handler) + () + }) + } + + private def track(resource: Closeable): Unit = synchronized { + if (closed.get()) closeQuietly(resource) else resources += resource + } + + private def closeEverything(): Unit = { + val snapshot = synchronized { + val current = resources.toSeq + resources.clear() + current + } + snapshot.foreach(closeQuietly) + } + + private def closeQuietly(resource: Closeable): Unit = { + try { + resource.close() + } catch { + case NonFatal(failure) => log.debug(s"$threadName could not close a channel", failure) + } + } + + @tailrec + private def runLoop(): Unit = { + if (!closed.get()) { + step() + runLoop() + } + } + + private def step(): Unit = { + try { + runTasks() + selector.select(SelectTimeoutMillis) + processSelectedKeys() + runTimers() + } catch { + case NonFatal(failure) => if (!closed.get()) log.warn(s"$threadName event loop iteration failed", failure) + } + } + + @tailrec + private def runTasks(): Unit = { + val task = tasks.poll() + if (task != None.orNull) { + try { + task() + } catch { + case NonFatal(failure) => recordTerminalFailure("could not apply a registration", failure) + } + runTasks() + } + } + + private def processSelectedKeys(): Unit = { + val selected = selector.selectedKeys() + val snapshot = selected.asScala.toSeq + selected.clear() + snapshot.foreach(handleKey) + } + + private def handleKey(key: SelectionKey): Unit = { + val handler = key.attachment().asInstanceOf[Handler] + try { + if (key.isValid && key.isAcceptable) handler.onAcceptable(key) + if (key.isValid && key.isConnectable) handler.onConnectable(key) + if (key.isValid && (key.isReadable || key.isWritable)) handler.onReadWrite(key) + } catch { + case NonFatal(failure) => handler.onFailure(key, failure) + } + } + + private def runTimers(): Unit = { + val now = System.currentTimeMillis() + val due = mutable.ListBuffer.empty[Timer] + + @tailrec + def collect(): Unit = { + if (timers.nonEmpty && timers.head.deadline <= now) { + due += timers.dequeue() + collect() + } + } + + collect() + due.foreach { timer => + try { + timer.action() + } catch { + // A retry that cannot even be started is terminal: nothing else will drive it. + case NonFatal(failure) => recordTerminalFailure("could not run a scheduled retry", failure) + } + } + } + + private def scheduleAt(delayMillis: Long)(action: => Unit): Unit = + timers.enqueue(Timer(System.currentTimeMillis() + delayMillis, () => action)) + + private def openChannel(): SocketChannel = { + val channel = SocketChannel.open() + channel.configureBlocking(false) + configure(channel) + track(channel) + channel + } + + private def configure(channel: SocketChannel): Unit = { + try { + channel.setOption[java.lang.Boolean](StandardSocketOptions.TCP_NODELAY, true) + channel.setOption[Integer](StandardSocketOptions.SO_RCVBUF, SocketBufferSize) + channel.setOption[Integer](StandardSocketOptions.SO_SNDBUF, SocketBufferSize) + } catch { + case NonFatal(failure) => log.debug(s"$threadName could not configure a channel", failure) + } + } + + /** One admitted link, released exactly once however its life ends. */ + private final class LinkSlot { + private val released = new AtomicBoolean(false) + + def release(): Unit = if (released.compareAndSet(false, true)) liveLinks.decrementAndGet() + } + + /** Admit a link only while the topology could still need one. */ + private def admit(): Option[LinkSlot] = { + if (liveLinks.get() >= maxLinks) { + None + } else { + liveLinks.incrementAndGet() + Some(new LinkSlot) + } + } + + private def splice(first: SocketChannel, second: SocketChannel, slot: LinkSlot): Unit = + new RelayLink(first, second, slot).register() + + /** Open, register, and start a dial, so a synchronous failure follows the same policy as a late one. */ + private def beginDial(address: InetSocketAddress, + onConnected: SocketChannel => Unit, + onFailed: Throwable => Unit): Unit = { + val opened = try { + Some(openChannel()) + } catch { + case NonFatal(failure) => + onFailed(failure) + None + } + opened.foreach { channel => + try { + val key = channel.register(selector, SelectionKey.OP_CONNECT, new Dialer(channel, onConnected, onFailed)) + // A connect to an address with no route fails here rather than through the selector. + if (connectAttempt(channel, address)) { + key.interestOps(0) + onConnected(channel) + } + } catch { + case NonFatal(failure) => + closeQuietly(channel) + onFailed(failure) + } + } + } + + private def retryOrGiveUp(deadline: Long, + interval: Long, + failure: Throwable, + retry: Long => Unit, + giveUp: Throwable => Unit): Unit = { + if (!closed.get()) { + if (System.currentTimeMillis() >= deadline) { + giveUp(failure) + } else { + scheduleAt(interval)(retry(math.min(interval * 2, MaxRetryIntervalMillis))) + } + } + } + + private def dialNative(address: InetSocketAddress, peerRank: Int, deadline: Long, interval: Long): Unit = { + if (!closed.get()) { + beginDial(address, + channel => { + sendRank(channel, peerRank) + log.info(s"$threadName claimed the native link slot for rank $peerRank") + nativeLinkReady(channel, peerRank) + }, + failure => retryOrGiveUp(deadline, interval, failure, + next => dialNative(address, peerRank, deadline, next), + giveUp => nativeLinkFailed(peerRank, giveUp))) + } + } + + private def dialPeer(native: SocketChannel, + slot: LinkSlot, + address: InetSocketAddress, + description: String, + deadline: Long, + interval: Long): Unit = { + if (!closed.get()) { + beginDial(address, + channel => splice(native, channel, slot), + failure => retryOrGiveUp(deadline, interval, failure, + next => dialPeer(native, slot, address, description, deadline, next), + giveUp => { + recordTerminalFailure(s"could not reach $description, so its LightGBM link cannot be relayed", giveUp) + slot.release() + closeQuietly(native) + })) + } + } + + private def sendRank(channel: SocketChannel, rank: Int): Unit = { + val buffer = rankBuffer(rank) + buffer.flip() + + @tailrec + def write(attemptsLeft: Int): Unit = { + if (buffer.hasRemaining && attemptsLeft > 0) { + channel.write(buffer) + write(attemptsLeft - 1) + } + } + + write(RankBytes) + if (buffer.hasRemaining) throw new IOException(s"$threadName could not send a rank in one write") + } + + /** Hand a peer's connection to the native link slot already claimed for its rank. */ + private def linkPeerToNative(peer: SocketChannel, rank: Int, slot: LinkSlot): Unit = { + primedNativeLinks.remove(rank) match { + case Some(nativeChannel) => splice(peer, nativeChannel, slot) + case None => peersAwaitingNativeLink.put(rank, (peer, slot)).foreach { case (previous, previousSlot) => + previousSlot.release() + closeQuietly(previous) + } + } + } + + private def nativeLinkReady(nativeChannel: SocketChannel, rank: Int): Unit = { + peersAwaitingNativeLink.remove(rank) match { + case Some((peer, slot)) => splice(peer, nativeChannel, slot) + case None => primedNativeLinks.put(rank, nativeChannel).foreach(closeQuietly) + } + } + + private def nativeLinkFailed(rank: Int, failure: Throwable): Unit = { + recordTerminalFailure(s"could not claim the native link slot for rank $rank, so the native listener " + + "may keep waiting for a link that will never arrive", failure) + peersAwaitingNativeLink.remove(rank).foreach { case (peer, slot) => + slot.release() + closeQuietly(peer) + } + } + + private trait Handler { + def onAcceptable(key: SelectionKey): Unit = () + def onConnectable(key: SelectionKey): Unit = () + def onReadWrite(key: SelectionKey): Unit = () + def onFailure(key: SelectionKey, failure: Throwable): Unit = { + log.debug(s"$threadName closing a channel after a failure", failure) + key.cancel() + closeQuietly(key.channel()) + } + } + + /** A listener outlives the connections it accepts, so their failures never close it. */ + private trait ListenerHandler extends Handler { + override def onFailure(key: SelectionKey, failure: Throwable): Unit = { + if (!key.channel().isOpen) { + key.cancel() + if (!closed.get()) recordTerminalFailure("lost a listening socket", failure) + } else if (!closed.get()) { + log.warn(s"$threadName ignored a failure while accepting; the listener stays open", failure) + } + } + + /** Handle one accepted connection without ever letting its failure reach the listener. */ + protected def guard(accepted: SocketChannel, slot: Option[LinkSlot])(work: => Unit): Unit = { + try { + work + } catch { + case NonFatal(failure) => + log.warn(s"$threadName dropped an accepted connection", failure) + slot.foreach(_.release()) + closeQuietly(accepted) + } + } + } + + /** Accepts peer connections, which have to pass the rank handshake before they are relayed. */ + private final class InboundAcceptor extends ListenerHandler { + override def onAcceptable(key: SelectionKey): Unit = { + val listener = key.channel().asInstanceOf[ServerSocketChannel] + Option(listener.accept()).foreach { peer => + // Whether a link is wanted at all is decided before admission, so refusing one here can + // never consume a slot that the outbound links to peers also draw on. + if (expectedInboundRanks.isEmpty) { + log.warn(s"$threadName refused a connection from ${remoteAddress(peer)}: this task is the lowest " + + "rank, so no machine opens a link to it") + closeQuietly(peer) + } else { + admitPeer(peer) + } + } + } + + private def admitPeer(peer: SocketChannel): Unit = { + admit() match { + case None => + log.warn(s"$threadName refused a connection from ${remoteAddress(peer)}: this task expects " + + s"${expectedInboundRanks.size} inbound links and already holds ${liveLinks.get()}") + closeQuietly(peer) + case Some(slot) => guard(peer, Some(slot))(beginHandshake(peer, slot)) + } + } + + private def beginHandshake(peer: SocketChannel, slot: LinkSlot): Unit = { + peer.configureBlocking(false) + configure(peer) + track(peer) + val handshake = new RankHandshake(peer, slot) + peer.register(selector, SelectionKey.OP_READ, handshake) + scheduleAt(handshakeTimeoutMillis)(handshake.onDeadline()) + } + } + + /** Reads and validates the rank a peer sends before any of its bytes reach the native library. */ + private final class RankHandshake(peer: SocketChannel, slot: LinkSlot) extends Handler { + private val buffer = ByteBuffer.allocate(RankBytes) + private var settled = false + + override def onReadWrite(key: SelectionKey): Unit = { + val count = peer.read(buffer) + if (count < 0) { + reject(key, "closed the connection before sending its rank") + } else if (!buffer.hasRemaining) { + complete(key) + } + } + + def onDeadline(): Unit = { + if (!settled) { + settled = true + slot.release() + log.warn(s"$threadName closed a connection from ${remoteAddress(peer)} that did not send a " + + s"LightGBM rank within ${handshakeTimeoutMillis}ms") + closeQuietly(peer) + } + } + + override def onFailure(key: SelectionKey, failure: Throwable): Unit = reject(key, s"failed: $failure") + + private def complete(key: SelectionKey): Unit = { + buffer.flip() + val rank = buffer.order(ByteOrder.nativeOrder()).getInt + if (!expectedInboundRanks.contains(rank)) { + reject(key, s"claimed rank $rank, which is not one of the lower ranks this task expects") + } else if (!linkedRanks.add(rank)) { + reject(key, s"claimed rank $rank, which is already linked") + } else { + settled = true + // The key is reused by the relayed link, so it must not be cancelled here. + key.interestOps(0) + log.info(s"$threadName accepted the link from rank $rank at ${remoteAddress(peer)}") + linkPeerToNative(peer, rank, slot) + } + } + + private def reject(key: SelectionKey, reason: String): Unit = { + if (!settled) { + settled = true + slot.release() + log.warn(s"$threadName refused a connection from ${remoteAddress(peer)}: it $reason") + } + key.cancel() + closeQuietly(peer) + } + } + + /** Accepts the native library's link to one peer and forwards it to that peer's real address. */ + private final class OutboundAcceptor(target: InetSocketAddress, description: String) extends ListenerHandler { + override def onAcceptable(key: SelectionKey): Unit = { + val listener = key.channel().asInstanceOf[ServerSocketChannel] + Option(listener.accept()).foreach { native => + admit() match { + case None => + log.warn(s"$threadName refused a native link to $description beyond the $maxLinks links " + + "this topology can need") + closeQuietly(native) + case Some(slot) => guard(native, Some(slot)) { + native.configureBlocking(false) + configure(native) + track(native) + native.register(selector, 0, new Handler {}) + dialPeer(native, slot, target, description, + System.currentTimeMillis() + connectTimeoutMillis, RetryIntervalMillis) + } + } + } + } + } + + /** A non blocking connect whose outcome, early or late, is handed to the same policy. */ + private final class Dialer(channel: SocketChannel, + onConnected: SocketChannel => Unit, + onFailed: Throwable => Unit) extends Handler { + override def onConnectable(key: SelectionKey): Unit = { + if (channel.finishConnect()) { + key.interestOps(0) + onConnected(channel) + } + } + + override def onFailure(key: SelectionKey, failure: Throwable): Unit = { + key.cancel() + closeQuietly(channel) + onFailed(failure) + } + } + + private def remoteAddress(channel: SocketChannel): String = + try { + Option(channel.getRemoteAddress).map(_.toString).getOrElse("an unknown address") + } catch { + case NonFatal(_) => "an unknown address" + } + + /** Two channels relayed in both directions, closed together. */ + private final class RelayLink(first: SocketChannel, second: SocketChannel, slot: LinkSlot) { + private val forward = new Direction(first, second) + private val backward = new Direction(second, first) + private val ended = new AtomicBoolean(false) + + def register(): Unit = { + first.register(selector, SelectionKey.OP_READ, new LinkEnd(this)) + second.register(selector, SelectionKey.OP_READ, new LinkEnd(this)) + pump() + } + + def pump(): Unit = { + forward.pump() + backward.pump() + if (forward.finished && backward.finished) { + finish() + } else { + updateInterest(first) + updateInterest(second) + } + } + + /** Any I/O failure ends both directions, so the other one can never wait on a dead channel. */ + def abort(failure: Throwable): Unit = { + log.debug(s"$threadName aborting a relayed link", failure) + finish() + } + + private def finish(): Unit = { + if (ended.compareAndSet(false, true)) { + slot.release() + closeQuietly(first) + closeQuietly(second) + } + } + + private def updateInterest(channel: SocketChannel): Unit = { + val key = channel.keyFor(selector) + if (key != None.orNull && key.isValid) { + val reading = Seq(forward, backward).find(_.source eq channel).exists(_.wantsRead) + val writing = Seq(forward, backward).find(_.target eq channel).exists(_.wantsWrite) + val ops = (if (reading) SelectionKey.OP_READ else 0) | (if (writing) SelectionKey.OP_WRITE else 0) + key.interestOps(ops) + } + } + } + + private final class LinkEnd(link: RelayLink) extends Handler { + override def onReadWrite(key: SelectionKey): Unit = link.pump() + + override def onFailure(key: SelectionKey, failure: Throwable): Unit = link.abort(failure) + } + + /** One half of a relayed link: read into a fixed buffer, write it out, then propagate the close. */ + private final class Direction(val source: SocketChannel, val target: SocketChannel) { + private val buffer = ByteBuffer.allocate(BufferSize) + private var sourceEnded = false + private var targetShutdown = false + + def wantsRead: Boolean = !sourceEnded && buffer.hasRemaining + + def wantsWrite: Boolean = buffer.position() > 0 + + def finished: Boolean = sourceEnded && buffer.position() == 0 && targetShutdown + + def pump(): Unit = { + if (wantsRead && source.read(buffer) < 0) sourceEnded = true + buffer.flip() + if (buffer.hasRemaining) target.write(buffer) + buffer.compact() + // A clean end of stream is passed on as a half close, which is what LightGBM sends. + if (sourceEnded && buffer.position() == 0 && !targetShutdown) { + target.shutdownOutput() + targetShutdown = true + } + } + } +} diff --git a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRanker.scala b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRanker.scala index f469fbd5d08..a87fc59392b 100644 --- a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRanker.scala +++ b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRanker.scala @@ -50,7 +50,7 @@ class LightGBMRanker(override val uid: String) def getTrainParams(numTasks: Int, featuresSchema: StructField, numTasksPerExec: Int): BaseTrainParams = { RankerTrainParams( - get(passThroughArgs), + getEffectivePassThroughArgs, getMaxPosition, getLabelGain, getEvalAt, diff --git a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRegressor.scala b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRegressor.scala index 405a1a85bb8..dfe5ee8629e 100644 --- a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRegressor.scala +++ b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRegressor.scala @@ -59,7 +59,7 @@ class LightGBMRegressor(override val uid: String) def getTrainParams(numTasks: Int, featuresSchema: StructField, numTasksPerExec: Int): BaseTrainParams = { RegressorTrainParams( - get(passThroughArgs), + getEffectivePassThroughArgs, getAlpha, getTweedieVariancePower, getBoostFromAverage, diff --git a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMUtils.scala b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMUtils.scala index e04b896e399..52da2fceb0e 100644 --- a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMUtils.scala +++ b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMUtils.scala @@ -10,8 +10,50 @@ import org.apache.spark.ml.PipelineModel import org.apache.spark.sql.Dataset import org.apache.spark.{SparkEnv, TaskContext} +import java.util.Locale + /** Helper utilities for LightGBM learners */ object LightGBMUtils { + private val DeviceParamNames = Set("device", "device_type") + + private def removeLightGBMQuotationSymbols(value: String): String = { + def isQuote(char: Char): Boolean = char == '\'' || char == '"' + value.dropWhile(isQuote).reverse.dropWhile(isQuote).reverse + } + + private[lightgbm] def parseLightGBMParams(args: String): Map[String, String] = { + args.split("[ \\t\\n\\r]+").iterator.filter(_.nonEmpty).foldLeft(Map.empty[String, String]) { + case (params, token) => + val parts = token.split("=", -1).filter(_.nonEmpty) + if (parts.length == 2) { + val key = removeLightGBMQuotationSymbols(parts(0).trim) + val value = removeLightGBMQuotationSymbols(parts(1).trim) + if (key.nonEmpty && !params.contains(key)) params + (key -> value) else params + } else { + params + } + } + } + + private[lightgbm] def hasDeviceParameter(parameters: String): Boolean = + parseLightGBMParams(parameters).keys.exists(DeviceParamNames) + + private[lightgbm] def effectiveDeviceType(parameters: String): Option[String] = { + val params = parseLightGBMParams(parameters) + params.get("device_type").orElse(params.get("device")).map(_.toLowerCase(Locale.ROOT)) + } + + private[lightgbm] def boosterFailureGuidance(parameters: String): String = { + effectiveDeviceType(parameters) + .filter(device => device == LightGBMConstants.GPUDeviceType || device == LightGBMConstants.CUDADeviceType) + .map { device => + s" Requested device_type=$device. SynapseML's bundled LightGBM native libraries are CPU-only; " + + "GPU/CUDA training requires compatible custom lib_lightgbm and lib_lightgbm_swig libraries on " + + "java.library.path for every Spark driver and executor before LightGBM is initialized." + } + .getOrElse("") + } + def validate(result: Int, component: String): Unit = { if (result == -1) { throw new Exception(component + " call failed in LightGBM with error: " @@ -19,6 +61,14 @@ object LightGBMUtils { } } + def validateBooster(result: Int, parameters: String): Unit = { + if (result == -1) { + val nativeError = lightgbmlib.LGBM_GetLastError() + val guidance = boosterFailureGuidance(parameters) + throw new Exception(s"Booster call failed in LightGBM with error: $nativeError$guidance") + } + } + def validateArray(result: SWIGTYPE_p_void, component: String): Unit = { if (result == null) { throw new Exception(component + " call failed in LightGBM with error: " diff --git a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/NetworkManager.scala b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/NetworkManager.scala index c91c01644f5..65714dc3c58 100644 --- a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/NetworkManager.scala +++ b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/NetworkManager.scala @@ -5,141 +5,36 @@ package com.microsoft.azure.synapse.ml.lightgbm import com.microsoft.azure.synapse.ml.core.env.StreamUtilities.{using, usingMany} import com.microsoft.azure.synapse.ml.core.utils.{ClusterUtil, FaultToleranceUtils} -import com.microsoft.azure.synapse.ml.lightgbm.NetworkManager.parseWorkerMessage import com.microsoft.ml.lightgbm.lightgbmlib import org.apache.spark.BarrierTaskContext +import org.apache.spark.TaskContext import org.apache.spark.internal.Logging import org.apache.spark.sql.SparkSession import org.slf4j.Logger import java.io.{BufferedReader, BufferedWriter, IOException, InputStreamReader, OutputStreamWriter} -import java.net.{BindException, InetSocketAddress, ServerSocket, Socket} -import java.util.concurrent.Executors +import java.net.{ConnectException, ServerSocket, Socket, SocketException, SocketTimeoutException} +import java.util.concurrent.{ExecutorService, Executors} import scala.annotation.tailrec import scala.collection.mutable -import scala.collection.mutable.ListBuffer import scala.concurrent.{Await, ExecutionContext, ExecutionContextExecutor, Future} import scala.concurrent.duration.{Duration, SECONDS} import scala.language.existentials - -case class TaskMessageInfo(status: String, - taskHost: String, - localListenPort: Int, - partitionId: Int, - executorId: String) { - def this(status: String) = this(status, "", -1, -1, "") // Constructor for general messages, not Task-connected - - val isForTraining: Boolean = status == LightGBMConstants.EnabledTask - val isForLoadOnly: Boolean = status == LightGBMConstants.IgnoreStatus - val isFinished: Boolean = status == LightGBMConstants.FinishedStatus - - // Format all the information as a delimited string to send to driver - override def toString: String = s"$status:$taskHost:$localListenPort:$partitionId:$executorId" -} - -case class NetworkTopologyInfo(lightgbmNetworkString: String, - executorPartitionIdList: Array[Int], - localListenPort: Int) { - @transient private var portReservation: Option[Socket] = None - - private def currentPortReservation: Option[Socket] = Option(portReservation).flatten - - private[lightgbm] def hasPortReservation: Boolean = synchronized { - currentPortReservation.nonEmpty - } - - private[lightgbm] def retainPortReservation(reservation: Socket): NetworkTopologyInfo = synchronized { - require(!reservation.isClosed, "Cannot retain a closed port reservation") - require(reservation.isBound, "Cannot retain an unbound port reservation") - require(reservation.getLocalPort == localListenPort, - s"Port reservation ${reservation.getLocalPort} does not match topology port $localListenPort") - require(currentPortReservation.isEmpty, s"Port $localListenPort already has a reservation") - portReservation = Option(reservation) - this - } - - /** Release the temporary JVM reservation immediately before LightGBM binds the same port. - * - * The operation is idempotent so final cleanup can safely call it after any success or failure path. - */ - private[lightgbm] def releasePortReservation(): Unit = synchronized { - currentPortReservation.foreach { reservation => - try { - NetworkManager.closeSocketWithRetry(reservation) - } finally { - // Keep an open socket reachable for a later final-cleanup attempt. - if (reservation.isClosed) portReservation = None - } - } - } -} +import scala.util.control.NonFatal object NetworkManager { - private val MaxSocketCloseAttempts = 2 - - private def addSuppressed(primaryFailure: Throwable, secondaryFailure: Throwable): Unit = { - if (primaryFailure ne secondaryFailure) primaryFailure.addSuppressed(secondaryFailure) - } - - /** Close a socket with one immediate retry, retaining every observed cleanup failure. */ - private[lightgbm] def closeSocketWithRetry(socket: Socket): Unit = { - @tailrec - def attemptClose(attemptsRemaining: Int, - firstFailure: Option[IOException]): Option[IOException] = { - if (socket.isClosed || attemptsRemaining == 0) { - firstFailure - } else { - val updatedFailure = try { - socket.close() - firstFailure - } catch { - case failure: IOException => - firstFailure.foreach(existing => addSuppressed(existing, failure)) - firstFailure.orElse(Option(failure)) - } - attemptClose(attemptsRemaining - 1, updatedFailure) - } - } + private def addSuppressed(primaryFailure: Throwable, secondaryFailure: Throwable): Unit = + NetworkManagerSocketSupport.addSuppressed(primaryFailure, secondaryFailure) - val closeFailure = attemptClose(MaxSocketCloseAttempts, None).orElse { - if (socket.isClosed) None else Option(new IOException("Socket remained open after cleanup attempts")) - } - closeFailure.foreach(throw _) - } + private[lightgbm] def closeSocketWithRetry(socket: Socket): Unit = + NetworkManagerSocketSupport.closeSocketWithRetry(socket) - /** Run cleanup without allowing it to replace a failure from the protected operation. - * - * The Throwable catch is deliberately limited to recording and immediately rethrowing the original - * failure; it is not a fallback or recovery boundary. - */ - private[lightgbm] def withCleanupPreservingPrimary[T](cleanup: => Unit)(operation: => T): T = { - var primaryFailure: Option[Throwable] = None - try { - operation - } catch { - case failure: Throwable => - primaryFailure = Option(failure) - throw failure - } finally { - try { - cleanup - } catch { - case cleanupFailure: Throwable if primaryFailure.isDefined => - addSuppressed(primaryFailure.get, cleanupFailure) - } - } - } + private[lightgbm] def withCleanupPreservingPrimary[T](cleanup: => Unit)(operation: => T): T = + NetworkManagerSocketSupport.withCleanupPreservingPrimary(cleanup)(operation) + + private[lightgbm] def withCleanupOnFailurePreservingPrimary[T](cleanup: => Unit)(operation: => T): T = + NetworkManagerSocketSupport.withCleanupOnFailurePreservingPrimary(cleanup)(operation) - /** Run cleanup only when the protected operation fails, preserving that primary failure. */ - private[lightgbm] def withCleanupOnFailurePreservingPrimary[T] - (cleanup: => Unit)(operation: => T): T = { - var completed = false - withCleanupPreservingPrimary(if (!completed) cleanup) { - val result = operation - completed = true - result - } - } /** * Create a NetworkManager, which will encapsulate all network operations. * This method will opens a socket communications channel on the driver, and then initialize @@ -210,13 +105,18 @@ object NetworkManager { localListenPort => log.info(s"LightGBM task $taskId connecting to host: " + s"${networkParams.ipAddress}, port: ${networkParams.port}") - FaultToleranceUtils.retryWithTimeout() { - getNetworkTopologyInfoFromDriver(networkParams, - taskId, - partitionId, - localListenPort, - log, - shouldExecuteTraining) + try { + FaultToleranceUtils.retryWithTimeout() { + getNetworkTopologyInfoFromDriver(networkParams, + taskId, + partitionId, + localListenPort, + log, + shouldExecuteTraining) + } + } catch { + case connectFailure: ConnectException => + throw driverUnreachableException(networkParams, taskId, partitionId, log, connectFailure) } } } finally { @@ -224,6 +124,36 @@ object NetworkManager { } } + /** Explain why a task could not reach the driver's network topology endpoint. + * + * The driver serves the topology exchange exactly once per training round and then closes its + * server socket. A task that Spark retries after that point can therefore only ever see + * "connection refused", which silently replaces the failure that caused the retry in the first + * place. Naming that explicitly keeps the original failure discoverable. + */ + private[lightgbm] def driverUnreachableException(networkParams: NetworkParams, + taskId: Long, + partitionId: Int, + log: Logger, + cause: ConnectException): Exception = { + val attemptNumber = Option(TaskContext.get()).map(_.attemptNumber()).getOrElse(0) + val endpoint = s"${networkParams.ipAddress}:${networkParams.port}" + val message = if (attemptNumber > 0) { + s"LightGBM task $taskId (partition $partitionId) could not reach the driver network topology endpoint " + + s"$endpoint on retry attempt $attemptNumber. The driver serves the topology exchange once per training " + + "round and has already closed it, so a retried task can never rejoin the LightGBM network. This error " + + s"is therefore a consequence of an earlier failure: inspect the logs of the first failed attempt of " + + s"partition $partitionId to find the real cause. Distributed LightGBM training cannot recover from a " + + "partial task retry." + } else { + s"LightGBM task $taskId (partition $partitionId) could not reach the driver network topology endpoint " + + s"$endpoint on its first attempt. Verify that executors are allowed to open connections to the driver " + + "on that port, and that the driver was not shut down before training started." + } + log.error(message, cause) + new Exception(message, cause) + } + private def getNetworkTopologyInfoFromDriver(networkParams: NetworkParams, taskId: Long, partitionId: Int, @@ -239,13 +169,16 @@ object NetworkManager { val driverOutput = io(1).asInstanceOf[BufferedWriter] // Get message to send to driver with info about this task + val stageAttemptNumber = Option(TaskContext.get()).map(_.stageAttemptNumber()).getOrElse(0) + // A numeric IPv6 scope is an interface index that only means anything here, so it is + // replaced with the interface name before any peer sees it. val taskStatus = TaskMessageInfo( if (shouldExecuteTraining) LightGBMConstants.EnabledTask else LightGBMConstants.IgnoreStatus, - driverSocket.getLocalAddress.getHostAddress, + WorkerEndpoint.normalizeHost(driverSocket.getLocalAddress.getHostAddress), localListenPort, partitionId, LightGBMUtils.getExecutorId) // TODO can we use host for this? - val message = taskStatus.toString() + val message = WorkerMessage.format(taskStatus, stageAttemptNumber) log.info(s"task $taskId sending status message to driver: $message ") driverOutput.write(s"$message\n") driverOutput.flush() @@ -255,7 +188,7 @@ object NetworkManager { val context = BarrierTaskContext.get() context.barrier() if (context.partitionId() == 0) { - setFinishedStatus(networkParams, log) + setFinishedStatus(networkParams, stageAttemptNumber, context.getTaskInfos().length, log) } } @@ -274,6 +207,7 @@ object NetworkManager { val executorPartitionIds: Array[Int] = parseExecutorPartitionList(partitionsByExecutorStr, taskStatus.executorId, log) NetworkTopologyInfo(lightGbmMachineList, executorPartitionIds, localListenPort) + .withAdvertisedHost(taskStatus.taskHost) }.get }.get } @@ -301,15 +235,59 @@ object NetworkManager { log, retry, delay, - () => LightGBMUtils.validate(lightgbmlib.LGBM_NetworkInit( - ctx.lightGBMNetworkString, - ctx.localListenPort, - LightGBMConstants.DefaultListenTimeout, - ctx.lightGBMNetworkMachineCount), "Network init"), + () => initNativeNetwork(ctx.networkTopologyInfo, ctx.lightGBMNetworkMachineCount, log), port => reserveExactPort(port, log), delayMillis => Thread.sleep(delayMillis)) } + /** Initialize the native LightGBM network, bridging the transport when the topology is IPv6. + * + * Native LightGBM only speaks IPv4, so an IPv6 topology is relayed by [[LightGBMNetworkBridge]] + * and the native library is given an equivalent loopback machine list. An IPv4 topology takes + * exactly the same path it always has, with no bridge, no relay threads, and no extra sockets. + */ + private[lightgbm] def initNativeNetwork(networkTopologyInfo: NetworkTopologyInfo, + machineCount: Int, + log: Logger, + nativeInit: (String, Int, Int) => Unit = nativeNetworkInit): Unit = { + val machineList = networkTopologyInfo.lightgbmNetworkString + if (!LightGBMNetworkBridge.requiresBridge(machineList)) { + nativeInit(machineList, networkTopologyInfo.localListenPort, machineCount) + } else { + log.info(s"LightGBM network $machineList contains IPv6 endpoints, which the native library cannot " + + "dial, so this task is bridging the transport") + val bridge = LightGBMNetworkBridge.open(machineList, + networkTopologyInfo.taskHost, + networkTopologyInfo.localListenPort, + log) + // A failed attempt has to give the advertised port back, because the retry re-reserves it. + withCleanupOnFailurePreservingPrimary(bridge.close()) { + val bridged = bridge.bridgedNetwork + // A relay that has already failed can never carry this network, and the native call would + // wait on links that will not arrive, so the attempt fails here instead. + failIfBridgeIsBroken(bridge) + nativeInit(bridged.machineList, bridged.localListenPort, bridged.machineCount) + failIfBridgeIsBroken(bridge) + networkTopologyInfo.retainNetworkBridge(bridge) + } + } + } + + /** Surface a relay failure as this task's failure, rather than training on a dead transport. */ + private[lightgbm] def failIfBridgeIsBroken(bridge: LightGBMNetworkBridge): Unit = { + bridge.terminalFailure.foreach { failure => + throw new Exception("The LightGBM IPv6 network bridge for this task failed, so its training " + + s"network cannot be established: ${failure.getMessage}", failure) + } + } + + private def nativeNetworkInit(machineList: String, localListenPort: Int, machineCount: Int): Unit = { + LightGBMUtils.validate(lightgbmlib.LGBM_NetworkInit(machineList, + localListenPort, + LightGBMConstants.DefaultListenTimeout, + machineCount), "Network init") + } + /** Retry native network initialization without leaving the advertised port open during backoff. */ private[lightgbm] def initLightGBMNetworkWithRetry(networkTopologyInfo: NetworkTopologyInfo, log: Logger, @@ -411,20 +389,15 @@ object NetworkManager { * Used to minimize network communication overhead in reduce step. * @return The main node's port number. */ + private[lightgbm] def parseHostAndPort(endpoint: String): (String, Int) = { + val parsed = WorkerEndpoint.parse(endpoint) + (parsed.host, parsed.port) + } + def getMainWorkerPort(nodes: String, log: Logger): Int = { - val nodesList = nodes.split(",") - if (nodesList.isEmpty) { - throw new Exception("Error: could not split nodes list correctly") - } - val mainNode = nodesList(0) - val hostAndPort = mainNode.split(":") - if (hostAndPort.length != 2) { - throw new Exception("Error: could not parse main worker host and port correctly") - } - val mainHost = hostAndPort(0) - val mainPort = hostAndPort(1) - log.info(s"LightGBM setting main worker host: $mainHost and port: $mainPort") - mainPort.toInt + val mainWorker = WorkerEndpoint.parseFirst(nodes) + log.info(s"LightGBM setting main worker host: ${mainWorker.host} and port: ${mainWorker.port}") + mainWorker.port } private def findOpenPort(ctx: TrainingContext, log: Logger): Socket = { @@ -433,113 +406,42 @@ object NetworkManager { reserveOpenPort(basePort, log) } - /** Reserve the first available port at or above basePort. - * - * Only address-in-use failures advance to another port. Other failures propagate after the candidate - * socket is closed, rather than silently falling back to a different port. - */ - private[lightgbm] def reserveOpenPort(basePort: Int, log: Logger): Socket = { - reserveOpenPort(basePort, log, () => new Socket()) - } + private[lightgbm] def reserveOpenPort(basePort: Int, log: Logger): Socket = + NetworkManagerSocketSupport.reserveOpenPort(basePort, log) private[lightgbm] def reserveOpenPort(basePort: Int, log: Logger, - createSocket: () => Socket): Socket = { - validatePort(basePort) - - @tailrec - def reservePort(localListenPort: Int): Socket = { - val bindResult: Either[BindException, Socket] = try { - Right(reserveExactPort(localListenPort, log, createSocket)) - } catch { - // A suppressed exception means candidate cleanup failed, so proceeding would leak a socket. - case contention: BindException if contention.getSuppressed.isEmpty => Left(contention) - } - - bindResult match { - case Right(reservation) => reservation - case Left(_) => - log.warn(s"Could not bind to port $localListenPort...") - val nextPort = localListenPort + 1 - if (nextPort > LightGBMConstants.MaxPort) { - throw new Exception(s"Error: port $basePort out of range, " + - "possibly due to networking or firewall issues") - } - if (nextPort - basePort > 1000) { - throw new Exception("Error: Could not find open port after 1k tries") - } - reservePort(nextPort) - } - } - - reservePort(basePort) - } - - /** Reserve one exact port. Native-init retries cannot change the previously advertised port. */ - private[lightgbm] def reserveExactPort(localListenPort: Int, log: Logger): Socket = { - reserveExactPort(localListenPort, log, () => new Socket()) - } - - private def reserveExactPort(localListenPort: Int, - log: Logger, - createSocket: () => Socket): Socket = { - validatePort(localListenPort) - val candidate = createSocket() - withCleanupOnFailurePreservingPrimary(closeSocketWithRetry(candidate)) { - candidate.bind(new InetSocketAddress(localListenPort)) - log.info(s"Successfully bound to port $localListenPort") - candidate - } - } + createSocket: () => Socket): Socket = + NetworkManagerSocketSupport.reserveOpenPort(basePort, log, createSocket) - private def validatePort(port: Int): Unit = { - if (port < 0 || port > LightGBMConstants.MaxPort) { - throw new Exception(s"Error: port $port out of range, possibly due to too many executors or unknown error") - } - } + private[lightgbm] def reserveExactPort(localListenPort: Int, log: Logger): Socket = + NetworkManagerSocketSupport.reserveExactPort(localListenPort, log) - /** Keep a training task's port reserved, while releasing helper and failed-task reservations immediately. */ private[lightgbm] def withPortReservation(reservation: Socket, shouldExecuteTraining: Boolean) - (getTopology: Int => NetworkTopologyInfo): NetworkTopologyInfo = { - var retained = false - withCleanupPreservingPrimary(if (!retained) closeSocketWithRetry(reservation)) { - val topology = getTopology(reservation.getLocalPort) - if (shouldExecuteTraining) { - topology.retainPortReservation(reservation) - retained = true - } - topology - } - } + (getTopology: Int => NetworkTopologyInfo): NetworkTopologyInfo = + NetworkManagerSocketSupport.withPortReservation(reservation, shouldExecuteTraining)(getTopology) - private def setFinishedStatus(networkParams: NetworkParams, log: Logger): Unit = { + private def setFinishedStatus(networkParams: NetworkParams, + stageAttemptNumber: Int, + barrierTaskCount: Int, + log: Logger): Unit = { using(new Socket(networkParams.ipAddress, networkParams.port)) { driverSocket => using(new BufferedWriter(new OutputStreamWriter(driverSocket.getOutputStream))) { driverOutput => - log.info("sending finished status to driver") - // If barrier execution mode enabled, create a barrier across tasks - driverOutput.write(s"${LightGBMConstants.FinishedStatus}\n") + log.info(s"sending finished status to driver for $barrierTaskCount barrier tasks") + // The barrier task count tells the driver how many topology reports to expect. It can be + // smaller than numTasks, because barrier mode never repartitions upwards when the input + // has fewer partitions than numTasks. + driverOutput.write(s"${WorkerMessage.formatFinished(stageAttemptNumber, barrierTaskCount)}\n") driverOutput.flush() }.get }.get } def parseWorkerMessage(message: String): TaskMessageInfo = { - val components = message.split(":") - val status = components(0) - - if (status == LightGBMConstants.FinishedStatus) new TaskMessageInfo(status) - else { - if (components.length != 5) throw new Exception(s"Unexpected message: $message") - - val host = components(1) - val port = components(2).toInt - val partitionId: Int = components(3).toInt - val executorId = components(4) //scalastyle:ignore magic.number - TaskMessageInfo(status, host, port, partitionId, executorId) - } + WorkerMessage.parse(message).toTaskMessage } } @@ -554,43 +456,103 @@ case class NetworkManager(numTasks: Int, timeout: Double, useBarrierExecutionMode: Boolean) extends Logging { - // Arrays to store network topology in as it arrives - private val hostAndPorts = ListBuffer[(Socket, String)]() - private val loadOnlyHostAndPorts = ListBuffer[Socket]() // TODO we know this count right? - private val hostToMinPartition = mutable.Map[String, Int]() - private val partitionsByExecutor = mutable.Map[String, List[Int]]() + private final class TaskConnection(val socket: Socket, val message: WorkerMessage) { + // The machine list is comma delimited and every entry is host:port, so an IPv6 host has to be + // bracketed here or peers would read its trailing group as the port. + def networkInfoString: String = WorkerEndpoint.wireString(message.taskHost, message.localListenPort) + } + + // Spark can retry a task report within the same stage attempt. Keeping one connection per + // partition prevents duplicates from satisfying or permanently overshooting the round count. + private val taskConnectionsByPartition = mutable.Map[Int, TaskConnection]() + + // The Spark stage attempt whose topology is currently being collected. A barrier stage restarts + // as a whole, so a higher attempt number means everything gathered so far is obsolete. + private var currentStageAttempt = -1 + private var finishedForCurrentStageAttempt = false + private var expectedTaskCountForCurrentStageAttempt: Option[Int] = None + private var acceptedSocket: Option[Socket] = None + @volatile private var shutdownRequested = false // Concatenate with commas, eg: host1:port1,host2:port2, ... etc // Also make sure the order is deterministic by sorting on minimum partition id - private lazy val networkTopologyAsString: String = { - val hostPortsList = hostAndPorts.map(_._2).sortBy(hostPort => { - val host = hostPort.split(":")(0) - hostToMinPartition(host) - }) - hostPortsList.mkString(",") + private def networkTopologyAsString: String = synchronized { + val connections = taskConnectionsByPartition.values.toSeq + val minPartitionByHost = connections.groupBy(_.message.taskHost).map { case (taskHost, hostConnections) => + taskHost -> hostConnections.map(_.message.partitionId).min + } + connections + .filter(_.message.isForTraining) + .sortBy(connection => + (minPartitionByHost(connection.message.taskHost), connection.message.partitionId)) + .map(_.networkInfoString) + .mkString(",") } // Create a string representing of the partitionsByExecutor map // e.g. executor1=partition1,partition2:executor2=partition3,partition4 - private lazy val partitionsByExecutorAsString: String = { - val executorList = partitionsByExecutor.map { case (executor, partitionList) => - executor + "=" + partitionList.mkString(",") - } - executorList.mkString(":") + private def partitionsByExecutorAsString: String = synchronized { + taskConnectionsByPartition.values + .groupBy(_.message.executorId) + .toSeq + .sortBy(_._1) + .map { case (executorId, connections) => + s"$executorId=${connections.map(_.message.partitionId).toSeq.sorted.mkString(",")}" + } + .mkString(":") } + private val networkCommunicationExecutor: ExecutorService = Executors.newSingleThreadExecutor() + // This will be kicked off at object creation time, and can be waited on by waitForNetworkDone() private val networkCommunicationThread: Future[Unit] = Future { - log.info(s"driver waiting for connections on host: $host and port: $port") + try { + log.info(s"driver waiting for connections on host: $host and port: $port") + if (useBarrierExecutionMode) serveTopologyRoundsUntilShutdown() else serveTopologyRound() + } finally { + // Always release the sockets, including when the topology exchange fails or times out. + closeConnections() + // Release the dedicated thread so repeated fits on a long-lived driver do not leak threads. + networkCommunicationExecutor.shutdown() + } + } (ExecutionContext.fromExecutor(networkCommunicationExecutor)) + + private def serveTopologyRound(): Unit = { waitForAllTasksToReport() // We have all the information now, so report back to workers sendDataToExecutors(networkTopologyAsString, partitionsByExecutorAsString) + } - closeConnections() - } (ExecutionContext.fromExecutor(Executors.newSingleThreadExecutor())) + /** Serves topology rounds until the training job says it is done. + * + * A barrier stage is restarted in its entirety when any of its tasks fails, which is the only way + * a LightGBM network can legitimately re-form. Serving a single round means the restarted stage + * finds a closed port and dies with "connection refused", so keep listening instead. + */ + @tailrec + private def serveTopologyRoundsUntilShutdown(): Unit = { + val keepServing = + try { + serveTopologyRound() + resetRoundState() + true + } catch { + case _: SocketTimeoutException => + // Training can easily outlast the socket timeout, so an idle driver is not an error here. + log.info("driver saw no task connections within the timeout, still listening for a stage restart") + true + case socketFailure: SocketException => + if (!shutdownRequested) throw socketFailure + log.info("driver stopped serving topology rounds") + false + } + if (keepServing) serveTopologyRoundsUntilShutdown() + } def waitForNetworkCommunicationsDone(): Unit = { + // In barrier mode the driver keeps listening for a restarted stage, so it never stops on its own. + if (useBarrierExecutionMode) closeConnections() Await.result(networkCommunicationThread, Duration(timeout, SECONDS)) } @@ -608,89 +570,202 @@ case class NetworkManager(numTasks: Int, } else { log.info(s"driver expecting $numTasks connections...") + // Count the tasks actually recorded rather than the connections seen, so that connections + // discarded as belonging to a superseded stage attempt do not end the round early. @tailrec - def connectToWorkers(numProcessedTasks: Int): Unit = { + def connectToWorkers(): Unit = { handleNextWorkerConnection() - val newNumProcessedTasks = numProcessedTasks + 1 - if (newNumProcessedTasks != numTasks) connectToWorkers(newNumProcessedTasks) + if (reportedTaskCount < numTasks) connectToWorkers() } - connectToWorkers(0) + connectToWorkers() } } /** Handles the connection to a task from the driver. * - * @return Whether the response was a "Finished" response for barrier mode. + * @return Whether the current barrier round has both its Finished marker and every task report. * Always false for non-barrier mode. */ private def handleNextWorkerConnection(): Boolean = { log.info("driver accepting a new connection...") val socket = driverServerSocket.accept() // block until connection is made + if (!registerAcceptedSocket(socket)) { + closeQuietly(socket) + false + } else { + try { + val reader = new BufferedReader(new InputStreamReader(socket.getInputStream)) + val messageStr = reader.readLine() + log.info(s"received worker message string: $messageStr") + processWorkerConnection(socket, WorkerMessage.parse(messageStr)) + } catch { + case failure: Throwable => + closeAcceptedSocket(socket) + throw failure + } + } + } + + private def processWorkerConnection(socket: Socket, message: WorkerMessage): Boolean = synchronized { + if (shutdownRequested) { + closeAcceptedSocket(socket) + false + } else if (message.stageAttemptNumber < currentStageAttempt) { + // A straggler from a stage attempt that Spark has already abandoned. Recording it would put a + // dead host:port into the topology that every surviving task then tries to connect to. + log.info(s"driver ignoring message from superseded stage attempt ${message.stageAttemptNumber}") + closeAcceptedSocket(socket) + false + } else { + startNewStageAttemptIfNeeded(message.stageAttemptNumber) + recordWorkerConnection(socket, message) + } + } - val reader = new BufferedReader(new InputStreamReader(socket.getInputStream)) - val messageStr = reader.readLine() - log.info(s"received worker message string: $messageStr") - val message: TaskMessageInfo = parseWorkerMessage(messageStr) + private def startNewStageAttemptIfNeeded(stageAttemptNumber: Int): Unit = { + if (stageAttemptNumber > currentStageAttempt) { + if (currentStageAttempt >= 0) { + log.info(s"driver starting topology round for stage attempt $stageAttemptNumber, " + + s"discarding the partial topology collected for attempt $currentStageAttempt") + resetRoundState() + } + currentStageAttempt = stageAttemptNumber + } + } + private def recordWorkerConnection(socket: Socket, message: WorkerMessage): Boolean = { if (message.isFinished) { - log.info("driver received all tasks from barrier stage") - true + log.info(s"driver received finished marker from barrier stage for ${message.barrierTaskCount} tasks") + finishedForCurrentStageAttempt = true + message.barrierTaskCount.filter(_ > 0).foreach(count => expectedTaskCountForCurrentStageAttempt = Some(count)) + closeAcceptedSocket(socket) // The finished message uses its own short-lived connection. } else { - message match { - case m if m.isForLoadOnly => - log.info("driver received load-only status from task") - loadOnlyHostAndPorts += socket - case m if m.isForTraining => - val networkInfoString = s"${message.taskHost}:${message.localListenPort}" - log.info(s"driver received socket from task: $networkInfoString") - val socketAndMessage = (socket, networkInfoString) - hostAndPorts += socketAndMessage - case _ => throw new Exception(s"Unknown message type: ${message.toString()}") - } + recordTaskConnection(socket, message) + } - // Update the min partition/executor tracking - if (!hostToMinPartition.contains(message.taskHost) - || hostToMinPartition(message.taskHost) > message.partitionId) { - hostToMinPartition(message.taskHost) = message.partitionId - } + val roundComplete = barrierRoundComplete + if (roundComplete) log.info("driver received all task reports and the finished marker from barrier stage") + roundComplete + } - // Update the tracking of which partitions are on which executor - if (!partitionsByExecutor.contains(message.executorId)) { - partitionsByExecutor(message.executorId) = List { message.partitionId } - } else { - val currentPartitionList = partitionsByExecutor(message.executorId) - partitionsByExecutor(message.executorId) = currentPartitionList :+ message.partitionId - } + /** + * The finished marker only tells the driver that the barrier stage synchronized; the task reports + * can still be sitting in the accept backlog, so completing on the marker alone risks broadcasting + * a partial topology. Wait for as many reports as the barrier stage actually ran, which the marker + * carries. That count is not always numTasks: barrier mode never repartitions upwards, so a user + * who sets numTasks above the input's partition count runs fewer tasks, and waiting for numTasks + * reports would hang until Spark's barrier timeout. + */ + private def barrierRoundComplete: Boolean = { + if (!useBarrierExecutionMode || !finishedForCurrentStageAttempt) { false + } else { + expectedTaskCountForCurrentStageAttempt match { + case Some(expected) => reportedTaskCount >= math.min(expected, numTasks) + // A sender that did not report a count leaves the marker as the only signal available. + case None => true + } + } + } + + private def recordTaskConnection(socket: Socket, message: WorkerMessage): Unit = { + if (message.partitionId < 0 || message.partitionId >= numTasks) { + throw new Exception(s"Unexpected partition id ${message.partitionId}; expected a value in [0, $numTasks)") + } + + val connection = new TaskConnection(socket, message) + message match { + case m if m.isForLoadOnly => + log.info("driver received load-only status from task") + case m if m.isForTraining => + log.info(s"driver received socket from task: ${connection.networkInfoString}") + case _ => throw new Exception(s"Unknown message type: ${message.status}") + } + + val previousConnection = taskConnectionsByPartition.put(message.partitionId, connection) + acceptedSocket = None + previousConnection.foreach { previous => + log.info(s"driver replacing duplicate report for partition ${message.partitionId}") + if (previous.socket ne socket) closeQuietly(previous.socket) } } private def sendDataToExecutors(lightGBMNetworkTopology: String, partitionsByExecutor: String): Unit = { // TODO optimize and not send for bulk mode helpers // Send aggregated network information back to all tasks and helper tasks on executors - val count = hostAndPorts.length + loadOnlyHostAndPorts.length + val sockets = synchronized { + taskConnectionsByPartition.values.map(_.socket).toSeq + } + val count = sockets.length log.info(s"driver writing back network topology to $count connections: $lightGBMNetworkTopology") log.info(s"driver writing back partition topology to $count connections: $partitionsByExecutor") - hostAndPorts.foreach(hostAndPort => { - val writer = new BufferedWriter(new OutputStreamWriter(hostAndPort._1.getOutputStream)) - writer.write(lightGBMNetworkTopology + "\n") - writer.write(partitionsByExecutor + "\n") - writer.flush() - }) - loadOnlyHostAndPorts.foreach(hostAndPort => { - val writer = new BufferedWriter(new OutputStreamWriter(hostAndPort.getOutputStream)) + sockets.foreach(socket => { + val writer = new BufferedWriter(new OutputStreamWriter(socket.getOutputStream)) writer.write(lightGBMNetworkTopology + "\n") writer.write(partitionsByExecutor + "\n") writer.flush() }) } - private def closeConnections(): Unit = { + /** Release every driver-side socket for this training round. + * + * Safe to call more than once and from more than one thread, so both the network thread and the + * training job can guarantee cleanup. Closing the server socket also unblocks a network thread + * that is still parked in accept() waiting for tasks that will never arrive. + */ + private[lightgbm] def closeConnections(): Unit = synchronized { + shutdownRequested = true log.info("driver closing all sockets and server socket") - hostAndPorts.foreach(_._1.close()) - driverServerSocket.close() + acceptedSocket.foreach(closeQuietly) + acceptedSocket = None + closeRoundSockets() + closeQuietly(driverServerSocket) log.info("driver done closing all sockets and server socket") } + + /** Number of tasks whose topology has been recorded for the current stage attempt. */ + private def reportedTaskCount: Int = synchronized { + taskConnectionsByPartition.size + } + + /** Discards everything gathered for a stage attempt so the next one starts from a clean slate. */ + private def resetRoundState(): Unit = synchronized { + closeRoundSockets() + taskConnectionsByPartition.clear() + finishedForCurrentStageAttempt = false + expectedTaskCountForCurrentStageAttempt = None + } + + private def closeRoundSockets(): Unit = synchronized { + taskConnectionsByPartition.values.foreach(connection => closeQuietly(connection.socket)) + } + + /** Tracks the socket between accept() and its transfer into the current round's socket buffers. */ + private def registerAcceptedSocket(socket: Socket): Boolean = synchronized { + if (shutdownRequested) { + false + } else { + acceptedSocket = Some(socket) + true + } + } + + private def closeAcceptedSocket(socket: Socket): Unit = synchronized { + if (acceptedSocket.contains(socket)) { + acceptedSocket = None + closeQuietly(socket) + } + } + + private def closeQuietly(closeable: java.io.Closeable): Unit = { + try { + closeable.close() + } catch { + case NonFatal(closeFailure) => + // One socket refusing to close must not strand the others, especially the server socket. + log.warn("driver could not close a network socket cleanly", closeFailure) + } + } } diff --git a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/NetworkManagerSocketSupport.scala b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/NetworkManagerSocketSupport.scala new file mode 100644 index 00000000000..23256fad3ef --- /dev/null +++ b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/NetworkManagerSocketSupport.scala @@ -0,0 +1,157 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.lightgbm + +import org.slf4j.Logger + +import java.io.IOException +import java.net.{BindException, InetSocketAddress, Socket} +import scala.annotation.tailrec + +private[lightgbm] object NetworkManagerSocketSupport { + private val MaxSocketCloseAttempts = 2 + + def addSuppressed(primaryFailure: Throwable, secondaryFailure: Throwable): Unit = { + if (primaryFailure ne secondaryFailure) primaryFailure.addSuppressed(secondaryFailure) + } + + /** Close a socket with one immediate retry, retaining every observed cleanup failure. */ + def closeSocketWithRetry(socket: Socket): Unit = { + @tailrec + def attemptClose(attemptsRemaining: Int, + firstFailure: Option[IOException]): Option[IOException] = { + if (socket.isClosed || attemptsRemaining == 0) { + firstFailure + } else { + val updatedFailure = try { + socket.close() + firstFailure + } catch { + case failure: IOException => + firstFailure.foreach(existing => addSuppressed(existing, failure)) + firstFailure.orElse(Option(failure)) + } + attemptClose(attemptsRemaining - 1, updatedFailure) + } + } + + val closeFailure = attemptClose(MaxSocketCloseAttempts, None).orElse { + if (socket.isClosed) None else Option(new IOException("Socket remained open after cleanup attempts")) + } + closeFailure.foreach(throw _) + } + + /** Run cleanup without allowing it to replace a failure from the protected operation. + * + * The Throwable catch is deliberately limited to recording and immediately rethrowing the original + * failure; it is not a fallback or recovery boundary. + */ + def withCleanupPreservingPrimary[T](cleanup: => Unit)(operation: => T): T = { + var primaryFailure: Option[Throwable] = None + try { + operation + } catch { + case failure: Throwable => + primaryFailure = Option(failure) + throw failure + } finally { + try { + cleanup + } catch { + case cleanupFailure: Throwable if primaryFailure.isDefined => + addSuppressed(primaryFailure.get, cleanupFailure) + } + } + } + + /** Run cleanup only when the protected operation fails, preserving that primary failure. */ + def withCleanupOnFailurePreservingPrimary[T](cleanup: => Unit)(operation: => T): T = { + var completed = false + withCleanupPreservingPrimary(if (!completed) cleanup) { + val result = operation + completed = true + result + } + } + + /** Reserve the first available port at or above basePort. + * + * Only address-in-use failures advance to another port. Other failures propagate after the candidate + * socket is closed, rather than silently falling back to a different port. + */ + def reserveOpenPort(basePort: Int, log: Logger): Socket = { + reserveOpenPort(basePort, log, () => new Socket()) + } + + def reserveOpenPort(basePort: Int, + log: Logger, + createSocket: () => Socket): Socket = { + validatePort(basePort) + + @tailrec + def reservePort(localListenPort: Int): Socket = { + val bindResult: Either[BindException, Socket] = try { + Right(reserveExactPort(localListenPort, log, createSocket)) + } catch { + // A suppressed exception means candidate cleanup failed, so proceeding would leak a socket. + case contention: BindException if contention.getSuppressed.isEmpty => Left(contention) + } + + bindResult match { + case Right(reservation) => reservation + case Left(_) => + log.warn(s"Could not bind to port $localListenPort...") + val nextPort = localListenPort + 1 + if (nextPort > LightGBMConstants.MaxPort) { + throw new Exception(s"Error: port $basePort out of range, " + + "possibly due to networking or firewall issues") + } + if (nextPort - basePort > 1000) { + throw new Exception("Error: Could not find open port after 1k tries") + } + reservePort(nextPort) + } + } + + reservePort(basePort) + } + + /** Reserve one exact port. Native-init retries cannot change the previously advertised port. */ + def reserveExactPort(localListenPort: Int, log: Logger): Socket = { + reserveExactPort(localListenPort, log, () => new Socket()) + } + + private def reserveExactPort(localListenPort: Int, + log: Logger, + createSocket: () => Socket): Socket = { + validatePort(localListenPort) + val candidate = createSocket() + withCleanupOnFailurePreservingPrimary(closeSocketWithRetry(candidate)) { + candidate.bind(new InetSocketAddress(localListenPort)) + log.info(s"Successfully bound to port $localListenPort") + candidate + } + } + + private def validatePort(port: Int): Unit = { + if (port < 0 || port > LightGBMConstants.MaxPort) { + throw new Exception(s"Error: port $port out of range, possibly due to too many executors or unknown error") + } + } + + /** Keep a training task's port reserved, while releasing helper and failed-task reservations immediately. */ + def withPortReservation(reservation: Socket, + shouldExecuteTraining: Boolean) + (getTopology: Int => NetworkTopologyInfo): NetworkTopologyInfo = { + var retained = false + withCleanupPreservingPrimary(if (!retained) closeSocketWithRetry(reservation)) { + val topology = getTopology(reservation.getLocalPort) + if (shouldExecuteTraining) { + topology.retainPortReservation(reservation) + retained = true + } + topology + } + } +} diff --git a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/NetworkTopologyInfo.scala b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/NetworkTopologyInfo.scala new file mode 100644 index 00000000000..3168a68411b --- /dev/null +++ b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/NetworkTopologyInfo.scala @@ -0,0 +1,102 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.lightgbm + +import java.net.Socket + +case class TaskMessageInfo(status: String, + taskHost: String, + localListenPort: Int, + partitionId: Int, + executorId: String) { + def this(status: String) = this(status, "", -1, -1, "") // Constructor for general messages, not Task-connected + + val isForTraining: Boolean = status == LightGBMConstants.EnabledTask + val isForLoadOnly: Boolean = status == LightGBMConstants.IgnoreStatus + val isFinished: Boolean = status == LightGBMConstants.FinishedStatus + + // Format all the information as a delimited string to send to driver + override def toString: String = s"$status:$taskHost:$localListenPort:$partitionId:$executorId" +} + +case class NetworkTopologyInfo(lightgbmNetworkString: String, + executorPartitionIdList: Array[Int], + localListenPort: Int) { + @transient private var portReservation: Option[Socket] = None + @transient private var networkBridge: Option[LightGBMNetworkBridge] = None + @transient private var advertisedHost: String = "" + + private def currentPortReservation: Option[Socket] = Option(portReservation).flatten + + private def currentNetworkBridge: Option[LightGBMNetworkBridge] = Option(networkBridge).flatten + + /** The endpoint this task advertised to the driver, which is also its entry in the machine list. + * + * This is task local state rather than a constructor field, so the case class keeps the shape + * every existing caller, extractor, and serialized form depends on. + */ + private[lightgbm] def taskHost: String = Option(advertisedHost).getOrElse("") + + private[lightgbm] def withAdvertisedHost(host: String): NetworkTopologyInfo = synchronized { + advertisedHost = Option(host).getOrElse("") + this + } + + private[lightgbm] def hasPortReservation: Boolean = synchronized { + currentPortReservation.nonEmpty + } + + private[lightgbm] def retainPortReservation(reservation: Socket): NetworkTopologyInfo = synchronized { + require(!reservation.isClosed, "Cannot retain a closed port reservation") + require(reservation.isBound, "Cannot retain an unbound port reservation") + require(reservation.getLocalPort == localListenPort, + s"Port reservation ${reservation.getLocalPort} does not match topology port $localListenPort") + require(currentPortReservation.isEmpty, s"Port $localListenPort already has a reservation") + portReservation = Option(reservation) + this + } + + /** Keep an IPv6 transport bridge alive for as long as the native network uses it. */ + private[lightgbm] def retainNetworkBridge(bridge: LightGBMNetworkBridge): NetworkTopologyInfo = synchronized { + require(currentNetworkBridge.isEmpty, "This task already has a LightGBM network bridge") + networkBridge = Option(bridge) + this + } + + private[lightgbm] def hasNetworkBridge: Boolean = synchronized { + currentNetworkBridge.nonEmpty + } + + /** Release the temporary JVM reservation immediately before LightGBM binds the same port. + * + * The operation is idempotent so final cleanup can safely call it after any success or failure path. + */ + private[lightgbm] def releasePortReservation(): Unit = synchronized { + currentPortReservation.foreach { reservation => + try { + NetworkManager.closeSocketWithRetry(reservation) + } finally { + // Keep an open socket reachable for a later final-cleanup attempt. + if (reservation.isClosed) portReservation = None + } + } + } + + /** Tear down the IPv6 transport bridge, if this task needed one. Idempotent. */ + private[lightgbm] def releaseNetworkBridge(): Unit = synchronized { + currentNetworkBridge.foreach { bridge => + try { + bridge.close() + } finally { + networkBridge = None + } + } + } + + /** Release every network resource this task owns, whichever transport it ended up using. */ + private[lightgbm] def releaseNetworkResources(): Unit = { + NetworkManagerSocketSupport.withCleanupPreservingPrimary(releaseNetworkBridge())(releasePortReservation()) + } +} + diff --git a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/WorkerEndpoint.scala b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/WorkerEndpoint.scala new file mode 100644 index 00000000000..31e3dd757bb --- /dev/null +++ b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/WorkerEndpoint.scala @@ -0,0 +1,208 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.lightgbm + +import java.net.{InetAddress, NetworkInterface, SocketException, UnknownHostException} + +private[lightgbm] final case class WorkerEndpoint(host: String, port: Int) { + /** Whether the host is an IPv6 literal, which has to be bracketed before a ':' port separator. */ + def isIpv6Literal: Boolean = WorkerEndpoint.isIpv6Host(host) + + /** The zone identifier of a scoped IPv6 literal, if it carries one. */ + def zoneId: Option[String] = { + val separator = host.indexOf('%') + if (separator < 0) None else Some(host.substring(separator + 1)) + } + + /** Whether the zone identifier is a numeric interface index, which is local to one machine. */ + def hasNumericZone: Boolean = zoneId.exists(zone => zone.nonEmpty && zone.forall(_.isDigit)) + + /** The address without its zone identifier. */ + def address: String = { + val separator = host.indexOf('%') + if (separator < 0) host else host.substring(0, separator) + } + + /** The unambiguous wire form of this endpoint. */ + def wireString: String = WorkerEndpoint.wireString(host, port) +} + +/** Parses the worker addresses exchanged by the LightGBM network handshake. */ +private[lightgbm] object WorkerEndpoint { + private val EndpointPreviewLimit = 200 + private val MinNetworkPort = 1 + + /** Whether a host is an IPv6 literal. Hostnames and IPv4 literals never contain a ':'. */ + def isIpv6Host(host: String): Boolean = Option(host).exists(_.contains(":")) + + /** Replace a numeric IPv6 scope with the interface name it stands for on this machine. + * + * A numeric scope is an interface index, which is only meaningful on the machine that produced + * it, so it must never be published to peers. An interface name survives the trip whenever the + * cluster names its interfaces consistently, which is the only case where a link-local address + * can work at all. Anything else is returned unchanged. + */ + def normalizeHost(host: String): String = { + val endpoint = WorkerEndpoint(Option(host).getOrElse(""), 1) + endpoint.zoneId.filter(zone => zone.nonEmpty && zone.forall(_.isDigit)).flatMap { zone => + try { + Option(NetworkInterface.getByIndex(zone.toInt)).map(named => s"${endpoint.address}%${named.getName}") + } catch { + case _: SocketException => None + case _: IllegalArgumentException => None + } + }.getOrElse(host) + } + + /** Bracket an IPv6 literal so a ':' port separator stays unambiguous. Other hosts are unchanged. */ + def wireHost(host: String): String = + if (isIpv6Host(host) && !host.startsWith("[")) s"[$host]" else host + + /** Render an endpoint in the wire form every LightGBM component parses. + * + * The result is parsed back before it is returned, so a host carrying a control character, a + * delimiter, or an unbalanced bracket fails here instead of corrupting the line protocol or the + * comma-delimited machine list it would have been written into. + */ + def wireString(host: String, port: Int): String = { + val endpoint = s"${wireHost(host)}:$port" + parse(endpoint) + endpoint + } + + /** Parse the first (main) address from a comma-delimited LightGBM machine list. */ + def parseFirst(nodes: String): WorkerEndpoint = { + val nodeList = Option(nodes).getOrElse(invalid(nodes, "network node list is null")) + val firstSeparator = nodeList.indexOf(',') + parse(if (firstSeparator < 0) nodeList else nodeList.substring(0, firstSeparator)) + } + + /** Parse one endpoint without resolving or rewriting its host text. */ + def parse(endpoint: String): WorkerEndpoint = { + val value = Option(endpoint).getOrElse(invalid(endpoint, "endpoint is null")) + if (value.isEmpty) invalid(value, "endpoint is empty") + + val (host, portText, bracketed) = + if (value.startsWith("[")) splitBracketed(value) else splitUnbracketed(value) + validateHost(host, value, bracketed) + WorkerEndpoint(host, parsePort(portText, value)) + } + + private def splitBracketed(endpoint: String): (String, String, Boolean) = { + val closingBracket = endpoint.indexOf(']') + if (closingBracket < 0) invalid(endpoint, "bracketed IPv6 host is missing its closing ']'") + val suffix = endpoint.substring(closingBracket + 1) + if (suffix.isEmpty) invalid(endpoint, "bracketed IPv6 host is missing its port") + if (!suffix.startsWith(":")) { + invalid(endpoint, "bracketed IPv6 host must be followed by a ':' port separator") + } + (endpoint.substring(1, closingBracket), suffix.substring(1), true) + } + + private def splitUnbracketed(endpoint: String): (String, String, Boolean) = { + if (endpoint.exists(character => character == '[' || character == ']')) { + invalid(endpoint, "IPv6 brackets are unbalanced") + } + val portSeparator = endpoint.lastIndexOf(':') + if (portSeparator < 0) invalid(endpoint, "missing ':' port separator") + val host = endpoint.substring(0, portSeparator) + if (host.contains(":") && isValidIpv6Literal(endpoint)) { + invalid(endpoint, "bare IPv6 endpoint is ambiguous; use the unambiguous [IPv6]:port form") + } + (host, endpoint.substring(portSeparator + 1), false) + } + + private def validateHost(host: String, endpoint: String, bracketed: Boolean): Unit = { + if (host.isEmpty) invalid(endpoint, "host is empty") + if (host.exists(isInvalidHostCharacter)) { + invalid(endpoint, "host contains whitespace, control characters, or an endpoint delimiter") + } + + val isIpv6Literal = host.contains(":") + if (bracketed && !isIpv6Literal) { + invalid(endpoint, "brackets are only valid around an IPv6 literal") + } + if (!isIpv6Literal && host.contains("%")) { + invalid(endpoint, "a zone identifier is only valid on an IPv6 literal") + } + if (isIpv6Literal) validateIpv6Literal(host, endpoint) + } + + private def isInvalidHostCharacter(character: Char): Boolean = { + Character.isWhitespace(character) || Character.isISOControl(character) || + character == '[' || character == ']' || character == ',' + } + + private def validateIpv6Literal(host: String, endpoint: String): Unit = { + val addressParts = host.split("%", -1) + if (addressParts.length > 2) invalid(endpoint, "IPv6 zone identifier is malformed") + if (addressParts.length == 2) validateZone(addressParts(1), endpoint) + if (!canParseIpv6Address(addressParts(0))) invalid(endpoint, "host is not a valid IPv6 literal") + } + + private def isValidIpv6Literal(host: String): Boolean = { + val addressParts = host.split("%", -1) + addressParts.length <= 2 && addressParts(0).contains(":") && + (addressParts.length == 1 || isValidZone(addressParts(1))) && canParseIpv6Address(addressParts(0)) + } + + private def canParseIpv6Address(address: String): Boolean = { + try { + // A colon-bearing literal is parsed locally by the JDK and never triggers a hostname lookup. + InetAddress.getByName(address) + true + } catch { + case _: UnknownHostException => false + } + } + + private def validateZone(zone: String, endpoint: String): Unit = { + if (zone.isEmpty) invalid(endpoint, "IPv6 zone identifier is empty") + if (!isValidZone(zone)) { + invalid(endpoint, "IPv6 zone identifier contains whitespace, control characters, or a delimiter") + } + } + + private def isValidZone(zone: String): Boolean = zone.nonEmpty && !zone.exists(isInvalidZoneCharacter) + + private def isInvalidZoneCharacter(character: Char): Boolean = { + Character.isWhitespace(character) || Character.isISOControl(character) || + character == ':' || character == '[' || character == ']' || character == ',' || character == '%' + } + + private def parsePort(portText: String, endpoint: String): Int = { + if (portText.isEmpty) invalid(endpoint, "port is empty") + if (!portText.forall(character => character >= '0' && character <= '9')) { + invalid(endpoint, "port is not a decimal integer") + } + val port = try { + portText.toInt + } catch { + case _: NumberFormatException => invalid(endpoint, "port is too large") + } + if (port < MinNetworkPort || port > LightGBMConstants.MaxPort) { + invalid(endpoint, s"port is outside the valid range $MinNetworkPort-${LightGBMConstants.MaxPort}") + } + port + } + + private[lightgbm] def preview(endpoint: String): String = { + val escaped = Option(endpoint).getOrElse("").flatMap { + case '\r' => "\\r" + case '\n' => "\\n" + case '\t' => "\\t" + case character if Character.isISOControl(character) => f"\\u${character.toInt}%04x" + case character => character.toString + } + val preview = if (escaped.length <= EndpointPreviewLimit) escaped else escaped.take(EndpointPreviewLimit) + "..." + s"'$preview'" + } + + private def invalid(endpoint: String, reason: String): Nothing = { + throw new IllegalArgumentException( + s"Invalid LightGBM worker endpoint ${preview(endpoint)}: $reason. " + + s"Expected hostname:port, IPv4:port, [IPv6]:port, or bare IPv6:port with a decimal port " + + s"between $MinNetworkPort and ${LightGBMConstants.MaxPort}") + } +} diff --git a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/WorkerMessage.scala b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/WorkerMessage.scala new file mode 100644 index 00000000000..996e4d393e5 --- /dev/null +++ b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/WorkerMessage.scala @@ -0,0 +1,104 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.lightgbm + +import java.io.IOException + +import scala.util.Try + +/** + * The line protocol tasks use to report themselves to the driver while the LightGBM network + * topology is being assembled. + * + * A task report is `status:host:port:partitionId:executorId[:stageAttemptNumber]`, and the + * barrier-stage marker is `finished:stageAttemptNumber[:barrierTaskCount]`. The trailing + * fields are parsed defensively so a message written without them is still understood. + */ +private[lightgbm] final case class WorkerMessage(status: String, + taskHost: String, + localListenPort: Int, + partitionId: Int, + executorId: String, + stageAttemptNumber: Int, + barrierTaskCount: Option[Int] = None) { + val isForTraining: Boolean = status == LightGBMConstants.EnabledTask + val isForLoadOnly: Boolean = status == LightGBMConstants.IgnoreStatus + val isFinished: Boolean = status == LightGBMConstants.FinishedStatus + + def toTaskMessage: TaskMessageInfo = + TaskMessageInfo(status, taskHost, localListenPort, partitionId, executorId) +} + +private[lightgbm] object WorkerMessage { + private val TaskMessageFieldCount = 5 + private val TaskMessageFieldCountWithStageAttempt = 6 + + def parse(message: String): WorkerMessage = { + if (message == null) { + throw new IOException("Worker closed the connection before sending a status message") + } + val components = message.split(":", -1) + val status = components(0) + + if (status == LightGBMConstants.FinishedStatus) { + WorkerMessage(status, "", -1, -1, "", parseIntOrDefault(components, 1, 0), + parseOptionalInt(components, 2)) + } else { + val currentMessage = parseTaskMessage(components, hasStageAttempt = true) + val legacyMessage = parseTaskMessage(components, hasStageAttempt = false) + (currentMessage, legacyMessage) match { + case (Some(current), Some(_)) if components.length > 1 && components(1).startsWith("[") => current + // An unbracketed IPv6 message that fits both layouts predates the stage-attempt suffix. + case (Some(_), Some(legacy)) => legacy + case (Some(current), None) => current + case (None, Some(legacy)) => legacy + case _ => throw new IllegalArgumentException( + s"Unexpected worker message: expected status:host:port:partitionId:executorId[:stageAttemptNumber], " + + s"but received ${WorkerEndpoint.preview(message)}") + } + } + } + + private def parseTaskMessage(components: Array[String], hasStageAttempt: Boolean): Option[WorkerMessage] = { + val suffixFieldCount = if (hasStageAttempt) { + TaskMessageFieldCountWithStageAttempt - 2 + } else { + TaskMessageFieldCount - 2 + } + val portIndex = components.length - suffixFieldCount + if (portIndex <= 1) { + None + } else { + val host = components.slice(1, portIndex).mkString(":") + val portText = components(portIndex) + val partitionText = components(portIndex + 1) + val executorId = components(portIndex + 2) + val stageAttemptText = if (hasStageAttempt) Some(components(portIndex + 3)) else None + + val endpointText = if (host.contains(":") && !host.startsWith("[")) s"[$host]:$portText" else s"$host:$portText" + for { + endpoint <- Try(WorkerEndpoint.parse(endpointText)).toOption + partitionId <- Try(partitionText.toInt).toOption + stageAttemptNumber <- stageAttemptText.map(value => Try(value.toInt).toOption).getOrElse(Some(0)) + if executorId.nonEmpty && stageAttemptNumber >= 0 + } yield WorkerMessage(components(0), endpoint.host, endpoint.port, partitionId, executorId, stageAttemptNumber) + } + } + + def format(message: TaskMessageInfo, stageAttemptNumber: Int): String = { + // Validated bracketing keeps an IPv6 host unambiguous and keeps a host that carries a control + // character or a delimiter out of the line protocol entirely. + val endpoint = WorkerEndpoint.wireString(message.taskHost, message.localListenPort) + s"${message.status}:$endpoint:${message.partitionId}:${message.executorId}:" + stageAttemptNumber + } + + def formatFinished(stageAttemptNumber: Int, barrierTaskCount: Int): String = + s"${LightGBMConstants.FinishedStatus}:$stageAttemptNumber:$barrierTaskCount" + + private def parseIntOrDefault(components: Array[String], index: Int, default: Int): Int = + if (components.length > index && components(index).nonEmpty) components(index).toInt else default + + private def parseOptionalInt(components: Array[String], index: Int): Option[Int] = + if (components.length > index && components(index).nonEmpty) Some(components(index).toInt) else None +} diff --git a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/booster/LightGBMBooster.scala b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/booster/LightGBMBooster.scala index 27afab404ed..597b85b0c04 100644 --- a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/booster/LightGBMBooster.scala +++ b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/booster/LightGBMBooster.scala @@ -238,8 +238,9 @@ class LightGBMBooster(val trainDataset: Option[LightGBMDataset] = None, new BoosterHandler(modelStr.get) } else { val boosterOutPtr = lightgbmlib.voidpp_handle() - LightGBMUtils.validate(lightgbmlib.LGBM_BoosterCreate(trainDataset.map(_.datasetPtr).get, - parameters.get, boosterOutPtr), "Booster") + val trainingParameters = parameters.get + LightGBMUtils.validateBooster(lightgbmlib.LGBM_BoosterCreate(trainDataset.map(_.datasetPtr).get, + trainingParameters, boosterOutPtr), trainingParameters) new BoosterHandler(lightgbmlib.voidpp_value(boosterOutPtr)) } } diff --git a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/dataset/LightGBMDataset.scala b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/dataset/LightGBMDataset.scala index b841f2954c4..14b96de87a1 100644 --- a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/dataset/LightGBMDataset.scala +++ b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/dataset/LightGBMDataset.scala @@ -7,6 +7,7 @@ import com.microsoft.azure.synapse.ml.lightgbm.LightGBMUtils import com.microsoft.azure.synapse.ml.lightgbm.dataset.DatasetUtils.countCardinality import com.microsoft.lightgbm.SwigPtrWrapper import com.microsoft.ml.lightgbm._ +import org.slf4j.{Logger, LoggerFactory} import scala.reflect.ClassTag @@ -179,8 +180,17 @@ class LightGBMDataset(val datasetPtr: SWIGTYPE_p_void) extends AutoCloseable { // Add in slot names if they exist featureNamesOpt.foreach { featureNamesArray => if (featureNamesArray.nonEmpty) { - LightGBMUtils.validate(lightgbmlib.LGBM_DatasetSetFeatureNames(datasetPtr, featureNamesArray, numCols), - "Dataset set feature names") + // LGBM_DatasetSetFeatureNames reads numCols entries from the array, so a shorter array + // is an out-of-bounds native read. slotNames is user-supplied and unvalidated, so guard + // every dataset-naming path here rather than at individual call sites. LightGBM falls + // back to its own generated names when naming is skipped. + if (featureNamesArray.length != numCols) { + LightGBMDataset.Log.warn(s"Skipping feature names: got ${featureNamesArray.length} names " + + s"for $numCols feature columns.") + } else { + LightGBMUtils.validate(lightgbmlib.LGBM_DatasetSetFeatureNames(datasetPtr, featureNamesArray, numCols), + "Dataset set feature names") + } } } this @@ -191,3 +201,7 @@ class LightGBMDataset(val datasetPtr: SWIGTYPE_p_void) extends AutoCloseable { LightGBMUtils.validate(lightgbmlib.LGBM_DatasetFree(datasetPtr), "Free Dataset") } } + +object LightGBMDataset { + private val Log: Logger = LoggerFactory.getLogger(classOf[LightGBMDataset]) +} diff --git a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/dataset/ReferenceDatasetUtils.scala b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/dataset/ReferenceDatasetUtils.scala index 63743835f93..80b2e1585bf 100644 --- a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/dataset/ReferenceDatasetUtils.scala +++ b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/dataset/ReferenceDatasetUtils.scala @@ -16,25 +16,42 @@ object ReferenceDatasetUtils { numRows: Long, numCols: Int, sampledRowData: Array[Row], + featureNames: Option[Array[String]], measures: InstrumentationMeasures, log: Logger): Array[Byte] = { log.info(s"Creating reference training dataset with ${sampledRowData.length} samples and config: $datasetParams") - // Pre-create allocated native pointers so it's easy to clean them up - val datasetVoidPtr = lightgbmlib.voidpp_handle() val lenPtr = lightgbmlib.new_intp() val bufferHandlePtr = lightgbmlib.voidpp_handle() - val sampledData = SampledData(sampledRowData.length, numCols) + try { - // create properly formatted sampled data measures.markSamplingStart() sampledRowData.zipWithIndex.foreach({case (row, index) => sampledData.pushRow(row, index, featuresCol)}) measures.markSamplingStop() - // Create dataset from samples - // 1. Generate the dataset for features - val datasetVoidPtr = lightgbmlib.voidpp_handle() + val datasetHandle = createDatasetFromSamples(sampledData, numCols, numRows, datasetParams) + try { + setFeatureNamesIfProvided(datasetHandle, featureNames, numCols, log) + serializeReference(datasetHandle, bufferHandlePtr, lenPtr, log) + } finally { + // Free unconditionally so a failure while naming or serializing cannot leak the native + // Dataset. Deliberately not validated: throwing here would mask the original failure. + lightgbmlib.LGBM_DatasetFree(datasetHandle) + } + } finally { + sampledData.delete() + lightgbmlib.delete_voidpp(bufferHandlePtr) + lightgbmlib.delete_intp(lenPtr) + } + } + + private def createDatasetFromSamples(sampledData: SampledData, + numCols: Int, + numRows: Long, + datasetParams: String): SWIGTYPE_p_void = { + val datasetVoidPtr = lightgbmlib.voidpp_handle() + try { LightGBMUtils.validate(lightgbmlib.LGBM_DatasetCreateFromSampledColumn( sampledData.getSampleData, sampledData.getSampleIndices, @@ -45,31 +62,43 @@ object ReferenceDatasetUtils { numRows, datasetParams, datasetVoidPtr), "Dataset create from samples") - - - // 2. Serialize the raw dataset to a native buffer - val datasetHandle: SWIGTYPE_p_void = lightgbmlib.voidpp_value(datasetVoidPtr) - LightGBMUtils.validate(lightgbmlib.LGBM_DatasetSerializeReferenceToBinary( - datasetHandle, - bufferHandlePtr, - lenPtr), "Serialize ref") - val bufferLen: Int = lightgbmlib.intp_value(lenPtr) - log.info(s"Created serialized reference dataset of length $bufferLen") - - // The dataset is now serialized to a buffer, so we don't need original - LightGBMUtils.validate(lightgbmlib.LGBM_DatasetFree(datasetHandle), "Free Dataset") - - // This will also free the buffer - toByteArray(bufferHandlePtr, bufferLen) - } - finally { - sampledData.delete() + lightgbmlib.voidpp_value(datasetVoidPtr) + } finally { + // Frees the void** container only. The Dataset it points at is freed by LGBM_DatasetFree. lightgbmlib.delete_voidpp(datasetVoidPtr) - lightgbmlib.delete_voidpp(bufferHandlePtr) - lightgbmlib.delete_intp(lenPtr) } } + private def setFeatureNamesIfProvided(datasetHandle: SWIGTYPE_p_void, + featureNames: Option[Array[String]], + numCols: Int, + log: Logger): Unit = { + featureNames.foreach { names => + if (names.length != numCols) { + // LGBM_DatasetSetFeatureNames reads numCols entries from the array, so a shorter array + // would be an out-of-bounds native read. Skip naming rather than risk it; LightGBM then + // falls back to its own generated names, which is the behavior prior to this feature. + log.warn(s"Skipping feature names on reference dataset: got ${names.length} names " + + s"for $numCols feature columns.") + } else if (names.nonEmpty) { + log.info(s"Setting ${names.length} feature names on reference dataset") + LightGBMUtils.validate(lightgbmlib.LGBM_DatasetSetFeatureNames(datasetHandle, names, numCols), + "Dataset set feature names") + } + } + } + + private def serializeReference(datasetHandle: SWIGTYPE_p_void, + bufferHandlePtr: SWIGTYPE_p_p_void, + lenPtr: SWIGTYPE_p_int, + log: Logger): Array[Byte] = { + LightGBMUtils.validate(lightgbmlib.LGBM_DatasetSerializeReferenceToBinary( + datasetHandle, bufferHandlePtr, lenPtr), "Serialize ref") + val bufferLen: Int = lightgbmlib.intp_value(lenPtr) + log.info(s"Created serialized reference dataset of length $bufferLen") + toByteArray(bufferHandlePtr, bufferLen) + } + def getInitializedReferenceDataset(ctx: PartitionTaskContext): LightGBMDataset = { // The definition is broadcast from Spark, so retrieve it val serializedDataset: Array[Byte] = ctx.trainingCtx.serializedReferenceDataset.get diff --git a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/params/LightGBMParams.scala b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/params/LightGBMParams.scala index 57d40e89c17..d260da0c518 100644 --- a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/params/LightGBMParams.scala +++ b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/params/LightGBMParams.scala @@ -143,6 +143,23 @@ trait LightGBMExecutionParams extends Wrappable { def getMatrixType: String = $(matrixType) def setMatrixType(value: String): this.type = set(matrixType, value) + val deviceType = new Param[String](this, "deviceType", + "Device for tree learning: cpu or gpu (OpenCL). Default is cpu. SynapseML's bundled native LightGBM library " + + "is CPU-only; gpu requires compatible custom native libraries on java.library.path for every driver and " + + "executor. The LightGBM 3.3.510 CUDA backend is incompatible with SynapseML streaming Datasets.", + ParamValidators.inArray(Array(LightGBMConstants.CPUDeviceType, + LightGBMConstants.GPUDeviceType))) + setDefault(deviceType -> LightGBMConstants.CPUDeviceType) + def getDeviceType: String = $(deviceType) + def setDeviceType(value: String): this.type = { + if (value == LightGBMConstants.CUDADeviceType) { + throw new IllegalArgumentException( + "deviceType=cuda is not supported by SynapseML's LightGBM 3.3.510 integration because it can crash " + + "Spark executors. Use deviceType=gpu with an OpenCL-enabled custom native library.") + } + set(deviceType, value) + } + val numThreads = new IntParam(this, "numThreads", "Number of threads per executor for LightGBM. For the best speed, set this to the number of real CPU cores.") setDefault(numThreads -> 0) diff --git a/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/dataset/VerifyDatasetUtils.scala b/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/dataset/VerifyDatasetUtils.scala new file mode 100644 index 00000000000..8cf130585da --- /dev/null +++ b/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/dataset/VerifyDatasetUtils.scala @@ -0,0 +1,74 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.lightgbm.dataset + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase +import org.apache.spark.sql.types._ + +// scalastyle:off magic.number +class VerifyDatasetUtils extends TestBase { + + test("countCardinality with all same values") { + val result = DatasetUtils.countCardinality(Seq(1, 1, 1)) + assert(result === Array(3)) + } + + test("countCardinality with all different values") { + val result = DatasetUtils.countCardinality(Seq(1, 2, 3)) + assert(result === Array(1, 1, 1)) + } + + test("countCardinality with grouped values") { + val result = DatasetUtils.countCardinality(Seq(1, 1, 2, 2, 2, 3)) + assert(result === Array(2, 3, 1)) + } + + test("countCardinality with empty sequence") { + // Known quirk, asserted so a future change is deliberate: an empty partition folds to the + // initial triplet and reports one ranking group of zero rows rather than no groups at all. + val result = DatasetUtils.countCardinality(Seq.empty[Int]) + assert(result === Array(0)) + } + + test("getArrayType with sparse returns true") { + val iter = Iterator.empty + val (_, isSparse) = DatasetUtils.getArrayType(iter, "sparse", "features") + assert(isSparse) + } + + test("getArrayType with dense returns false") { + val iter = Iterator.empty + val (_, isSparse) = DatasetUtils.getArrayType(iter, "dense", "features") + assert(!isSparse) + } + + test("getArrayType with invalid type throws") { + intercept[Exception] { + DatasetUtils.getArrayType(Iterator.empty, "invalid", "features") + } + } + + test("validateGroupColumn throws for unsupported types") { + val schema = StructType(Seq(StructField("g", DoubleType))) + intercept[IllegalArgumentException] { + DatasetUtils.validateGroupColumn("g", schema) + } + } + + test("validateGroupColumn passes for IntegerType") { + val schema = StructType(Seq(StructField("g", IntegerType))) + DatasetUtils.validateGroupColumn("g", schema) + } + + test("validateGroupColumn passes for LongType") { + val schema = StructType(Seq(StructField("g", LongType))) + DatasetUtils.validateGroupColumn("g", schema) + } + + test("validateGroupColumn passes for StringType") { + val schema = StructType(Seq(StructField("g", StringType))) + DatasetUtils.validateGroupColumn("g", schema) + } +} +// scalastyle:on magic.number diff --git a/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/split1/BarrierNetworkRecoverySuite.scala b/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/split1/BarrierNetworkRecoverySuite.scala new file mode 100644 index 00000000000..825d8d49bfc --- /dev/null +++ b/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/split1/BarrierNetworkRecoverySuite.scala @@ -0,0 +1,372 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.lightgbm.split1 + +import com.microsoft.azure.synapse.ml.lightgbm.{LightGBMConstants, NetworkManager} +import org.scalatest.funsuite.AnyFunSuite + +import java.io.{BufferedReader, BufferedWriter, IOException, InputStreamReader, OutputStreamWriter} +import java.net.{InetSocketAddress, ServerSocket, Socket, SocketException, SocketTimeoutException} + +/** Covers recovery of the driver topology exchange when Spark restarts a barrier stage. + * + * A barrier stage is restarted in its entirety when any of its tasks fails, so it is the only case + * where a LightGBM network can legitimately be re-formed. The driver has to serve a second topology + * round for the new stage attempt, and must not mix it with the topology of the abandoned one. + */ +class BarrierNetworkRecoverySuite extends AnyFunSuite { + + private val timeout = 30.0 + private val socketTimeoutMillis = 30000 + private val noResponseTimeoutMillis = 250 + private val host = "127.0.0.1" + + private class FakeBarrierTask(port: Int, + partitionId: Int, + stageAttempt: Int, + loadOnly: Boolean = false, + listenPortOffset: Int = 0, + executorId: String = "") extends AutoCloseable { + private val socket = { + val result = new Socket() + try { + result.connect(new InetSocketAddress(host, port), socketTimeoutMillis) + result.setSoTimeout(socketTimeoutMillis) + result + } catch { + case failure: Throwable => + try result.close() + catch { + case closeFailure: IOException => failure.addSuppressed(closeFailure) + } + throw failure + } + } + private val reader = new BufferedReader(new InputStreamReader(socket.getInputStream)) + private val writer = new BufferedWriter(new OutputStreamWriter(socket.getOutputStream)) + + val listenPort: Int = + LightGBMConstants.DefaultLocalListenPort + stageAttempt * 100 + partitionId + listenPortOffset + def address: String = s"$host:$listenPort" + + def report(): Unit = { + val status = if (loadOnly) LightGBMConstants.IgnoreStatus else LightGBMConstants.EnabledTask + val reportedExecutorId = if (executorId.isEmpty) s"executor$partitionId" else executorId + writer.write( + s"$status:$host:$listenPort:$partitionId:$reportedExecutorId:$stageAttempt\n") + writer.flush() + } + + def readTopology(): String = reader.readLine() + + def readExecutorTopology(): String = reader.readLine() + + def assertNoTopologyYet(): Unit = { + socket.setSoTimeout(noResponseTimeoutMillis) + try { + intercept[SocketTimeoutException](reader.readLine()) + } finally { + socket.setSoTimeout(socketTimeoutMillis) + } + } + + def isClosedByDriver: Boolean = { + try { + reader.readLine() == null + } catch { + case _: SocketException => true + } + } + + override def close(): Unit = socket.close() + } + + /** Partition 0 signals the end of a barrier round over its own short-lived connection. */ + private def sendFinished(port: Int, stageAttempt: Int, barrierTaskCount: Option[Int] = Some(2)): Unit = { + val socket = new Socket() + try { + socket.connect(new InetSocketAddress(host, port), socketTimeoutMillis) + socket.setSoTimeout(socketTimeoutMillis) + val writer = new BufferedWriter(new OutputStreamWriter(socket.getOutputStream)) + val countSuffix = barrierTaskCount.map(count => s":$count").getOrElse("") + writer.write(s"${LightGBMConstants.FinishedStatus}:$stageAttempt$countSuffix\n") + writer.flush() + socket.shutdownOutput() + val reader = new BufferedReader(new InputStreamReader(socket.getInputStream)) + try { + assert(reader.readLine() == null, "The driver unexpectedly replied to a Finished marker") + } catch { + case _: SocketException => () // A reset also proves that the driver processed and rejected the marker socket. + } + } finally { + socket.close() + } + } + + private def newBarrierManager(numTasks: Int): (NetworkManager, Int) = { + val serverSocket = new ServerSocket(0) + serverSocket.setSoTimeout(socketTimeoutMillis) + val port = serverSocket.getLocalPort + (NetworkManager(numTasks, serverSocket, host, port, timeout, useBarrierExecutionMode = true), port) + } + + private def closeAll(manager: NetworkManager, tasks: Seq[FakeBarrierTask]): Unit = { + manager.closeConnections() + try { + tasks.foreach { task => + try task.close() + catch { + case _: IOException => () + } + } + } finally { + manager.waitForNetworkCommunicationsDone() + } + } + + private def partitionIds(executorTopology: String): Seq[Int] = { + executorTopology.split(":").flatMap { executorEntry => + executorEntry.split("=")(1).split(",").map(_.toInt) + }.toSeq + } + + test("A restarted barrier stage gets a fresh topology round instead of a closed port") { + val (manager, port) = newBarrierManager(numTasks = 2) + var tasks = Seq.empty[FakeBarrierTask] + try { + // First stage attempt reports but never reaches the barrier, so the round never completes. + val abandoned = (0 until 2).map { partitionId => + val task = new FakeBarrierTask(port, partitionId, stageAttempt = 0) + tasks :+= task + task + } + abandoned.foreach(_.report()) + + // Spark restarts the whole stage. Every task reconnects to the same driver endpoint. + val restarted = (0 until 2).map { partitionId => + val task = new FakeBarrierTask(port, partitionId, stageAttempt = 1) + tasks :+= task + task + } + restarted.foreach(_.report()) + sendFinished(port, stageAttempt = 1) + + val topologies = restarted.map(_.readTopology()) + assert(topologies.forall(_ != null), "The restarted stage attempt never received a topology") + + val expected = restarted.map(_.address).toSet + topologies.foreach { topology => + assert(topology.split(",").toSet == expected, + s"Topology '$topology' should contain only the restarted stage attempt, expected $expected") + } + + } finally { + closeAll(manager, tasks) + } + } + + test("A straggler from a superseded stage attempt is kept out of the topology") { + val (manager, port) = newBarrierManager(numTasks = 2) + var tasks = Seq.empty[FakeBarrierTask] + try { + val abandoned = new FakeBarrierTask(port, 0, stageAttempt = 0) + tasks :+= abandoned + abandoned.report() + + val restarted = (0 until 2).map { partitionId => + val task = new FakeBarrierTask(port, partitionId, stageAttempt = 1) + tasks :+= task + task + } + restarted.foreach(_.report()) + + // A task from the abandoned attempt reports late. Its host:port is already dead. + val straggler = new FakeBarrierTask(port, 1, stageAttempt = 0) + tasks :+= straggler + straggler.report() + assert(straggler.isClosedByDriver, "The straggler should have been rejected by the driver") + + sendFinished(port, stageAttempt = 1) + + val expected = restarted.map(_.address).toSet + restarted.foreach { task => + assert(task.readTopology().split(",").toSet == expected, + s"The straggler at ${straggler.address} should not appear in the topology") + } + } finally { + closeAll(manager, tasks) + } + } + + test("A Finished marker before task reports waits for every report in that stage attempt") { + val (manager, port) = newBarrierManager(numTasks = 2) + var tasks = Seq.empty[FakeBarrierTask] + try { + sendFinished(port, stageAttempt = 0) + + val current = (0 until 2).map { partitionId => + val task = new FakeBarrierTask(port, partitionId, stageAttempt = 0) + tasks :+= task + task + } + current.foreach(_.report()) + + val expected = current.map(_.address).toSet + current.foreach(task => assert(task.readTopology().split(",").toSet == expected)) + } finally { + closeAll(manager, tasks) + } + } + + test("Task reports before the Finished marker do not complete a barrier round early") { + val (manager, port) = newBarrierManager(numTasks = 2) + var tasks = Seq.empty[FakeBarrierTask] + try { + val current = (0 until 2).map { partitionId => + val task = new FakeBarrierTask(port, partitionId, stageAttempt = 0) + tasks :+= task + task + } + current.foreach(_.report()) + current.head.assertNoTopologyYet() + + sendFinished(port, stageAttempt = 0) + + val expected = current.map(_.address).toSet + current.foreach(task => assert(task.readTopology().split(",").toSet == expected)) + } finally { + closeAll(manager, tasks) + } + } + + test("A barrier stage smaller than numTasks completes instead of hanging") { + // Barrier mode never repartitions upwards, so setNumTasks above the input's partition + // count leaves the stage running fewer tasks than numTasks. Waiting for numTasks reports + // would block until Spark's barrier timeout (365 days by default). + val (manager, port) = newBarrierManager(numTasks = 4) + var tasks = Seq.empty[FakeBarrierTask] + try { + val current = (0 until 2).map { partitionId => + val task = new FakeBarrierTask(port, partitionId, stageAttempt = 0) + tasks :+= task + task + } + current.foreach(_.report()) + current.head.assertNoTopologyYet() + + sendFinished(port, stageAttempt = 0, barrierTaskCount = Some(2)) + + val expected = current.map(_.address).toSet + current.foreach(task => assert(task.readTopology().split(",").toSet == expected)) + } finally { + closeAll(manager, tasks) + } + } + + test("A Finished marker without a task count still completes the round") { + // Defensive: the marker is the only completion signal available when no count is reported. + val (manager, port) = newBarrierManager(numTasks = 2) + var tasks = Seq.empty[FakeBarrierTask] + try { + val current = (0 until 2).map { partitionId => + val task = new FakeBarrierTask(port, partitionId, stageAttempt = 0) + tasks :+= task + task + } + current.foreach(_.report()) + + sendFinished(port, stageAttempt = 0, barrierTaskCount = None) + + val expected = current.map(_.address).toSet + current.foreach(task => assert(task.readTopology().split(",").toSet == expected)) + } finally { + closeAll(manager, tasks) + } + } + + test("A duplicate partition report before Finished replaces the old socket without changing the count") { + val (manager, port) = newBarrierManager(numTasks = 2) + var tasks = Seq.empty[FakeBarrierTask] + try { + val original = new FakeBarrierTask( + port, partitionId = 0, stageAttempt = 0, loadOnly = true, executorId = "old-executor") + val replacement = new FakeBarrierTask( + port, partitionId = 0, stageAttempt = 0, listenPortOffset = 1000, executorId = "new-executor") + val secondPartition = new FakeBarrierTask(port, partitionId = 1, stageAttempt = 0) + tasks = Seq(original, replacement, secondPartition) + + original.report() + replacement.report() + assert(original.isClosedByDriver, "The superseded partition report socket was not closed") + secondPartition.report() + sendFinished(port, stageAttempt = 0) + + val expectedNetwork = Set(replacement.address, secondPartition.address) + Seq(replacement, secondPartition).foreach { task => + val networkNodes = task.readTopology().split(",").toSeq + assert(networkNodes.size == expectedNetwork.size) + assert(networkNodes.toSet == expectedNetwork) + val executorTopology = task.readExecutorTopology() + assert(partitionIds(executorTopology).sorted == Seq(0, 1)) + assert(!executorTopology.contains("old-executor")) + assert(executorTopology.contains("new-executor=0")) + } + } finally { + closeAll(manager, tasks) + } + } + + test("A duplicate partition report after Finished still waits for every unique partition") { + val (manager, port) = newBarrierManager(numTasks = 2) + var tasks = Seq.empty[FakeBarrierTask] + try { + sendFinished(port, stageAttempt = 0) + + val original = new FakeBarrierTask( + port, partitionId = 0, stageAttempt = 0, executorId = "old-executor") + val replacement = new FakeBarrierTask( + port, partitionId = 0, stageAttempt = 0, loadOnly = true, + listenPortOffset = 1000, executorId = "new-executor") + val secondPartition = new FakeBarrierTask(port, partitionId = 1, stageAttempt = 0) + tasks = Seq(original, replacement, secondPartition) + + original.report() + replacement.report() + assert(original.isClosedByDriver, "The superseded partition report socket was not closed") + secondPartition.report() + + Seq(replacement, secondPartition).foreach { task => + assert(task.readTopology() == secondPartition.address) + val executorTopology = task.readExecutorTopology() + assert(partitionIds(executorTopology).sorted == Seq(0, 1)) + assert(!executorTopology.contains("old-executor")) + assert(executorTopology.contains("new-executor=0")) + } + } finally { + closeAll(manager, tasks) + } + } + + test("A stale Finished marker cannot complete the current stage attempt") { + val (manager, port) = newBarrierManager(numTasks = 2) + var tasks = Seq.empty[FakeBarrierTask] + try { + val current = (0 until 2).map { partitionId => + val task = new FakeBarrierTask(port, partitionId, stageAttempt = 1) + tasks :+= task + task + } + current.foreach(_.report()) + + sendFinished(port, stageAttempt = 0) + current.head.assertNoTopologyYet() + + sendFinished(port, stageAttempt = 1) + val expected = current.map(_.address).toSet + current.foreach(task => assert(task.readTopology().split(",").toSet == expected)) + } finally { + closeAll(manager, tasks) + } + } +} diff --git a/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/split1/DriverSocketRetryE2ESuite.scala b/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/split1/DriverSocketRetryE2ESuite.scala new file mode 100644 index 00000000000..6944b07255e --- /dev/null +++ b/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/split1/DriverSocketRetryE2ESuite.scala @@ -0,0 +1,89 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.lightgbm.split1 + +import com.microsoft.azure.synapse.ml.lightgbm.params.BaseTrainParams +import com.microsoft.azure.synapse.ml.lightgbm.{ColumnParams, LightGBMClassifier, LightGBMDelegate} +import org.apache.spark.SparkException +import org.apache.spark.TaskContext +import org.apache.spark.ml.linalg.{SQLDataTypes, Vectors} +import org.apache.spark.sql.types.{DoubleType, StructField, StructType} +import org.apache.spark.sql.{DataFrame, Row} +import org.slf4j.Logger + +import java.io.{PrintWriter, StringWriter} + +object FailFirstAttemptDelegate { + val Sentinel: String = "INJECTED_TASK_FAILURE_AFTER_TOPOLOGY_HANDSHAKE" +} + +/** Fails the first attempt of a training task, after it has already completed the driver + * topology handshake. Later attempts do nothing, so the job could succeed on retry. + */ +class FailFirstAttemptDelegate extends LightGBMDelegate { + override def beforeGenerateTrainDataset(batchIndex: Int, + partitionId: Int, + columnParams: ColumnParams, + schema: StructType, + log: Logger, + trainParams: BaseTrainParams): Unit = { + val context = TaskContext.get() + if (context != null && context.attemptNumber() == 0) { + throw new RuntimeException(FailFirstAttemptDelegate.Sentinel) + } + } +} + +/** End-to-end coverage for distributed LightGBM training when a task fails after it has joined the + * LightGBM network. Every Spark retry of that task is refused by the driver, which used to replace + * the real failure with a bare "java.net.ConnectException: Connection refused". + */ +class DriverSocketRetryE2ESuite extends LightGBMTestUtils { + + private def makeDataframe: DataFrame = { + val schema = StructType(Seq( + StructField(labelCol, DoubleType), + StructField(featuresCol, SQLDataTypes.VectorType))) + val rows = (0 until 400).map(i => + Row(if (i % 2 == 0) 0.0 else 1.0, Vectors.dense((i % 13).toDouble, (i % 7).toDouble))) + spark.createDataFrame(spark.sparkContext.parallelize(rows, 2), schema) + } + + private def stackTraceOf(throwable: Throwable): String = { + val writer = new StringWriter() + throwable.printStackTrace(new PrintWriter(writer)) + writer.toString + } + + test("A task retry that cannot rejoin the LightGBM network reports why instead of Connection refused") { + // Spark must be allowed to retry tasks for the failure cascade to appear. + sparkProvider.resetSparkSession(numRetries = 4, numCores = Some(2)) + try { + val classifier = new LightGBMClassifier() + .setLabelCol(labelCol) + .setFeaturesCol(featuresCol) + .setNumLeaves(5) + .setNumIterations(5) + .setDefaultListenPort(getAndIncrementPort()) + .setDelegate(new FailFirstAttemptDelegate()) + + val thrown = intercept[SparkException] { + classifier.fit(makeDataframe) + } + val trace = stackTraceOf(thrown) + + // Spark only surfaces the last attempt, and that attempt can never reach the driver again. + assert(trace.contains("ConnectException"), + s"Expected the retries to fail against the closed driver endpoint, got:\n${trace.take(4000)}") + + // The reported message must explain the cascade and point at the attempt that really failed. + assert(thrown.getMessage.contains("retry attempt"), + s"Expected the failure to identify itself as a retry, got:\n${thrown.getMessage.take(2000)}") + assert(thrown.getMessage.contains("consequence of an earlier failure"), + s"Expected the failure to point at the original error, got:\n${thrown.getMessage.take(2000)}") + } finally { + sparkProvider.resetSparkSession() + } + } +} diff --git a/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/split1/DriverSocketRetrySuite.scala b/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/split1/DriverSocketRetrySuite.scala new file mode 100644 index 00000000000..87315c9e6cd --- /dev/null +++ b/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/split1/DriverSocketRetrySuite.scala @@ -0,0 +1,432 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.lightgbm.split1 + +import com.microsoft.azure.synapse.ml.lightgbm.{LightGBMConstants, NetworkManager, TaskMessageInfo, WorkerMessage} +import org.scalatest.funsuite.AnyFunSuite + +import java.io.{BufferedReader, BufferedWriter, IOException, InputStreamReader, OutputStreamWriter} +import java.net.{ConnectException, InetSocketAddress, ServerSocket, Socket, SocketException, SocketTimeoutException} +import java.util.concurrent.{CountDownLatch, TimeUnit} +import java.util.concurrent.atomic.AtomicInteger +import scala.collection.mutable.ListBuffer + +/** Covers the driver topology socket lifecycle behind repeated + * "java.net.ConnectException: Connection refused" failures in distributed LightGBM training. + */ +class DriverSocketRetrySuite extends AnyFunSuite { + + private val timeout = 30.0 + private val socketTimeoutMillis = 30000 + private val host = "127.0.0.1" + + private class FakeTask(host: String, + port: Int, + partitionId: Int, + loadOnly: Boolean = false, + listenPortOffset: Int = 0, + executorId: String = "") + extends AutoCloseable { + private val socket = { + val result = new Socket() + try { + result.connect(new InetSocketAddress(host, port), socketTimeoutMillis) + result.setSoTimeout(socketTimeoutMillis) + result + } catch { + case failure: Throwable => + try result.close() + catch { + case closeFailure: IOException => failure.addSuppressed(closeFailure) + } + throw failure + } + } + private val reader = new BufferedReader(new InputStreamReader(socket.getInputStream)) + private val writer = new BufferedWriter(new OutputStreamWriter(socket.getOutputStream)) + + def report(): Unit = { + val status = if (loadOnly) LightGBMConstants.IgnoreStatus else LightGBMConstants.EnabledTask + val reportedExecutorId = if (executorId.isEmpty) partitionId.toString else executorId + writer.write(s"$status:127.0.0.1:$listenPort:$partitionId:$reportedExecutorId\n") + writer.flush() + } + + val listenPort: Int = LightGBMConstants.DefaultLocalListenPort + partitionId + listenPortOffset + def address: String = s"127.0.0.1:$listenPort" + + def readTopology(): (String, String) = (reader.readLine(), reader.readLine()) + + /** The driver closing its end is observed here as end-of-stream. */ + def isClosedByDriver: Boolean = { + try { + reader.readLine() == null + } catch { + case _: SocketException => true + } + } + + override def close(): Unit = socket.close() + } + + private class SignalSecondAcceptServerSocket extends ServerSocket(0) { + private val acceptCount = new AtomicInteger() + private val secondAcceptStarted = new CountDownLatch(1) + setSoTimeout(socketTimeoutMillis) + + override def accept(): Socket = { + if (acceptCount.incrementAndGet() == 2) secondAcceptStarted.countDown() + super.accept() + } + + def awaitSecondAccept(): Unit = { + assert(secondAcceptStarted.await(socketTimeoutMillis, TimeUnit.MILLISECONDS), + "The driver never registered the first task socket") + } + } + + private class GateAfterAcceptServerSocket extends ServerSocket(0) { + private val socketAccepted = new CountDownLatch(1) + private val releaseAcceptedSocket = new CountDownLatch(1) + @volatile private var socket: Socket = _ + setSoTimeout(socketTimeoutMillis) + + override def accept(): Socket = { + val result = super.accept() + socket = result + socketAccepted.countDown() + if (!releaseAcceptedSocket.await(socketTimeoutMillis, TimeUnit.MILLISECONDS)) { + result.close() + throw new SocketTimeoutException("Timed out waiting to release the accepted socket") + } + result + } + + def awaitAccepted(): Unit = { + assert(socketAccepted.await(socketTimeoutMillis, TimeUnit.MILLISECONDS), + "The driver never accepted the task socket") + } + + def release(): Unit = releaseAcceptedSocket.countDown() + + def acceptedSocketIsClosed: Boolean = socket != null && socket.isClosed + } + + private class GateUnexpectedAcceptFailureServerSocket extends SignalSecondAcceptServerSocket { + private val acceptFailureObserved = new CountDownLatch(1) + private val releaseAcceptFailure = new CountDownLatch(1) + + override def accept(): Socket = { + try { + super.accept() + } catch { + case failure: SocketException => + acceptFailureObserved.countDown() + if (!releaseAcceptFailure.await(socketTimeoutMillis, TimeUnit.MILLISECONDS)) { + failure.addSuppressed(new SocketTimeoutException("Timed out waiting to release accept failure")) + } + throw failure + } + } + + def awaitAcceptFailure(): Unit = { + assert(acceptFailureObserved.await(socketTimeoutMillis, TimeUnit.MILLISECONDS), + "The driver's blocked accept did not observe the unexpected server close") + } + + def releaseFailure(): Unit = releaseAcceptFailure.countDown() + } + + private def newManager(numTasks: Int, + useBarrierExecutionMode: Boolean = false, + serverSocket: ServerSocket = new ServerSocket(0)): (NetworkManager, String, Int) = { + serverSocket.setSoTimeout(socketTimeoutMillis) + val port = serverSocket.getLocalPort + (NetworkManager(numTasks, serverSocket, host, port, timeout, useBarrierExecutionMode), host, port) + } + + private def closeTasks(tasks: Iterable[FakeTask]): Unit = { + tasks.foreach { task => + try task.close() + catch { + case _: IOException => () + } + } + } + + private def partitionIds(executorTopology: String): Seq[Int] = { + executorTopology.split(":").flatMap { executorEntry => + executorEntry.split("=")(1).split(",").map(_.toInt) + }.toSeq + } + + private def runTopologyRound(numTasks: Int, + numLoadOnlyTasks: Int = 0): (NetworkManager, String, Int, Seq[FakeTask]) = { + val (manager, host, port) = newManager(numTasks + numLoadOnlyTasks) + val tasks = ListBuffer.empty[FakeTask] + try { + (0 until numTasks).foreach(partitionId => tasks += new FakeTask(host, port, partitionId)) + (0 until numLoadOnlyTasks).foreach { index => + tasks += new FakeTask(host, port, numTasks + index, loadOnly = true) + } + tasks.foreach(_.report()) + tasks.foreach { task => + val (machineList, partitionList) = task.readTopology() + assert(machineList != null && machineList.nonEmpty) + assert(partitionList != null && partitionList.nonEmpty) + } + manager.waitForNetworkCommunicationsDone() + (manager, host, port, tasks.toList) + } catch { + case failure: Throwable => + manager.closeConnections() + closeTasks(tasks) + throw failure + } + } + + test("A retried task is refused by the driver once the topology round has completed") { + val (manager, host, port, tasks) = runTopologyRound(numTasks = 2) + try { + // A Spark task retry re-enters getGlobalNetworkInfo and reconnects to the same driver endpoint. + val retriedSocket = new Socket() + val thrown = try { + intercept[ConnectException] { + retriedSocket.connect(new InetSocketAddress(host, port), socketTimeoutMillis) + } + } finally { + retriedSocket.close() + } + + assert(thrown.getMessage.toLowerCase.contains("refused"), + s"Expected a connection-refused failure but got: ${thrown.getMessage}") + } finally { + manager.closeConnections() + closeTasks(tasks) + } + } + + test("Helper task sockets are released along with training task sockets") { + val (manager, _, _, tasks) = runTopologyRound(numTasks = 2, numLoadOnlyTasks = 2) + try { + tasks.zipWithIndex.foreach { case (task, index) => + assert(task.isClosedByDriver, s"Driver leaked the socket for task $index") + } + } finally { + manager.closeConnections() + closeTasks(tasks) + } + } + + test("Non-barrier topology waits for every unique partition when a report is retried") { + val (manager, _, port) = newManager(numTasks = 2) + var tasks = Seq.empty[FakeTask] + try { + val original = new FakeTask( + host, port, partitionId = 0, loadOnly = true, executorId = "old-executor") + tasks :+= original + val replacement = new FakeTask( + host, port, partitionId = 0, listenPortOffset = 1000, executorId = "new-executor") + tasks :+= replacement + + original.report() + replacement.report() + assert(original.isClosedByDriver, "The superseded partition report socket was not closed") + + val secondPartition = new FakeTask(host, port, partitionId = 1) + tasks :+= secondPartition + secondPartition.report() + + val expectedNetwork = Set(replacement.address, secondPartition.address) + Seq(replacement, secondPartition).foreach { task => + val (networkTopology, executorTopology) = task.readTopology() + val networkNodes = networkTopology.split(",").toSeq + assert(networkNodes.size == expectedNetwork.size) + assert(networkNodes.toSet == expectedNetwork) + assert(partitionIds(executorTopology).sorted == Seq(0, 1)) + assert(!executorTopology.contains("old-executor")) + assert(executorTopology.contains("new-executor=0")) + } + manager.waitForNetworkCommunicationsDone() + } finally { + manager.closeConnections() + closeTasks(tasks) + } + } + + test("closeConnections remains idempotent while a topology round owns task sockets") { + val serverSocket = new SignalSecondAcceptServerSocket() + val (manager, _, port) = + newManager(numTasks = 2, useBarrierExecutionMode = true, serverSocket = serverSocket) + var task = Option.empty[FakeTask] + try { + task = Some(new FakeTask(host, port, partitionId = 0)) + task.get.report() + serverSocket.awaitSecondAccept() + + manager.closeConnections() + manager.closeConnections() + + assert(task.get.isClosedByDriver, "The driver leaked a task socket during repeated cleanup") + manager.waitForNetworkCommunicationsDone() + } finally { + manager.closeConnections() + closeTasks(task) + } + } + + test("An unexpected server close preserves the accept failure and still closes task sockets") { + val serverSocket = new GateUnexpectedAcceptFailureServerSocket() + val (manager, _, port) = + newManager(numTasks = 2, useBarrierExecutionMode = true, serverSocket = serverSocket) + var task = Option.empty[FakeTask] + try { + task = Some(new FakeTask(host, port, partitionId = 0)) + task.get.report() + serverSocket.awaitSecondAccept() + + serverSocket.close() + serverSocket.awaitAcceptFailure() + serverSocket.releaseFailure() + + assert(task.get.isClosedByDriver, "The unexpected server close leaked a task socket") + intercept[SocketException] { + manager.waitForNetworkCommunicationsDone() + } + } finally { + serverSocket.releaseFailure() + manager.closeConnections() + closeTasks(task) + } + } + + test("A socket accepted during shutdown is rejected before round registration") { + val serverSocket = new GateAfterAcceptServerSocket() + val (manager, _, port) = + newManager(numTasks = 1, useBarrierExecutionMode = true, serverSocket = serverSocket) + var task = Option.empty[FakeTask] + try { + task = Some(new FakeTask(host, port, partitionId = 0)) + task.get.report() + serverSocket.awaitAccepted() + + manager.closeConnections() + serverSocket.release() + + assert(task.get.isClosedByDriver, "The socket accepted during shutdown was retained") + manager.waitForNetworkCommunicationsDone() + assert(serverSocket.acceptedSocketIsClosed, "The driver did not close the accepted socket") + } finally { + serverSocket.release() + manager.closeConnections() + closeTasks(task) + } + } + + test("The legacy TaskMessageInfo constructors, extractor, and product shape remain unchanged") { + val message = TaskMessageInfo( + LightGBMConstants.EnabledTask, + "127.0.0.1", + LightGBMConstants.DefaultLocalListenPort, + 3, + "executor-1") + val generalMessage = new TaskMessageInfo(LightGBMConstants.FinishedStatus) + + val TaskMessageInfo(status, taskHost, listenPort, partitionId, executorId) = message + assert(status == LightGBMConstants.EnabledTask) + assert(taskHost == "127.0.0.1") + assert(listenPort == LightGBMConstants.DefaultLocalListenPort) + assert(partitionId == 3) + assert(executorId == "executor-1") + assert(message.productArity == 5) + assert(message.toString == + s"${LightGBMConstants.EnabledTask}:127.0.0.1:${LightGBMConstants.DefaultLocalListenPort}:3:executor-1") + assert(NetworkManager.parseWorkerMessage(s"${message.toString}:7") == message) + assert(generalMessage.isFinished) + assert(generalMessage.productArity == 5) + val constructorArities = classOf[TaskMessageInfo].getConstructors.map(_.getParameterCount).toSet + assert(constructorArities.contains(1)) + assert(constructorArities.contains(5)) + assert(!constructorArities.contains(6)) + } + + test("Worker status parsing preserves IPv6 hosts in current and legacy messages") { + val hosts = Seq("2001:db8::1", "2001:db8:0:1:2:3:4:5", "fe80::a%3") + hosts.foreach { taskHost => + val message = TaskMessageInfo( + LightGBMConstants.EnabledTask, + taskHost, + LightGBMConstants.DefaultLocalListenPort, + 3, + "executor-1") + assert(NetworkManager.parseWorkerMessage(s"${message.toString}:7") == message) + } + + val legacyMessage = TaskMessageInfo( + LightGBMConstants.EnabledTask, + "2001:db8::1", + LightGBMConstants.DefaultLocalListenPort, + 3, + "7") + assert(NetworkManager.parseWorkerMessage(legacyMessage.toString) == legacyMessage) + + val ambiguousLegacyMessage = legacyMessage.copy(taskHost = "2001:db8::1:10") + assert(NetworkManager.parseWorkerMessage(ambiguousLegacyMessage.toString) == ambiguousLegacyMessage) + + val lowPortCurrentMessage = legacyMessage.copy(localListenPort = 80, executorId = "executor-1") + assert(NetworkManager.parseWorkerMessage(WorkerMessage.format(lowPortCurrentMessage, 7)) == lowPortCurrentMessage) + } + + test("Finished worker messages tolerate omitted trailing fields") { + val missingBarrierCount = WorkerMessage.parse(s"${LightGBMConstants.FinishedStatus}:7:") + assert(missingBarrierCount.stageAttemptNumber == 7) + assert(missingBarrierCount.barrierTaskCount.isEmpty) + + val missingSuffix = WorkerMessage.parse(s"${LightGBMConstants.FinishedStatus}::") + assert(missingSuffix.stageAttemptNumber == 0) + assert(missingSuffix.barrierTaskCount.isEmpty) + } + + test("Malformed worker messages escape control characters in errors") { + val nul = 0.toChar + val failure = intercept[IllegalArgumentException] { + NetworkManager.parseWorkerMessage(s"enabledTask:host:not-a-port:3:executor-1\r${nul}malformed") + } + assert(failure.getMessage.contains("\\r")) + assert(failure.getMessage.contains("\\u0000")) + assert(!failure.getMessage.contains("\r")) + assert(!failure.getMessage.contains(nul)) + } + + test("A worker that disconnects before sending a message reports the disconnect, not a NullPointerException") { + val failure = intercept[IOException](NetworkManager.parseWorkerMessage(null)) //scalastyle:ignore null + assert(failure.getMessage.contains("closed the connection before sending a status message")) + } + + test("The driver server socket is released when a training job fails before the round completes") { + // Only one of the two expected tasks reports, so the network thread stays blocked in accept(). + val (manager, _, port) = newManager(numTasks = 2, useBarrierExecutionMode = true) + var task = Option.empty[FakeTask] + try { + task = Some(new FakeTask(host, port, 0)) + task.get.report() + + // This is what executeTraining now does in its finally block when partition tasks fail. + manager.closeConnections() + manager.waitForNetworkCommunicationsDone() + + val retriedSocket = new Socket() + try { + intercept[ConnectException] { + retriedSocket.connect(new InetSocketAddress(host, port), socketTimeoutMillis) + } + } finally { + retriedSocket.close() + } + } finally { + manager.closeConnections() + closeTasks(task) + } + } +} diff --git a/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/split1/LightGBMIpv6NetworkE2ESuite.scala b/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/split1/LightGBMIpv6NetworkE2ESuite.scala new file mode 100644 index 00000000000..2dbab5d0a02 --- /dev/null +++ b/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/split1/LightGBMIpv6NetworkE2ESuite.scala @@ -0,0 +1,398 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.lightgbm.split1 + +import com.microsoft.azure.synapse.ml.lightgbm.{LightGBMConstants, LightGBMNetworkBridge, LightGBMUtils, + NetworkManager, NetworkTopologyInfo} +import com.microsoft.ml.lightgbm.{lightgbmlib, lightgbmlibConstants} +import org.scalatest.BeforeAndAfterEach +import org.scalatest.funsuite.AnyFunSuite +import org.slf4j.LoggerFactory + +import java.io.DataInputStream +import java.net.{InetAddress, InetSocketAddress, NetworkInterface, ServerSocket, Socket} +import java.nio.{ByteBuffer, ByteOrder} +import java.util.concurrent.atomic.{AtomicInteger, AtomicReference} +import java.util.concurrent.{ConcurrentLinkedQueue, CountDownLatch, Executors, ThreadFactory, TimeUnit} +import scala.annotation.tailrec +import scala.collection.JavaConverters._ +import scala.util.Try + +/** Drives the real native LightGBM network over IPv6. + * + * The native network state is thread local, so one JVM can hold several LightGBM ranks as long as + * each one stays on its own thread. These tests use that to run a real two worker training round + * over IPv6, and to check the single rank link behavior against simulated peers. + */ +class LightGBMIpv6NetworkE2ESuite extends AnyFunSuite with BeforeAndAfterEach { + + private val log = LoggerFactory.getLogger(classOf[LightGBMIpv6NetworkE2ESuite]) + private val ipv6Loopback = "::1" + private val ipv4Loopback = "127.0.0.1" + private val socketTimeoutMillis = 30000L + private val connectAttemptTimeoutMillis = 1000 + private val nativeWorkTimeoutMillis = 180000L + private val retryIntervalMillis = 10L + private val rankSize = 4 + private val trainingRows = 64 + private val trainingCols = 2 + private val trainingIterations = 5 + private val peerLabelOffset = 100.0f + private val modelBufferLength = 1L << 20 + private val closeables = new ConcurrentLinkedQueue[AutoCloseable]() + + private val peerExecutor = Executors.newCachedThreadPool(new ThreadFactory { + override def newThread(runnable: Runnable): Thread = { + val thread = new Thread(runnable, "lightgbm-ipv6-e2e-peer") + thread.setDaemon(true) + thread + } + }) + + override def beforeEach(): Unit = { + super.beforeEach() + LightGBMUtils.initializeNativeLibrary() + } + + override def afterEach(): Unit = { + closeables.asScala.foreach(resource => Try(resource.close())) + closeables.clear() + super.afterEach() + } + + private def register[T <: AutoCloseable](resource: T): T = { + closeables.add(resource) + resource + } + + private def ipv6LoopbackAvailable: Boolean = Try { + val probe = new ServerSocket() + try { + probe.bind(new InetSocketAddress(InetAddress.getByName(ipv6Loopback), 0)) + true + } finally { + probe.close() + } + }.getOrElse(false) + + /** An address the native library also finds in its own local address list, on every platform. */ + private def localIpv4Host: String = { + Try(NetworkInterface.getNetworkInterfaces.asScala + .filter(candidate => Try(candidate.isUp).getOrElse(false)) + .flatMap(_.getInetAddresses.asScala) + .find(address => address.isSiteLocalAddress && address.getAddress.length == 4) + .map(_.getHostAddress)).toOption.flatten.getOrElse(ipv4Loopback) + } + + private def freePort(host: String): Int = { + val probe = new ServerSocket() + try { + probe.bind(new InetSocketAddress(InetAddress.getByName(host), 0)) + probe.getLocalPort + } finally { + probe.close() + } + } + + private def listenOn(host: String): ServerSocket = { + val listener = new ServerSocket() + listener.bind(new InetSocketAddress(InetAddress.getByName(host), 0)) + listener.setSoTimeout(socketTimeoutMillis.toInt) + register(listener) + } + + private def rankBytes(rank: Int): Array[Byte] = + ByteBuffer.allocate(rankSize).order(ByteOrder.nativeOrder()).putInt(rank).array() + + private def readRank(socket: Socket): Int = { + val buffer = new Array[Byte](rankSize) + new DataInputStream(socket.getInputStream).readFully(buffer) + ByteBuffer.wrap(buffer).order(ByteOrder.nativeOrder()).getInt + } + + /** Stand in for a machine LightGBM expects to dial this task, which every lower rank does. */ + private def startDialingPeer(host: String, port: Int, rank: Int): CountDownLatch = { + val linked = new CountDownLatch(1) + peerExecutor.execute(new Runnable { + override def run(): Unit = { + val deadline = System.currentTimeMillis() + socketTimeoutMillis + + @tailrec + def dial(): Unit = { + val socket = new Socket() + val connected = try { + socket.connect(new InetSocketAddress(InetAddress.getByName(host), port), connectAttemptTimeoutMillis) + register(socket) + socket.getOutputStream.write(rankBytes(rank)) + socket.getOutputStream.flush() + true + } catch { + case _: Exception => + Try(socket.close()) + false + } + if (connected) { + linked.countDown() + // Hold the link open: LightGBM keeps every peer socket for the whole training run. + Thread.sleep(socketTimeoutMillis) + } else if (System.currentTimeMillis() < deadline) { + Thread.sleep(retryIntervalMillis) + dial() + } + } + + dial() + } + }) + linked + } + + /** Stand in for a machine LightGBM dials, which every higher rank is. */ + private def startAcceptingPeer(listener: ServerSocket, receivedRank: AtomicInteger): CountDownLatch = { + val linked = new CountDownLatch(1) + peerExecutor.execute(new Runnable { + override def run(): Unit = { + val socket = register(listener.accept()) + receivedRank.set(readRank(socket)) + linked.countDown() + Thread.sleep(socketTimeoutMillis) + } + }) + linked + } + + private def nativeNetworkInit(machineList: String, localListenPort: Int, machineCount: Int): Int = + lightgbmlib.LGBM_NetworkInit(machineList, localListenPort, LightGBMConstants.DefaultListenTimeout, machineCount) + + /** Run native work on its own thread, because LightGBM keeps its network state thread local. */ + private def onNativeThread[T](name: String)(work: => T): () => T = { + val outcome = new AtomicReference[Either[Throwable, T]]() + val done = new CountDownLatch(1) + val thread = new Thread(new Runnable { + override def run(): Unit = { + try { + outcome.set(Right(work)) + } catch { + case failure: Throwable => outcome.set(Left(failure)) + } finally { + done.countDown() + } + } + }, name) + thread.setDaemon(true) + thread.start() + () => { + assert(done.await(nativeWorkTimeoutMillis, TimeUnit.MILLISECONDS), s"$name never finished") + outcome.get() match { + case Right(result) => result + case Left(failure) => throw failure + } + } + } + + private def await[T](pending: () => T): T = pending() + + /** Initialize the production network path, run the body, and always release the native network. */ + private def withNativeNetwork[T](topology: NetworkTopologyInfo, machineCount: Int)(body: => T): T = { + var initialized = false + try { + NetworkManager.initNativeNetwork(topology, machineCount, log) + initialized = true + body + } finally { + if (initialized) lightgbmlib.LGBM_NetworkFree() + topology.releaseNetworkResources() + } + } + + /** Train a tiny model on this rank's shard, which allreduces with every peer on every iteration. */ + private def trainShard(rank: Int, numMachines: Int): String = { + val features = lightgbmlib.new_doubleArray((trainingRows * trainingCols).toLong) + val labels = lightgbmlib.new_floatArray(trainingRows.toLong) + (0 until trainingRows).foreach { row => + lightgbmlib.doubleArray_setitem(features, (row * trainingCols).toLong, row.toDouble) + lightgbmlib.doubleArray_setitem(features, (row * trainingCols + 1).toLong, (row % 8).toDouble) + // Each rank holds a clearly different slice, so a model built from both is easy to tell apart. + lightgbmlib.floatArray_setitem(labels, row.toLong, row.toFloat + rank * peerLabelOffset) + } + + val datasetParams = "max_bin=15 min_data_in_bin=1 min_data_in_leaf=1 verbosity=-1" + val datasetOut = lightgbmlib.voidpp_handle() + LightGBMUtils.validate(lightgbmlib.LGBM_DatasetCreateFromMat( + lightgbmlib.double_to_voidp_ptr(features), + lightgbmlibConstants.C_API_DTYPE_FLOAT64, + trainingRows, + trainingCols, + 1, + datasetParams, + None.orNull, + datasetOut), "Dataset create") + val dataset = lightgbmlib.voidpp_value(datasetOut) + LightGBMUtils.validate(lightgbmlib.LGBM_DatasetSetField(dataset, "label", + lightgbmlib.float_to_voidp_ptr(labels), trainingRows, lightgbmlibConstants.C_API_DTYPE_FLOAT32), + "Dataset set label") + + val boosterOut = lightgbmlib.voidpp_handle() + LightGBMUtils.validate(lightgbmlib.LGBM_BoosterCreate(dataset, + s"objective=regression tree_learner=data num_machines=$numMachines num_leaves=4 learning_rate=0.5 " + + s"$datasetParams num_threads=1", boosterOut), "Booster create") + val booster = lightgbmlib.voidpp_value(boosterOut) + val isFinished = lightgbmlib.new_intp() + val modelLength = lightgbmlib.new_int64_tp() + try { + (0 until trainingIterations).foreach(_ => + LightGBMUtils.validate(lightgbmlib.LGBM_BoosterUpdateOneIter(booster, isFinished), "Update one iteration")) + lightgbmlib.LGBM_BoosterSaveModelToStringSWIG(booster, 0, -1, 0, modelBufferLength, modelLength) + } finally { + lightgbmlib.delete_intp(isFinished) + lightgbmlib.delete_int64_tp(modelLength) + lightgbmlib.LGBM_BoosterFree(booster) + lightgbmlib.LGBM_DatasetFree(dataset) + lightgbmlib.delete_doubleArray(features) + lightgbmlib.delete_floatArray(labels) + } + } + + private def leafValues(model: String): Seq[Double] = { + model.split("\n").filter(_.startsWith("leaf_value=")).flatMap(line => + line.stripPrefix("leaf_value=").trim.split("\\s+").filter(_.nonEmpty).map(_.toDouble)).toSeq + } + + test("Without a bridge the native library cannot form a network from any IPv6 machine list") { + assume(ipv6LoopbackAvailable, "IPv6 loopback is not available on this machine") + val selfPort = freePort(ipv6Loopback) + val peerPort = freePort(ipv6Loopback) + // The bracketed form is what an IPv6 aware driver publishes. + val bracketed = nativeNetworkInit(s"[$ipv6Loopback]:$selfPort,[$ipv6Loopback]:$peerPort", selfPort, 2) + assert(bracketed == -1, "The native library unexpectedly accepted a bracketed IPv6 machine list") + assert(lightgbmlib.LGBM_GetLastError().contains("Cannot find any ip and port")) + + // The bare form is what the driver published before this change: the native parser splits it on + // ':' and keeps a meaningless host instead of rejecting the entry. + val bare = nativeNetworkInit(s"$ipv6Loopback:$selfPort,$ipv6Loopback:$peerPort", selfPort, 2) + assert(bare == -1, "The native library unexpectedly accepted a bare IPv6 machine list") + assert(lightgbmlib.LGBM_GetLastError().contains("doesn't contain the local machine")) + + // A routable IPv6 address fares no better, so this is not a loopback quirk. + val routable = nativeNetworkInit(s"[2001:db8::1]:$selfPort,[2001:db8::2]:$peerPort", selfPort, 2) + assert(routable == -1, "The native library unexpectedly accepted a routable IPv6 machine list") + } + + test("An IPv4 topology still initializes natively with no bridge and no rewriting") { + val host = localIpv4Host + val selfPort = freePort(ipv4Loopback) + val peerListener = listenOn(ipv4Loopback) + val receivedRank = new AtomicInteger(-1) + val peerLinked = startAcceptingPeer(peerListener, receivedRank) + + val machineList = s"$host:$selfPort,$ipv4Loopback:${peerListener.getLocalPort}" + val topology = NetworkTopologyInfo(machineList, Array(0), selfPort).withAdvertisedHost(host) + await(onNativeThread("lightgbm-ipv4-rank") { + withNativeNetwork(topology, 2) { + assert(!topology.hasNetworkBridge, "An IPv4 topology has to reach the native library untouched") + assert(peerLinked.await(socketTimeoutMillis, TimeUnit.MILLISECONDS), + "The native library never linked to its IPv4 peer") + assert(receivedRank.get() == 0, "The native library announced the wrong rank to its peer") + } + }) + } + + test("A LightGBM worker links to IPv6 peers in both directions through the bridge") { + assume(ipv6LoopbackAvailable, "IPv6 loopback is not available on this machine") + // Rank 1 of 3 is the only rank that both accepts a link (from rank 0) and dials one (to rank 2). + val advertisedPort = freePort(ipv6Loopback) + val lowerRankPort = freePort(ipv6Loopback) + val higherRankListener = listenOn(ipv6Loopback) + val receivedRank = new AtomicInteger(-1) + val higherRankLinked = startAcceptingPeer(higherRankListener, receivedRank) + val lowerRankLinked = startDialingPeer(ipv6Loopback, advertisedPort, 0) + + val machineList = s"[$ipv6Loopback]:$lowerRankPort,[$ipv6Loopback]:$advertisedPort," + + s"[$ipv6Loopback]:${higherRankListener.getLocalPort}" + val topology = NetworkTopologyInfo(machineList, Array(0), advertisedPort).withAdvertisedHost(ipv6Loopback) + await(onNativeThread("lightgbm-ipv6-rank") { + withNativeNetwork(topology, 3) { + assert(topology.hasNetworkBridge, "An IPv6 topology has to be bridged onto the native transport") + assert(lowerRankLinked.await(socketTimeoutMillis, TimeUnit.MILLISECONDS), + "The lower rank never linked to this worker over IPv6") + assert(higherRankLinked.await(socketTimeoutMillis, TimeUnit.MILLISECONDS), + "This worker never linked to the higher rank over IPv6") + assert(receivedRank.get() == 1, + s"The higher rank received rank ${receivedRank.get()} instead of the bridged worker's rank 1") + } + }) + } + + test("The native listener stops accepting once the bridge has claimed its link slots") { + assume(ipv6LoopbackAvailable, "IPv6 loopback is not available on this machine") + // Rank 1 of 2, so the native listener has exactly one slot, which the bridge claims from loopback. + val advertisedPort = freePort(ipv6Loopback) + val machineList = s"[$ipv6Loopback]:${freePort(ipv6Loopback)},[$ipv6Loopback]:$advertisedPort" + val bridge = register(LightGBMNetworkBridge.open(machineList, ipv6Loopback, advertisedPort, log)) + val bridged = bridge.bridgedNetwork + val lowerRankLinked = startDialingPeer(ipv6Loopback, advertisedPort, 0) + + await(onNativeThread("lightgbm-native-listener") { + val result = nativeNetworkInit(bridged.machineList, bridged.localListenPort, bridged.machineCount) + try { + assert(result == 0, s"Native init failed: ${lightgbmlib.LGBM_GetLastError()}") + assert(lowerRankLinked.await(socketTimeoutMillis, TimeUnit.MILLISECONDS), + "The lower rank never linked through the bridge") + } finally { + if (result == 0) lightgbmlib.LGBM_NetworkFree() + } + }) + + // The native library closes its listener as soon as its slots are filled, and only the bridge + // ever filled them, so nothing external can still reach that port. + val probe = new Socket() + val refused = try { + probe.connect(new InetSocketAddress(InetAddress.getByName(ipv4Loopback), bridged.localListenPort), + connectAttemptTimeoutMillis) + false + } catch { + case _: java.io.IOException => true + } finally { + Try(probe.close()) + } + assert(refused, s"The native listener on port ${bridged.localListenPort} is still accepting connections") + } + + test("Two LightGBM workers train one distributed model over IPv6") { + assume(ipv6LoopbackAvailable, "IPv6 loopback is not available on this machine") + val firstPort = freePort(ipv6Loopback) + val secondPort = freePort(ipv6Loopback) + val machineList = s"[$ipv6Loopback]:$firstPort,[$ipv6Loopback]:$secondPort" + + def worker(rank: Int, advertisedPort: Int): () => String = { + val topology = NetworkTopologyInfo(machineList, Array(rank), advertisedPort).withAdvertisedHost(ipv6Loopback) + onNativeThread(s"lightgbm-ipv6-worker-$rank") { + withNativeNetwork(topology, 2) { + assert(topology.hasNetworkBridge, s"Worker $rank did not bridge its IPv6 topology") + trainShard(rank, 2) + } + } + } + + // Both workers have to be running before either can link, exactly as two Spark tasks would be. + val pendingFirst = worker(0, firstPort) + val pendingSecond = worker(1, secondPort) + val firstModel = await(pendingFirst) + val secondModel = await(pendingSecond) + + assert(firstModel.contains("Tree=0"), "The distributed training produced no trees") + assert(firstModel.contains("split_feature="), "The distributed training produced only empty trees") + assert(firstModel == secondModel, + "Data parallel LightGBM builds the same model on every rank, so the two IPv6 workers disagreeing " + + "means their allreduce traffic did not cross the bridge") + + // A model built from only one shard cannot see the peer's labels, so its leaves sit far lower. + val localModel = await(onNativeThread("lightgbm-single-machine")(trainShard(0, 1))) + val distributedMean = leafValues(firstModel).sum / leafValues(firstModel).length + val localMean = leafValues(localModel).sum / leafValues(localModel).length + assert(math.abs(distributedMean - localMean) > 1.0, + s"The distributed leaves ($distributedMean) match a single machine model ($localMean), so the " + + "peer's data never reached this worker") + } +} diff --git a/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/split1/LightGBMNetworkBridgeLifecycleSuite.scala b/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/split1/LightGBMNetworkBridgeLifecycleSuite.scala new file mode 100644 index 00000000000..550a27e576f --- /dev/null +++ b/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/split1/LightGBMNetworkBridgeLifecycleSuite.scala @@ -0,0 +1,608 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.lightgbm.split1 + +import com.microsoft.azure.synapse.ml.lightgbm.{LightGBMNetworkBridge, LightGBMNetworkRelay, NetworkManager, + NetworkTopologyInfo, WorkerEndpoint} +import org.scalatest.BeforeAndAfterEach +import org.scalatest.funsuite.AnyFunSuite +import org.slf4j.LoggerFactory + +import java.io.{DataInputStream, IOException} +import java.net.{InetAddress, InetSocketAddress, ServerSocket, Socket} +import java.nio.channels.ServerSocketChannel +import java.nio.{ByteBuffer, ByteOrder} +import java.util.Random +import java.util.concurrent.atomic.{AtomicInteger, AtomicLong} +import java.util.concurrent.{Callable, ConcurrentLinkedQueue, Executors, ThreadFactory, TimeUnit} +import scala.annotation.tailrec +import scala.collection.JavaConverters._ +import scala.util.Try + +/** Covers what the IPv6 bridge does over the life of a task: admission, concurrency, and cleanup. */ +class LightGBMNetworkBridgeLifecycleSuite extends AnyFunSuite with BeforeAndAfterEach { + + private val log = LoggerFactory.getLogger(classOf[LightGBMNetworkBridgeLifecycleSuite]) + private val ipv6Loopback = "::1" + private val ipv4Loopback = LightGBMNetworkBridge.LoopbackHost + private val socketTimeoutMillis = 30000 + private val smallPayloadSize = 64 * 1024 + private val concurrentPeers = 8 + private val sequentialPeers = 8 + private val largeTopologySize = 33 + private val stallWriteBytes = 64L * 1024 * 1024 + private val stallObservationMillis = 2000L + private val maxBufferedBytes = 8L * 1024 * 1024 + private val threadShutdownMillis = 10000L + private val unsolicitedConnections = 6 + private val shortHandshakeMillis = 500L + private val closeables = new ConcurrentLinkedQueue[AutoCloseable]() + private val pool = Executors.newCachedThreadPool(new ThreadFactory { + override def newThread(runnable: Runnable): Thread = { + val thread = new Thread(runnable, "lightgbm-network-bridge-lifecycle-test") + thread.setDaemon(true) + thread + } + }) + + override def afterEach(): Unit = { + closeables.asScala.foreach(resource => Try(resource.close())) + closeables.clear() + super.afterEach() + } + + private def register[T <: AutoCloseable](resource: T): T = { + closeables.add(resource) + resource + } + + private def ipv6LoopbackAvailable: Boolean = Try { + val probe = new ServerSocket() + try { + probe.bind(new InetSocketAddress(InetAddress.getByName(ipv6Loopback), 0)) + true + } finally { + probe.close() + } + }.getOrElse(false) + + private def freePort(host: String): Int = { + val probe = new ServerSocket() + try { + probe.bind(new InetSocketAddress(InetAddress.getByName(host), 0)) + probe.getLocalPort + } finally { + probe.close() + } + } + + private def listenOnLoopback(port: Int): ServerSocket = { + val listener = new ServerSocket() + listener.bind(new InetSocketAddress(InetAddress.getByName(ipv4Loopback), port)) + listener.setSoTimeout(socketTimeoutMillis) + register(listener) + } + + private def connect(host: String, port: Int): Socket = { + val socket = new Socket() + socket.connect(new InetSocketAddress(InetAddress.getByName(host), port), socketTimeoutMillis) + socket.setSoTimeout(socketTimeoutMillis) + register(socket) + } + + private def dialAsRank(port: Int, rank: Int): Socket = { + val socket = connect(ipv6Loopback, port) + socket.getOutputStream.write(LightGBMNetworkRelay.rankBuffer(rank).array()) + socket.getOutputStream.flush() + socket + } + + private def acceptNativeSlot(listener: ServerSocket): (Int, Socket) = { + val socket = register(listener.accept()) + val bytes = new Array[Byte](LightGBMNetworkRelay.RankBytes) + new DataInputStream(socket.getInputStream).readFully(bytes) + (ByteBuffer.wrap(bytes).order(ByteOrder.nativeOrder()).getInt, socket) + } + + /** A machine list where this task is the last entry, so every other machine is a lower rank. */ + private def machineListWithSelfLast(peerCount: Int, advertisedPort: Int): String = + ((1 to peerCount).map(_ => s"[$ipv6Loopback]:${freePort(ipv6Loopback)}") :+ + s"[$ipv6Loopback]:$advertisedPort").mkString(",") + + private def twoMachineList(selfPort: Int, peerPort: Int): String = + s"[$ipv6Loopback]:$selfPort,[$ipv6Loopback]:$peerPort" + + private def payload(seed: Int, size: Int): Array[Byte] = { + val bytes = new Array[Byte](size) + new Random(seed.toLong).nextBytes(bytes) + bytes + } + + private def assertPayloadCrosses(sender: Socket, receiver: Socket, bytes: Array[Byte]): Unit = { + val received = pool.submit(new Callable[Array[Byte]] { + override def call(): Array[Byte] = { + val buffer = new Array[Byte](bytes.length) + new DataInputStream(receiver.getInputStream).readFully(buffer) + buffer + } + }) + sender.getOutputStream.write(bytes) + sender.getOutputStream.flush() + assert(received.get(socketTimeoutMillis.toLong, TimeUnit.MILLISECONDS).sameElements(bytes), + "A relayed payload did not arrive intact") + } + + private def assertEndOfStream(socket: Socket, clue: String): Unit = { + val ended = try { + socket.setSoTimeout(socketTimeoutMillis) + socket.getInputStream.read() == -1 + } catch { + case _: IOException => true + } + assert(ended, clue) + } + + private def relayThreads(bridge: LightGBMNetworkBridge): Seq[Thread] = + Thread.getAllStackTraces.keySet.asScala + .filter(thread => thread.isAlive && thread.getName.startsWith(bridge.threadNamePrefix)).toSeq + + private def awaitNoRelayThreads(bridge: LightGBMNetworkBridge): Unit = { + val deadline = System.currentTimeMillis() + threadShutdownMillis + + @tailrec + def poll(): Seq[Thread] = { + val alive = relayThreads(bridge) + if (alive.isEmpty || System.currentTimeMillis() > deadline) { + alive + } else { + Thread.sleep(50L) + poll() + } + } + + assert(poll().isEmpty, s"Relay threads of ${bridge.threadNamePrefix} outlived the bridge") + } + + private def awaitLinkCount(bridge: LightGBMNetworkBridge, expected: Int, clue: String): Unit = { + val deadline = System.currentTimeMillis() + threadShutdownMillis + + @tailrec + def poll(): Int = { + val count = bridge.relayLinkCount + if (count == expected || System.currentTimeMillis() > deadline) count else { + Thread.sleep(25L) + poll() + } + } + + assert(poll() == expected, clue) + } + + private def assertPortIsFree(port: Int): Unit = { + val rebound = new ServerSocket() + try { + rebound.bind(new InetSocketAddress(port)) + assert(rebound.isBound) + } finally { + rebound.close() + } + } + + test("A connection that never sends its rank is dropped instead of stalling the native listener") { + assume(ipv6LoopbackAvailable, "IPv6 loopback is not available on this machine") + val advertisedPort = freePort(ipv6Loopback) + val bridge = register(LightGBMNetworkBridge.open(machineListWithSelfLast(1, advertisedPort), + ipv6Loopback, advertisedPort, log, handshakeTimeoutMillis = shortHandshakeMillis)) + listenOnLoopback(bridge.bridgedNetwork.localListenPort) + + // The native accept loop has no timeout of its own, so a silent caller has to be dropped here. + val silent = connect(ipv6Loopback, advertisedPort) + assertEndOfStream(silent, "A connection that never sent a rank was left open") + awaitLinkCount(bridge, 0, "A dropped handshake still holds a link slot") + } + + test("A connection claiming a rank the topology does not have is refused") { + assume(ipv6LoopbackAvailable, "IPv6 loopback is not available on this machine") + val advertisedPort = freePort(ipv6Loopback) + // Rank 1 of 2, so rank 0 is the only rank allowed to open a link here. + val bridge = register(LightGBMNetworkBridge.open(machineListWithSelfLast(1, advertisedPort), + ipv6Loopback, advertisedPort, log)) + listenOnLoopback(bridge.bridgedNetwork.localListenPort) + + Seq(1, 7, -1).foreach { forged => + val stray = dialAsRank(advertisedPort, forged) + assertEndOfStream(stray, s"A connection claiming rank $forged was relayed to the native library") + } + awaitLinkCount(bridge, 0, "A refused handshake still holds a link slot") + } + + test("A second connection claiming an already linked rank is refused") { + assume(ipv6LoopbackAvailable, "IPv6 loopback is not available on this machine") + val advertisedPort = freePort(ipv6Loopback) + val bridge = register(LightGBMNetworkBridge.open(machineListWithSelfLast(1, advertisedPort), + ipv6Loopback, advertisedPort, log)) + val nativeListener = listenOnLoopback(bridge.bridgedNetwork.localListenPort) + val (claimedRank, nativeSide) = acceptNativeSlot(nativeListener) + assert(claimedRank == 0) + + val first = dialAsRank(advertisedPort, 0) + assertPayloadCrosses(first, nativeSide, payload(1, 64)) + + val impostor = dialAsRank(advertisedPort, 0) + assertEndOfStream(impostor, "A duplicate rank was allowed to take over an established link") + assertPayloadCrosses(first, nativeSide, payload(2, 64)) + } + + test("A mixed IPv4 and IPv6 topology relays only its IPv6 peer and still accepts both families") { + assume(ipv6LoopbackAvailable, "IPv6 loopback is not available on this machine") + val advertisedPort = freePort(ipv6Loopback) + val ipv4PeerPort = freePort(ipv4Loopback) + val ipv6PeerPort = freePort(ipv6Loopback) + val machineList = s"$ipv4Loopback:$ipv4PeerPort,[$ipv6Loopback]:$ipv6PeerPort,[$ipv6Loopback]:$advertisedPort" + val bridge = register(LightGBMNetworkBridge.open(machineList, ipv6Loopback, advertisedPort, log)) + val bridged = bridge.bridgedNetwork + val entries = bridged.machineList.split(",").toSeq + + // The IPv4 peer keeps its own entry, so the native library dials it with no relay in between. + assert(entries(1) == s"$ipv4Loopback:$ipv4PeerPort") + assert(WorkerEndpoint.parse(entries(2)).host == ipv4Loopback) + + val nativeListener = listenOnLoopback(bridged.localListenPort) + val slots = (1 to 2).map(_ => acceptNativeSlot(nativeListener)).toMap + assert(slots.keySet == Set(0, 1)) + + // The advertised port is a dual stack listener, so peers of either family land on the native side. + val ipv4Peer = connect(ipv4Loopback, advertisedPort) + ipv4Peer.getOutputStream.write(LightGBMNetworkRelay.rankBuffer(0).array()) + ipv4Peer.getOutputStream.flush() + assertPayloadCrosses(ipv4Peer, slots(0), payload(1, smallPayloadSize)) + + val ipv6Peer = dialAsRank(advertisedPort, 1) + assertPayloadCrosses(ipv6Peer, slots(1), payload(2, smallPayloadSize)) + assertPayloadCrosses(slots(1), ipv6Peer, payload(3, smallPayloadSize)) + } + + test("Sequential links from every lower rank are served without accumulating threads") { + assume(ipv6LoopbackAvailable, "IPv6 loopback is not available on this machine") + val advertisedPort = freePort(ipv6Loopback) + val bridge = register(LightGBMNetworkBridge.open(machineListWithSelfLast(sequentialPeers, advertisedPort), + ipv6Loopback, advertisedPort, log)) + val nativeListener = listenOnLoopback(bridge.bridgedNetwork.localListenPort) + val slots = (1 to sequentialPeers).map(_ => acceptNativeSlot(nativeListener)).toMap + val threadsAfterStart = relayThreads(bridge).size + + (0 until sequentialPeers).foreach { rank => + val peer = dialAsRank(advertisedPort, rank) + assertPayloadCrosses(peer, slots(rank), payload(rank, 4096)) + peer.close() + slots(rank).close() + } + + assert(relayThreads(bridge).size == threadsAfterStart, + s"The relay thread count changed while serving $sequentialPeers links") + } + + test("Concurrent relayed links keep their streams separate") { + assume(ipv6LoopbackAvailable, "IPv6 loopback is not available on this machine") + val advertisedPort = freePort(ipv6Loopback) + val bridge = register(LightGBMNetworkBridge.open(machineListWithSelfLast(concurrentPeers, advertisedPort), + ipv6Loopback, advertisedPort, log)) + val nativeListener = listenOnLoopback(bridge.bridgedNetwork.localListenPort) + val slots = (1 to concurrentPeers).map(_ => acceptNativeSlot(nativeListener)).toMap + + val peers = (0 until concurrentPeers).map(rank => rank -> dialAsRank(advertisedPort, rank)) + val transfers = peers.map { case (rank, peer) => + pool.submit(new Runnable { + override def run(): Unit = assertPayloadCrosses(peer, slots(rank), payload(rank, smallPayloadSize)) + }) + } + transfers.foreach(_.get(socketTimeoutMillis.toLong, TimeUnit.MILLISECONDS)) + } + + test("One relay thread serves a bridge whatever the machine count is") { + assume(ipv6LoopbackAvailable, "IPv6 loopback is not available on this machine") + val smallPort = freePort(ipv6Loopback) + val small = register(LightGBMNetworkBridge.open(machineListWithSelfLast(1, smallPort), + ipv6Loopback, smallPort, log)) + val largePort = freePort(ipv6Loopback) + val large = register(LightGBMNetworkBridge.open(machineListWithSelfLast(largeTopologySize - 1, largePort), + ipv6Loopback, largePort, log)) + val nativeListener = listenOnLoopback(large.bridgedNetwork.localListenPort) + val slots = (1 until largeTopologySize).map(_ => acceptNativeSlot(nativeListener)).toMap + + val peers = (0 until largeTopologySize - 1).map(rank => rank -> dialAsRank(largePort, rank)) + peers.foreach { case (rank, peer) => assertPayloadCrosses(peer, slots(rank), payload(rank, 1024)) } + + assert(relayThreads(small).size == 1, "A two machine bridge should run exactly one relay thread") + assert(relayThreads(large).size == 1, + s"A $largeTopologySize machine bridge with ${peers.size} live links should still run one relay thread") + } + + test("An abrupt failure on one relay direction closes the other one too") { + assume(ipv6LoopbackAvailable, "IPv6 loopback is not available on this machine") + val advertisedPort = freePort(ipv6Loopback) + val bridge = register(LightGBMNetworkBridge.open(machineListWithSelfLast(1, advertisedPort), + ipv6Loopback, advertisedPort, log)) + val nativeListener = listenOnLoopback(bridge.bridgedNetwork.localListenPort) + val (_, nativeSide) = acceptNativeSlot(nativeListener) + val peer = dialAsRank(advertisedPort, 0) + assertPayloadCrosses(peer, nativeSide, payload(1, 64)) + awaitLinkCount(bridge, 1, "The established link was not counted") + + // A reset, which is what a killed executor or a dropped route looks like to the other side. + nativeSide.setSoLinger(true, 0) + nativeSide.close() + + assertEndOfStream(peer, "The peer was left waiting on a link whose other half had failed") + awaitLinkCount(bridge, 0, "An aborted link never released its slot") + } + + test("A stalled reader stops the sender instead of buffering inside the bridge") { + assume(ipv6LoopbackAvailable, "IPv6 loopback is not available on this machine") + val advertisedPort = freePort(ipv6Loopback) + val bridge = register(LightGBMNetworkBridge.open(machineListWithSelfLast(1, advertisedPort), + ipv6Loopback, advertisedPort, log)) + val nativeListener = listenOnLoopback(bridge.bridgedNetwork.localListenPort) + val (_, nativeSide) = acceptNativeSlot(nativeListener) // deliberately never read from again + val peer = dialAsRank(advertisedPort, 0) + + val accepted = new AtomicLong(0) + pool.submit(new Runnable { + override def run(): Unit = { + val chunk = new Array[Byte](64 * 1024) + + @tailrec + def writeChunk(): Unit = { + if (accepted.get() < stallWriteBytes) { + peer.getOutputStream.write(chunk) + accepted.addAndGet(chunk.length.toLong) + writeChunk() + } + } + + Try(writeChunk()) + } + }) + Thread.sleep(stallObservationMillis) + + val buffered = accepted.get() + assert(buffered < maxBufferedBytes, + s"The bridge absorbed $buffered bytes for a reader that never read, so backpressure is not reaching the peer") + assert(buffered > 0, "The relay never forwarded anything to the stalled reader") + assert(nativeSide.isConnected) + } + + test("Closing the bridge ends its relay thread, which is a daemon") { + assume(ipv6LoopbackAvailable, "IPv6 loopback is not available on this machine") + val advertisedPort = freePort(ipv6Loopback) + val bridge = LightGBMNetworkBridge.open(machineListWithSelfLast(1, advertisedPort), + ipv6Loopback, advertisedPort, log) + val nativeListener = listenOnLoopback(bridge.bridgedNetwork.localListenPort) + acceptNativeSlot(nativeListener) + val peer = dialAsRank(advertisedPort, 0) + + val running = relayThreads(bridge) + assert(running.size == 1, "The bridge should run exactly one relay thread") + assert(running.forall(_.isDaemon), "A relay thread would keep the executor JVM alive after training") + + bridge.close() + awaitNoRelayThreads(bridge) + assertPortIsFree(advertisedPort) + Try(peer.close()) + } + + test("Cleanup after a cancelled task closes the bridge and preserves the interrupt") { + assume(ipv6LoopbackAvailable, "IPv6 loopback is not available on this machine") + val advertisedPort = freePort(ipv6Loopback) + val topology = NetworkTopologyInfo(twoMachineList(advertisedPort, freePort(ipv6Loopback)), + Array(0), advertisedPort).withAdvertisedHost(ipv6Loopback) + NetworkManager.initNativeNetwork(topology, 2, log, (_, _, _) => ()) + assert(topology.hasNetworkBridge) + + // Spark cancels a task by interrupting its thread, and cleanup still has to finish. + Thread.currentThread().interrupt() + try { + topology.releaseNetworkResources() + assert(Thread.currentThread().isInterrupted, + "Bridge cleanup swallowed the interrupt that tells Spark the task was cancelled") + } finally { + Thread.interrupted() + } + assert(!topology.hasNetworkBridge) + assertPortIsFree(advertisedPort) + } + + test("A failed native init closes the bridge so the advertised port is free for the retry") { + assume(ipv6LoopbackAvailable, "IPv6 loopback is not available on this machine") + val advertisedPort = freePort(ipv6Loopback) + val topology = NetworkTopologyInfo(twoMachineList(advertisedPort, freePort(ipv6Loopback)), + Array(0), advertisedPort).withAdvertisedHost(ipv6Loopback) + + val failure = intercept[RuntimeException] { + NetworkManager.initNativeNetwork(topology, 2, log, + (_, _, _) => throw new RuntimeException("native init failed")) + } + + assert(failure.getMessage.contains("native init failed")) + assert(!topology.hasNetworkBridge, "A bridge outlived the native failure it was opened for") + assertPortIsFree(advertisedPort) + val reservation = NetworkManager.reserveExactPort(advertisedPort, log) + try { + assert(reservation.getLocalPort == advertisedPort) + } finally { + reservation.close() + } + } + + test("The native init retry rebinds the advertised port and rebuilds the bridge on every attempt") { + assume(ipv6LoopbackAvailable, "IPv6 loopback is not available on this machine") + val advertisedPort = freePort(ipv6Loopback) + val topology = NetworkTopologyInfo(twoMachineList(advertisedPort, freePort(ipv6Loopback)), + Array(0), advertisedPort).withAdvertisedHost(ipv6Loopback) + val attempts = new AtomicInteger(0) + val nativePorts = new ConcurrentLinkedQueue[Int]() + val machineLists = new ConcurrentLinkedQueue[String]() + + NetworkManager.initLightGBMNetworkWithRetry( + topology, + log, + retry = 2, + delay = 1L, + networkInit = () => NetworkManager.initNativeNetwork(topology, 2, log, (machines, port, _) => { + machineLists.add(machines) + nativePorts.add(port) + if (attempts.incrementAndGet() < 3) throw new RuntimeException("native init failed") + }), + reservePort = port => NetworkManager.reserveExactPort(port, log), + sleep = _ => ()) + + assert(attempts.get() == 3, "The retry did not reach the attempt that succeeds") + assert(machineLists.asScala.forall(_.startsWith(s"${LightGBMNetworkBridge.RankPrefix}0,")), + "Every attempt has to pin the rank for the native library") + assert(nativePorts.asScala.forall(_ != advertisedPort), + "The native listener must never be given the port the bridge owns") + assert(topology.hasNetworkBridge, "The successful attempt did not keep its bridge") + + topology.releaseNetworkResources() + assertPortIsFree(advertisedPort) + } + + /** A connect that fails the way an address with no route does: synchronously, on the caller. */ + private def failingConnect(failuresPerAddress: Int, + matches: InetSocketAddress => Boolean, + attempts: AtomicInteger): LightGBMNetworkRelay.ConnectAttempt = + (channel, address) => { + if (matches(address) && attempts.incrementAndGet() <= failuresPerAddress) { + throw new java.net.SocketException("Network is unreachable") + } + LightGBMNetworkRelay.DefaultConnect(channel, address) + } + + private def isLoopbackAddress(address: InetSocketAddress): Boolean = + address.getAddress.isLoopbackAddress + + test("A dial that fails synchronously is retried on the timer until it connects") { + assume(ipv6LoopbackAvailable, "IPv6 loopback is not available on this machine") + val advertisedPort = freePort(ipv6Loopback) + val attempts = new AtomicInteger(0) + // The native link slot is claimed over loopback, so this fails that dial three times first. + val bridge = register(LightGBMNetworkBridge.open(machineListWithSelfLast(1, advertisedPort), + ipv6Loopback, advertisedPort, log, connectAttempt = failingConnect(3, isLoopbackAddress, attempts))) + val nativeListener = listenOnLoopback(bridge.bridgedNetwork.localListenPort) + + val (claimedRank, nativeSide) = acceptNativeSlot(nativeListener) + assert(claimedRank == 0, "The retried dial did not claim the slot for the lower rank") + assert(attempts.get() > 3, "The connect hook was not exercised") + assert(bridge.terminalFailure.isEmpty, "A retried failure was recorded as terminal") + + val peer = dialAsRank(advertisedPort, 0) + assertPayloadCrosses(peer, nativeSide, payload(1, smallPayloadSize)) + } + + test("A peer that can never be reached releases its slot, keeps the listener, and is recorded") { + assume(ipv6LoopbackAvailable, "IPv6 loopback is not available on this machine") + val peerPort = freePort(ipv6Loopback) + val advertisedPort = freePort(ipv6Loopback) + val attempts = new AtomicInteger(0) + // Every dial to the peer fails synchronously, which is what ENETUNREACH looks like. + val bridge = register(LightGBMNetworkBridge.open( + s"[$ipv6Loopback]:$advertisedPort,[$ipv6Loopback]:$peerPort", ipv6Loopback, advertisedPort, log, + connectTimeoutMillis = 300L, + connectAttempt = failingConnect(Int.MaxValue, address => address.getPort == peerPort, attempts))) + val relayPort = WorkerEndpoint.parse(bridge.bridgedNetwork.machineList.split(",")(2)).port + + val nativeSide = connect(ipv4Loopback, relayPort) + assertEndOfStream(nativeSide, "The native link was left open although its peer was unreachable") + awaitLinkCount(bridge, 0, "An unreachable peer never released its link slot") + assert(attempts.get() > 1, "The dial was not retried before giving up") + + // The listener has to survive the failure of the connection it accepted. + val second = connect(ipv4Loopback, relayPort) + assertEndOfStream(second, "The outbound listener stopped serving after a failed dial") + awaitLinkCount(bridge, 0, "The second attempt never released its link slot") + assert(bridge.terminalFailure.isDefined, "An unreachable peer was not recorded as a terminal failure") + } + + test("A terminal relay failure fails the native init path instead of leaving the task waiting") { + assume(ipv6LoopbackAvailable, "IPv6 loopback is not available on this machine") + val peerPort = freePort(ipv6Loopback) + val advertisedPort = freePort(ipv6Loopback) + val attempts = new AtomicInteger(0) + val bridge = register(LightGBMNetworkBridge.open( + s"[$ipv6Loopback]:$advertisedPort,[$ipv6Loopback]:$peerPort", ipv6Loopback, advertisedPort, log, + connectTimeoutMillis = 300L, + connectAttempt = failingConnect(Int.MaxValue, address => address.getPort == peerPort, attempts))) + val relayPort = WorkerEndpoint.parse(bridge.bridgedNetwork.machineList.split(",")(2)).port + val nativeSide = connect(ipv4Loopback, relayPort) + assertEndOfStream(nativeSide, "The native link was left open although its peer was unreachable") + + val failure = intercept[Exception](NetworkManager.failIfBridgeIsBroken(bridge)) + assert(failure.getMessage.contains("network bridge for this task failed")) + assert(failure.getMessage.contains("could not reach")) + } + + test("A peer that disconnects mid handshake leaves the advertised listener serving") { + assume(ipv6LoopbackAvailable, "IPv6 loopback is not available on this machine") + val advertisedPort = freePort(ipv6Loopback) + val bridge = register(LightGBMNetworkBridge.open(machineListWithSelfLast(1, advertisedPort), + ipv6Loopback, advertisedPort, log)) + val nativeListener = listenOnLoopback(bridge.bridgedNetwork.localListenPort) + val (_, nativeSide) = acceptNativeSlot(nativeListener) + + // A half sent rank followed by a reset, which is what a killed peer looks like. + val aborted = connect(ipv6Loopback, advertisedPort) + aborted.getOutputStream.write(Array[Byte](0, 0)) + aborted.getOutputStream.flush() + aborted.setSoLinger(true, 0) + aborted.close() + awaitLinkCount(bridge, 0, "An aborted handshake never released its link slot") + + val peer = dialAsRank(advertisedPort, 0) + assertPayloadCrosses(peer, nativeSide, payload(2, smallPayloadSize)) + } + + test("Unsolicited connections to the lowest rank never consume the links its peers need") { + assume(ipv6LoopbackAvailable, "IPv6 loopback is not available on this machine") + // Rank 0 is the only rank no machine opens a link to, so every inbound connection is unsolicited. + val advertisedPort = freePort(ipv6Loopback) + val peerListener = ServerSocketChannel.open() + peerListener.bind(new InetSocketAddress(InetAddress.getByName(ipv6Loopback), 0)) + register(peerListener) + val peerPort = peerListener.socket().getLocalPort + val machineList = s"[$ipv6Loopback]:$advertisedPort,[$ipv6Loopback]:$peerPort" + val bridge = register(LightGBMNetworkBridge.open(machineList, ipv6Loopback, advertisedPort, log)) + assert(bridge.bridgedNetwork.rank == 0) + + // More than the two machine cap allows, so a slot leaked per refusal would exhaust it. + (1 to unsolicitedConnections).foreach { attempt => + val stray = connect(ipv6Loopback, advertisedPort) + assertEndOfStream(stray, s"Unsolicited connection $attempt was not refused") + } + awaitLinkCount(bridge, 0, "Refusing an unsolicited connection consumed a link slot") + + // The native library can still open its own outbound link, which draws on the same cap. + val relayPort = WorkerEndpoint.parse(bridge.bridgedNetwork.machineList.split(",")(2)).port + val nativeSide = connect(ipv4Loopback, relayPort) + val peerSide = register(peerListener.accept().socket()) + peerSide.setSoTimeout(socketTimeoutMillis) + assertPayloadCrosses(nativeSide, peerSide, payload(7, smallPayloadSize)) + awaitLinkCount(bridge, 1, "The native outbound link was not admitted after the refusals") + } + + test("NetworkTopologyInfo keeps the three field shape callers and serialized forms depend on") { + val partitions = Array(0, 1) + val topology = NetworkTopologyInfo("10.0.0.4:12400", partitions, 12400) + assert(topology.productArity == 3) + assert(NetworkTopologyInfo.unapply(topology).map { case (machines, ids, port) => + (machines, ids.toSeq, port) + }.contains(("10.0.0.4:12400", Seq(0, 1), 12400))) + assert(topology.copy(localListenPort = 12401).localListenPort == 12401) + + // The advertised host is task local state, so it changes neither the shape nor equality. + val withHost = topology.withAdvertisedHost("2001:db8::1") + assert(withHost.taskHost == "2001:db8::1") + assert(withHost == NetworkTopologyInfo("10.0.0.4:12400", partitions, 12400).withAdvertisedHost("other")) + assert(NetworkTopologyInfo("10.0.0.4:12400", partitions, 12400).taskHost.isEmpty) + } +} diff --git a/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/split1/LightGBMNetworkBridgeSuite.scala b/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/split1/LightGBMNetworkBridgeSuite.scala new file mode 100644 index 00000000000..2bc020292cd --- /dev/null +++ b/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/split1/LightGBMNetworkBridgeSuite.scala @@ -0,0 +1,361 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.lightgbm.split1 + +import com.microsoft.azure.synapse.ml.lightgbm.{LightGBMNetworkBridge, LightGBMNetworkRelay, WorkerEndpoint} +import org.scalatest.BeforeAndAfterEach +import org.scalatest.funsuite.AnyFunSuite +import org.slf4j.LoggerFactory + +import java.io.{DataInputStream, IOException} +import java.net.{InetAddress, InetSocketAddress, NetworkInterface, ServerSocket, Socket} +import java.nio.{ByteBuffer, ByteOrder} +import java.util.Random +import java.util.concurrent.{Callable, ConcurrentLinkedQueue, Executors, ThreadFactory, TimeUnit} +import scala.collection.JavaConverters._ +import scala.util.Try + +/** Covers the IPv6 transport bridge that carries the LightGBM traffic the native library cannot. */ +class LightGBMNetworkBridgeSuite extends AnyFunSuite with BeforeAndAfterEach { + + private val log = LoggerFactory.getLogger(classOf[LightGBMNetworkBridgeSuite]) + private val socketTimeoutMillis = 30000 + private val ipv6Loopback = "::1" + private val payloadSize = 1 << 20 + private val closeables = new ConcurrentLinkedQueue[AutoCloseable]() + private val pool = Executors.newCachedThreadPool(new ThreadFactory { + override def newThread(runnable: Runnable): Thread = { + val thread = new Thread(runnable, "lightgbm-network-bridge-test") + thread.setDaemon(true) + thread + } + }) + + override def afterEach(): Unit = { + closeables.asScala.foreach(resource => Try(resource.close())) + closeables.clear() + super.afterEach() + } + + private def register[T <: AutoCloseable](resource: T): T = { + closeables.add(resource) + resource + } + + private def ipv6LoopbackAvailable: Boolean = Try { + val probe = new ServerSocket() + try { + probe.bind(new InetSocketAddress(InetAddress.getByName(ipv6Loopback), 0)) + true + } finally { + probe.close() + } + }.getOrElse(false) + + private def listenOnIpv6(): ServerSocket = { + val listener = new ServerSocket() + listener.bind(new InetSocketAddress(InetAddress.getByName(ipv6Loopback), 0)) + listener.setSoTimeout(socketTimeoutMillis) + register(listener) + } + + private def listenOnLoopback(port: Int): ServerSocket = { + val listener = new ServerSocket() + listener.bind(new InetSocketAddress(InetAddress.getByName(LightGBMNetworkBridge.LoopbackHost), port)) + listener.setSoTimeout(socketTimeoutMillis) + register(listener) + } + + private def freeIpv6Port(): Int = { + val probe = new ServerSocket() + try { + probe.bind(new InetSocketAddress(InetAddress.getByName(ipv6Loopback), 0)) + probe.getLocalPort + } finally { + probe.close() + } + } + + private def connect(host: String, port: Int): Socket = { + val socket = new Socket() + socket.connect(new InetSocketAddress(InetAddress.getByName(host), port), socketTimeoutMillis) + socket.setSoTimeout(socketTimeoutMillis) + register(socket) + } + + /** Stand in for a peer opening a LightGBM link, which starts by sending its own rank. */ + private def dialAsRank(port: Int, rank: Int, host: String = "::1"): Socket = { + val socket = connect(host, port) + socket.getOutputStream.write(LightGBMNetworkRelay.rankBuffer(rank).array()) + socket.getOutputStream.flush() + socket + } + + /** Stand in for the native listener, which the bridge claims a link slot on for every lower rank. */ + private def acceptNativeSlot(listener: ServerSocket): (Int, Socket) = { + val socket = register(listener.accept()) + val bytes = new Array[Byte](LightGBMNetworkRelay.RankBytes) + new DataInputStream(socket.getInputStream).readFully(bytes) + (ByteBuffer.wrap(bytes).order(ByteOrder.nativeOrder()).getInt, socket) + } + + private def endpoint(entry: String): WorkerEndpoint = WorkerEndpoint.parse(entry) + + private def bridgedEntries(machineList: String): Seq[String] = machineList.split(",").toSeq + + private def randomPayload(size: Int): Array[Byte] = { + val payload = new Array[Byte](size) + new Random(size.toLong).nextBytes(payload) + payload + } + + private def assertPayloadCrosses(sender: Socket, receiver: Socket, payload: Array[Byte]): Unit = { + val received = pool.submit(new Callable[Array[Byte]] { + override def call(): Array[Byte] = { + val buffer = new Array[Byte](payload.length) + new DataInputStream(receiver.getInputStream).readFully(buffer) + buffer + } + }) + sender.getOutputStream.write(payload) + sender.getOutputStream.flush() + assert(received.get(socketTimeoutMillis.toLong, TimeUnit.MILLISECONDS).sameElements(payload), + "The relayed payload did not arrive intact") + } + + private def assertEndOfStream(socket: Socket, clue: String): Unit = { + val ended = try { + socket.getInputStream.read() == -1 + } catch { + // A reset is the other legitimate way for the far side to report that it is gone. + case _: IOException => true + } + assert(ended, clue) + } + + test("An IPv4 machine list never needs the bridge") { + assert(!LightGBMNetworkBridge.requiresBridge("127.0.0.1:12400,10.0.0.4:12400")) + assert(!LightGBMNetworkBridge.requiresBridge("worker-1:12400,worker-2:12401")) + assert(!LightGBMNetworkBridge.requiresBridge("")) + assert(!LightGBMNetworkBridge.requiresBridge(None.orNull)) + } + + test("Every IPv6 machine list form needs the bridge") { + assert(LightGBMNetworkBridge.requiresBridge("[2001:db8::1]:12400,[2001:db8::2]:12400")) + assert(LightGBMNetworkBridge.requiresBridge("2001:db8::1:12400")) + assert(LightGBMNetworkBridge.requiresBridge("[fe80::1%eth0]:12400")) + // A single IPv6 machine in an otherwise IPv4 list is still unreachable for the native library. + assert(LightGBMNetworkBridge.requiresBridge("10.0.0.4:12400,[2001:db8::2]:12400")) + } + + test("Machine list parsing keeps entry order and rejects malformed entries") { + val parsed = LightGBMNetworkBridge.parseMachineList(" [2001:db8::2]:12401 ,10.0.0.4:12400 ") + assert(parsed.map(_.wireString) == Seq("[2001:db8::2]:12401", "10.0.0.4:12400")) + assert(intercept[IllegalArgumentException](LightGBMNetworkBridge.parseMachineList("")).getMessage + .contains("does not contain any endpoint")) + assert(intercept[IllegalArgumentException](LightGBMNetworkBridge.parseMachineList("[2001:db8::2]")) + .getMessage.contains("missing its port")) + } + + test("A task finds its own machine list entry even when peers share its host or port") { + val entries = Seq("[2001:db8::1]:12400", "[2001:db8::2]:12400", "[2001:db8::2]:12401").map(endpoint) + assert(LightGBMNetworkBridge.findSelf(entries, "2001:db8::2", 12400) == 1) + assert(LightGBMNetworkBridge.findSelf(entries, "2001:db8::1", 12400) == 0) + assert(LightGBMNetworkBridge.findSelf(entries, "2001:db8::2", 12401) == 2) + // A host written in a different but equivalent form still resolves to the same entry. + assert(LightGBMNetworkBridge.findSelf(entries, "2001:0db8:0000:0000:0000:0000:0000:0002", 12401) == 2) + // A unique port identifies the entry even when the reported host was not preserved. + assert(LightGBMNetworkBridge.findSelf(entries, "", 12401) == 2) + } + + test("A machine list without this task's endpoint fails with an actionable error") { + val entries = Seq("[2001:db8::1]:12400", "[2001:db8::2]:12400").map(endpoint) + val failure = intercept[IllegalArgumentException](LightGBMNetworkBridge.findSelf(entries, "2001:db8::9", 12999)) + assert(failure.getMessage.contains("does not contain this task's own endpoint")) + assert(failure.getMessage.contains("[2001:db8::9]:12999")) + } + + test("The bridged machine list keeps entry order, pins the rank, and only relays IPv6 peers") { + assume(ipv6LoopbackAvailable, "IPv6 loopback is not available on this machine") + val advertisedPort = freeIpv6Port() + val peerPort = freeIpv6Port() + val machineList = s"[$ipv6Loopback]:$peerPort,10.0.0.4:12400,[$ipv6Loopback]:$advertisedPort" + val bridge = register(LightGBMNetworkBridge.open(machineList, ipv6Loopback, advertisedPort, log)) + + val bridged = bridge.bridgedNetwork + val entries = bridgedEntries(bridged.machineList) + assert(entries.head == s"${LightGBMNetworkBridge.RankPrefix}2", "The rank has to be pinned for the native library") + assert(entries.length == 4) + assert(bridged.machineCount == 3, "The bridge must not change the number of machines") + assert(bridged.rank == 2) + // The IPv4 peer stays a direct native connection; the IPv6 peer and this task are relayed. + assert(endpoint(entries(1)).host == LightGBMNetworkBridge.LoopbackHost) + assert(entries(2) == "10.0.0.4:12400") + assert(endpoint(entries(3)) == + WorkerEndpoint.parse(s"${LightGBMNetworkBridge.LoopbackHost}:${bridged.localListenPort}")) + assert(bridged.localListenPort != advertisedPort, + "The native listener needs its own port, because the bridge owns the advertised one") + } + + test("The bridge claims a native link slot for every lower rank as soon as the port is bound") { + assume(ipv6LoopbackAvailable, "IPv6 loopback is not available on this machine") + // Rank 2 of 3, so the native listener would accept two links before closing itself. + val advertisedPort = freeIpv6Port() + val machineList = s"[$ipv6Loopback]:${freeIpv6Port()},[$ipv6Loopback]:${freeIpv6Port()}," + + s"[$ipv6Loopback]:$advertisedPort" + val bridge = register(LightGBMNetworkBridge.open(machineList, ipv6Loopback, advertisedPort, log)) + val nativeListener = listenOnLoopback(bridge.bridgedNetwork.localListenPort) + + // No peer has connected yet, so these can only be the bridge claiming the slots from loopback. + val claimed = (1 to 2).map(_ => acceptNativeSlot(nativeListener)) + assert(claimed.map { case (rank, _) => rank }.toSet == Set(0, 1), + "The bridge has to identify each claimed slot with the rank that will use it") + claimed.foreach { case (_, socket) => + assert(socket.getInetAddress.isLoopbackAddress, "A native link slot was claimed from off the machine") + } + } + + test("Peer traffic arriving over IPv6 reaches the native listener and flows both ways") { + assume(ipv6LoopbackAvailable, "IPv6 loopback is not available on this machine") + val advertisedPort = freeIpv6Port() + val peerPort = freeIpv6Port() + val machineList = s"[$ipv6Loopback]:$peerPort,[$ipv6Loopback]:$advertisedPort" + val bridge = register(LightGBMNetworkBridge.open(machineList, ipv6Loopback, advertisedPort, log)) + val bridged = bridge.bridgedNetwork + + // Stand in for the native LightGBM listener, which only ever binds an IPv4 socket. + val nativeListener = listenOnLoopback(bridged.localListenPort) + val (claimedRank, nativeConnection) = acceptNativeSlot(nativeListener) + assert(claimedRank == 0) + + val peerConnection = dialAsRank(advertisedPort, 0) + assertPayloadCrosses(peerConnection, nativeConnection, randomPayload(payloadSize)) + assertPayloadCrosses(nativeConnection, peerConnection, randomPayload(payloadSize)) + + // LightGBM ends a link by closing it, so the half close has to reach the peer. + nativeConnection.shutdownOutput() + assertEndOfStream(peerConnection, "The peer never saw the native end of stream") + } + + test("Native traffic to an IPv6 peer is relayed to that peer's real address") { + assume(ipv6LoopbackAvailable, "IPv6 loopback is not available on this machine") + val peerListener = listenOnIpv6() + val advertisedPort = freeIpv6Port() + val machineList = s"[$ipv6Loopback]:$advertisedPort,[$ipv6Loopback]:${peerListener.getLocalPort}" + val bridge = register(LightGBMNetworkBridge.open(machineList, ipv6Loopback, advertisedPort, log)) + val bridged = bridge.bridgedNetwork + + // The native library dials the loopback entry the bridge published for machine 1. + val relayPort = endpoint(bridgedEntries(bridged.machineList)(2)).port + val nativeConnection = connect(LightGBMNetworkBridge.LoopbackHost, relayPort) + val peerConnection = register(peerListener.accept()) + + assertPayloadCrosses(nativeConnection, peerConnection, randomPayload(payloadSize)) + assertPayloadCrosses(peerConnection, nativeConnection, randomPayload(payloadSize)) + } + + test("A rank handshake in the LightGBM wire form survives the bridge") { + assume(ipv6LoopbackAvailable, "IPv6 loopback is not available on this machine") + // An outbound link is forwarded verbatim, so the peer reads the rank the native library sent. + val peerListener = listenOnIpv6() + val advertisedPort = freeIpv6Port() + val machineList = s"[$ipv6Loopback]:$advertisedPort,[$ipv6Loopback]:${peerListener.getLocalPort}" + val bridge = register(LightGBMNetworkBridge.open(machineList, ipv6Loopback, advertisedPort, log)) + val relayPort = endpoint(bridgedEntries(bridge.bridgedNetwork.machineList)(2)).port + + val nativeConnection = connect(LightGBMNetworkBridge.LoopbackHost, relayPort) + nativeConnection.getOutputStream.write(LightGBMNetworkRelay.rankBuffer(0).array()) + nativeConnection.getOutputStream.flush() + + val peerConnection = register(peerListener.accept()) + val rankBytes = new Array[Byte](LightGBMNetworkRelay.RankBytes) + new DataInputStream(peerConnection.getInputStream).readFully(rankBytes) + assert(ByteBuffer.wrap(rankBytes).order(ByteOrder.nativeOrder()).getInt == 0) + } + + test("A relayed connection whose far side never answers is closed instead of stalling LightGBM") { + assume(ipv6LoopbackAvailable, "IPv6 loopback is not available on this machine") + val deadPeerPort = freeIpv6Port() + val advertisedPort = freeIpv6Port() + val machineList = s"[$ipv6Loopback]:$advertisedPort,[$ipv6Loopback]:$deadPeerPort" + val bridge = register( + LightGBMNetworkBridge.open(machineList, ipv6Loopback, advertisedPort, log, connectTimeoutMillis = 200L)) + val relayPort = endpoint(bridgedEntries(bridge.bridgedNetwork.machineList)(2)).port + + val nativeConnection = connect(LightGBMNetworkBridge.LoopbackHost, relayPort) + // Nothing is listening on the peer port, so the relay gives up and closes the native side. + assertEndOfStream(nativeConnection, + "The bridge kept a relayed connection open even though its far side was unreachable") + } + + test("An IPv6 link-local peer is only accepted with a zone this machine can resolve") { + val unscoped = intercept[IllegalArgumentException]( + LightGBMNetworkBridge.resolvePeer(endpoint("[fe80::1]:12400"), log)) + assert(unscoped.getMessage.contains("has to advertise a zone identifier")) + + val unknownZone = intercept[IllegalArgumentException]( + LightGBMNetworkBridge.resolvePeer(endpoint("[fe80::1%no-such-interface]:12400"), log)) + assert(unknownZone.getMessage.contains("does not name an interface on this machine")) + + // A zone this machine does know keeps its scope all the way through resolution. + val scopedZone = NetworkInterface.getNetworkInterfaces.asScala.map(_.getName) + .find(name => Try(InetAddress.getByName(s"fe80::1%$name")).isSuccess) + scopedZone.foreach { zone => + val resolved = LightGBMNetworkBridge.resolvePeer(endpoint(s"[fe80::1%$zone]:12400"), log) + assert(resolved.isLinkLocalAddress) + assert(resolved.getHostAddress.contains("%"), "The resolved link-local address lost its zone") + } + } + + test("A numeric IPv6 scope is normalized before it is published and rejected when a peer sends one") { + // An interface index only means something on the machine that produced it. + val named = NetworkInterface.getNetworkInterfaces.asScala.find(_.getIndex > 0) + named.foreach { candidate => + val normalized = WorkerEndpoint.normalizeHost(s"fe80::1%${candidate.getIndex}") + assert(normalized == s"fe80::1%${candidate.getName}", + s"A numeric scope was published as is instead of as an interface name") + } + assert(WorkerEndpoint.normalizeHost("10.0.0.4") == "10.0.0.4") + assert(WorkerEndpoint.normalizeHost("fe80::1%eth0") == "fe80::1%eth0") + + val numericPeer = intercept[IllegalArgumentException]( + LightGBMNetworkBridge.resolvePeer(endpoint("[fe80::1%3]:12400"), log)) + assert(numericPeer.getMessage.contains("numeric interface index")) + } + + test("A JVM pinned to the IPv4 stack is told why it cannot join an IPv6 network") { + val property = "java.net.preferIPv4Stack" + val previous = Option(System.getProperty(property)) + try { + System.setProperty(property, "true") + val failure = intercept[IllegalStateException]( + LightGBMNetworkBridge.open("[2001:db8::1]:12400,[2001:db8::2]:12400", "2001:db8::1", 12400, log)) + assert(failure.getMessage.contains("-Djava.net.preferIPv4Stack=true")) + } finally { + previous.map(value => System.setProperty(property, value)).getOrElse(System.clearProperty(property)) + } + } + + test("Closing the bridge releases the advertised port and its relay ports") { + assume(ipv6LoopbackAvailable, "IPv6 loopback is not available on this machine") + val advertisedPort = freeIpv6Port() + val peerPort = freeIpv6Port() + val machineList = s"[$ipv6Loopback]:$advertisedPort,[$ipv6Loopback]:$peerPort" + val bridge = LightGBMNetworkBridge.open(machineList, ipv6Loopback, advertisedPort, log) + val relayPort = endpoint(bridgedEntries(bridge.bridgedNetwork.machineList)(2)).port + + bridge.close() + bridge.close() // The training path can close a bridge more than once. + + Seq(advertisedPort, relayPort).foreach { port => + val rebound = new ServerSocket() + try { + rebound.bind(new InetSocketAddress(port)) + assert(rebound.isBound) + } finally { + rebound.close() + } + } + } +} diff --git a/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/split1/TrainUtilsSuite.scala b/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/split1/TrainUtilsSuite.scala index 277725c9dfd..c89f6140af3 100644 --- a/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/split1/TrainUtilsSuite.scala +++ b/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/split1/TrainUtilsSuite.scala @@ -3,11 +3,14 @@ package com.microsoft.azure.synapse.ml.lightgbm.split1 -import com.microsoft.azure.synapse.ml.lightgbm.{LightGBMRegressor, TrainUtils} +import com.microsoft.azure.synapse.ml.lightgbm.{LightGBMRegressor, NetworkManager, TrainUtils} import org.scalatest.funsuite.AnyFunSuite +import org.slf4j.LoggerFactory class TrainUtilsSuite extends AnyFunSuite { + private val log = LoggerFactory.getLogger(classOf[TrainUtilsSuite]) + private val lowerIsBetterMetrics = Seq( "rmse", "l1", @@ -61,6 +64,75 @@ class TrainUtilsSuite extends AnyFunSuite { } } + test("Main worker endpoint parser keeps hostname and IPv4 behavior") { + val endpoints = Seq( + "worker.example.test:12404" -> ("worker.example.test", 12404), + "localhost:80" -> ("localhost", 80), + "192.0.2.10:65535" -> ("192.0.2.10", 65535), + "127.0.0.1:00080" -> ("127.0.0.1", 80)) + + endpoints.foreach { case (endpoint, expected) => + assert(NetworkManager.parseHostAndPort(endpoint) == expected) + assert(NetworkManager.getMainWorkerPort(s"$endpoint,backup.example.test:12405", log) == expected._2) + } + } + + test("Main worker endpoint parser supports bracketed and practical bare IPv6") { + val endpoints = Seq( + "[2001:db8::1]:12404" -> ("2001:db8::1", 12404), + "2001:db8::1:12404" -> ("2001:db8::1", 12404), + "2001:db8:0:1:2:3:4:5:12404" -> ("2001:db8:0:1:2:3:4:5", 12404), + "[::1]:443" -> ("::1", 443), + "::1:12404" -> ("::1", 12404), + "2001:db8:::12404" -> ("2001:db8::", 12404), + "[fe80::a%eth0]:12404" -> ("fe80::a%eth0", 12404), + "fe80::a%3:12404" -> ("fe80::a%3", 12404)) + + endpoints.foreach { case (endpoint, expected) => + assert(NetworkManager.parseHostAndPort(endpoint) == expected) + assert(NetworkManager.getMainWorkerPort(endpoint, log) == expected._2) + } + } + + test("Main worker endpoint parser rejects malformed hosts and ports with actionable errors") { + val malformedEndpoints = Seq( + "" -> "endpoint is empty", + "worker.example.test" -> "missing ':' port separator", + ":12404" -> "host is empty", + "worker.example.test:" -> "port is empty", + "worker.example.test:not-a-port" -> "not a decimal integer", + "worker.example.test:+80" -> "not a decimal integer", + "worker.example.test:-1" -> "not a decimal integer", + "worker.example.test:0" -> "outside the valid range", + "worker.example.test:65536" -> "outside the valid range", + "worker.example.test:999999999999999999999" -> "too large", + "[2001:db8::1]12404" -> "must be followed by a ':' port separator", + "[2001:db8::1" -> "missing its closing ']'", + "[2001:db8::1]" -> "missing its port", + "[worker.example.test]:12404" -> "brackets are only valid around an IPv6 literal", + "2001:db8::1" -> "bare IPv6 endpoint is ambiguous", + "2001:db8::1:10" -> "bare IPv6 endpoint is ambiguous", + "2001:db8:::1:12404" -> "not a valid IPv6 literal", + "fe80::1%:12404" -> "IPv6 zone identifier is empty", + "fe80::1%eth0%extra:12404" -> "IPv6 zone identifier is malformed", + "worker name:12404" -> "host contains whitespace, control characters, or an endpoint delimiter") + + malformedEndpoints.foreach { case (endpoint, expectedMessage) => + val failure = intercept[IllegalArgumentException](NetworkManager.parseHostAndPort(endpoint)) + assert(failure.getMessage.contains(expectedMessage)) + assert(failure.getMessage.contains("Expected hostname:port")) + } + + val nullFailure = intercept[IllegalArgumentException] { + NetworkManager.parseHostAndPort(null) //scalastyle:ignore null + } + assert(nullFailure.getMessage.contains("endpoint is null")) + assert(intercept[IllegalArgumentException](NetworkManager.getMainWorkerPort( + null, log)).getMessage.contains("network node list is null")) //scalastyle:ignore null + assert(intercept[IllegalArgumentException](NetworkManager.getMainWorkerPort( + ",worker.example.test:12404", log)).getMessage.contains("endpoint is empty")) + } + test("Zero early stopping rounds disable wrapper early stopping") { assert(!TrainUtils.shouldStopEarly( iteration = 100, diff --git a/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/split1/VerifyLightGBMCommon.scala b/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/split1/VerifyLightGBMCommon.scala index c4169cd5bd6..a57b5427f03 100644 --- a/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/split1/VerifyLightGBMCommon.scala +++ b/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/split1/VerifyLightGBMCommon.scala @@ -6,10 +6,15 @@ package com.microsoft.azure.synapse.ml.lightgbm.split1 import com.microsoft.azure.synapse.ml.core.test.base.TestBase import com.microsoft.azure.synapse.ml.lightgbm._ import com.microsoft.azure.synapse.ml.lightgbm.dataset.{ChunkedArrayUtils, SampledData} +import com.microsoft.azure.synapse.ml.lightgbm.params.{BaseTrainParams, ExecutionParams} import com.microsoft.azure.synapse.ml.lightgbm.swig.{DoubleChunkedArray, DoubleSwigArray, IntSwigArray, SwigUtils} import com.microsoft.ml.lightgbm.{SWIGTYPE_p_p_void, SWIGTYPE_p_void, lightgbmlib} -import org.apache.spark.ml.linalg.{DenseVector, SparseVector} +import org.apache.spark.ml.attribute.{Attribute, AttributeGroup, NumericAttribute} +import org.apache.spark.ml.linalg.{DenseVector, SparseVector, Vectors} import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.types.StructField + +import scala.util.{Failure, Success, Try} // scalastyle:off magic.number // scalastyle:off method.length @@ -18,6 +23,24 @@ class VerifyLightGBMCommon extends TestBase with LightGBMTestUtils { lazy val taskDF: DataFrame = loadBinary("task.train.csv", "TaskFailed10").cache() lazy val pimaDF: DataFrame = loadBinary("PimaIndian.csv", "Diabetes mellitus").cache() + /** Builds a tiny 4-row frame whose features vectors have `numFeatures` columns. */ + private def makeDuplicateNameDF(numFeatures: Int): DataFrame = { + val rows = Seq(0.0, 1.0, 0.0, 1.0).zipWithIndex.map { case (label, row) => + (label, Vectors.dense(Array.tabulate(numFeatures)(col => (row + col + 1).toDouble))) + } + spark.createDataFrame(rows).toDF(labelCol, featuresCol) + } + + /** A fresh minimal classifier per call, so slot names from one test cannot leak into another. */ + private def duplicateNameModel: LightGBMClassifier = new LightGBMClassifier() + .setFeaturesCol(featuresCol) + .setLabelCol(labelCol) + .setDefaultListenPort(getAndIncrementPort()) + .setNumLeaves(5) + .setNumIterations(5) + .setObjective("binary") + .setDataTransferMode(LightGBMConstants.StreamingDataTransferMode) + lazy val baseModel: LightGBMClassifier = new LightGBMClassifier() .setFeaturesCol(featuresCol) .setRawPredictionCol(rawPredCol) @@ -294,4 +317,378 @@ class VerifyLightGBMCommon extends TestBase with LightGBMTestUtils { (conv(up.last) + conv(down.head)) / fromInt(2) } } + + test("Verify duplicate feature names are handled correctly") { + // Regression test: LightGBM rejects a Dataset whose feature names repeat, failing with + // "Feature (Column_) appears more than one time". Spark can surface repeated names through + // AttributeGroup metadata on the features column, so SynapseML de-duplicates them first. + val attrs: Array[Attribute] = Array( + NumericAttribute.defaultAttr.withName("Column_").withIndex(0), + NumericAttribute.defaultAttr.withName("Column_").withIndex(1), + NumericAttribute.defaultAttr.withName("Column_").withIndex(2), + NumericAttribute.defaultAttr.withName("unique_col").withIndex(3)) + val attrGroup = new AttributeGroup(featuresCol, attrs) + + val df = makeDuplicateNameDF(4) + val dfWithDuplicateNames = df.withColumn( + featuresCol, + df(featuresCol).as(featuresCol, attrGroup.toMetadata())) + + val predictions = duplicateNameModel.fit(dfWithDuplicateNames).transform(dfWithDuplicateNames) + assert(predictions.count() == 4) + } + + test("Verify explicit slotNames parameter is used") { + val df = makeDuplicateNameDF(3) + val model = duplicateNameModel.setSlotNames(Array("feature_a", "feature_b", "feature_c")) + assert(model.fit(df).transform(df).count() == 4) + } + + test("Verify duplicate explicit slotNames are made unique") { + val df = makeDuplicateNameDF(3) + val model = duplicateNameModel.setSlotNames(Array("Column_", "Column_", "Column_")) + assert(model.fit(df).transform(df).count() == 4) + } + + test("Verify a generated slot name cannot collide with a later original name") { + // "Column_" repeats, so the second occurrence is renamed. A naive implementation renames it + // to "Column__1", which is already taken by the third slot, so LightGBM still fails with + // "Feature (Column__1) appears more than one time". The renamed slot must skip past every + // original name, not just the ones seen so far. + val df = makeDuplicateNameDF(3) + val model = duplicateNameModel.setSlotNames(Array("Column_", "Column_", "Column__1")) + assert(model.fit(df).transform(df).count() == 4) + } + + test("Verify names differing only by space vs underscore are made unique") { + // LightGBM replaces spaces with underscores before checking for duplicates, so "a b" and + // "a_b" are the same feature natively and fail with "Feature (a_b) appears more than one + // time" even though the two strings differ in Scala. + val df = makeDuplicateNameDF(3) + val model = duplicateNameModel.setSlotNames(Array("a b", "a_b", "c")) + assert(model.fit(df).transform(df).count() == 4) + } + + test("Verify slotNames of the wrong length are skipped rather than read out of bounds") { + // LGBM_DatasetSetFeatureNames reads numCols entries from the array, so a short slotNames + // array is an out-of-bounds native read. slotNames is user-supplied and never length-checked + // upstream, so LightGBMDataset.setFeatureNames guards every dataset-naming path. Training + // proceeds with LightGBM's own generated names instead of crashing the executor. + val df = makeDuplicateNameDF(4) + val model = duplicateNameModel.setSlotNames(Array("only_one_name")) + assert(model.fit(df).transform(df).count() == 4) + } + + test("Verify slotNames of the wrong length are skipped in bulk mode too") { + val df = makeDuplicateNameDF(4) + val model = duplicateNameModel + .setDataTransferMode(LightGBMConstants.BulkDataTransferMode) + .setSlotNames(Array("a", "b")) + assert(model.fit(df).transform(df).count() == 4) + } + + /** getTrainParams only reads attribute metadata off the features field, so any schema will do. */ + private lazy val deviceFeaturesSchema: StructField = makeDuplicateNameDF(3).schema(featuresCol) + + private def paramTokens(params: BaseTrainParams): Set[String] = params.toString.split(" ").toSet + + private class DeviceParameterProbe extends LightGBMClassifier { + def datasetCreationParams: String = getDatasetCreationParams(Array.empty[Int], 1) + def effectiveDeviceType: String = getEffectiveDeviceType + } + + test("Verify deviceType preserves the nine-field ExecutionParams case class API") { + val original = ExecutionParams(10000, "auto", 4, "streaming", "global", 100000, 100, false, 16) + val copied: ExecutionParams = original.copy(numThreads = 8) + val ExecutionParams(chunkSize, matrixType, copiedNumThreads, dataTransferMode, samplingMode, + samplingSetSize, microBatchSize, useSingleDatasetMode, maxStreamingOMPThreads) = copied + + assert(original.productArity == 9) + assert((chunkSize, matrixType, copiedNumThreads, dataTransferMode, samplingMode, + samplingSetSize, microBatchSize, useSingleDatasetMode, maxStreamingOMPThreads) == + (10000, "auto", 8, "streaming", "global", 100000, 100, false, 16)) + assert(classOf[ExecutionParams].getConstructors.exists(_.getParameterCount == 9)) + assert(classOf[ExecutionParams].getMethods.exists(method => + method.getName == "copy" && method.getParameterCount == 9)) + assert(ExecutionParams.getClass.getMethods.exists(method => + method.getName == "apply" && method.getParameterCount == 9)) + } + + test("Verify deviceType reaches the LightGBM parameter string for every learner") { + val classifier = new LightGBMClassifier().setDeviceType(LightGBMConstants.GPUDeviceType) + val regressor = new LightGBMRegressor().setDeviceType(LightGBMConstants.GPUDeviceType) + val ranker = new LightGBMRanker().setDeviceType(LightGBMConstants.GPUDeviceType) + assert(paramTokens(classifier.getTrainParams(1, deviceFeaturesSchema, 2)).contains("device_type=gpu")) + assert(paramTokens(regressor.getTrainParams(1, deviceFeaturesSchema, 2)).contains("device_type=gpu")) + assert(paramTokens(ranker.getTrainParams(1, deviceFeaturesSchema, 2)).contains("device_type=gpu")) + } + + test("Verify non-default deviceType reaches Dataset creation while cpu remains unchanged") { + val cpu = new DeviceParameterProbe + val gpu = new DeviceParameterProbe().setDeviceType(LightGBMConstants.GPUDeviceType) + assert(!cpu.datasetCreationParams.contains("device_type")) + assert(gpu.datasetCreationParams.split(" ").contains("device_type=gpu")) + } + + test("Verify the default cpu deviceType leaves the LightGBM parameter string untouched") { + // cpu is LightGBM's own default, so emitting it would only risk overriding a "device" alias + // that an existing caller passed through passThroughArgs. + Seq(new LightGBMClassifier().getTrainParams(1, deviceFeaturesSchema, 2), + new LightGBMRegressor().getTrainParams(1, deviceFeaturesSchema, 2), + new LightGBMRanker().getTrainParams(1, deviceFeaturesSchema, 2)) + .foreach(params => assert(!params.toString.contains("device_type"))) + } + + test("Verify passThroughArgs device aliases take precedence over deviceType") { + Seq("device_type=cuda", "device=cuda").foreach { args => + val learner = new DeviceParameterProbe() + .setPassThroughArgs(args) + .setDeviceType(LightGBMConstants.GPUDeviceType) + val parameters = learner.getTrainParams(1, deviceFeaturesSchema, 2).toString + assert(parameters.startsWith(args)) + assert(!parameters.contains("device_type=gpu")) + assert(learner.effectiveDeviceType == LightGBMConstants.CUDADeviceType) + assert(learner.datasetCreationParams.split(" ").contains("device_type=cuda")) + } + } + + test("Verify canonical device_type cuda wins over device gpu in either order") { + Seq("device=gpu device_type=cuda", "device_type=cuda device=gpu").foreach { args => + val learner = new DeviceParameterProbe() + .setPassThroughArgs(args) + .setDeviceType(LightGBMConstants.GPUDeviceType) + assert(learner.effectiveDeviceType == LightGBMConstants.CUDADeviceType) + assert(learner.datasetCreationParams.split(" ").contains("device_type=cuda")) + val error = intercept[IllegalArgumentException] { + learner + .setLabelCol(labelCol) + .setFeaturesCol(featuresCol) + .setNumTasks(1) + .fit(deviceTrainingDF) + } + assert(error.getMessage.contains("missing CUDA metadata")) + } + } + + test("Verify canonical device_type gpu wins over device cuda in either order") { + Seq("device=cuda device_type=gpu", "device_type=gpu device=cuda").foreach { args => + val learner = new DeviceParameterProbe() + .setPassThroughArgs(args) + .setDeviceType(LightGBMConstants.CPUDeviceType) + assert(learner.effectiveDeviceType == LightGBMConstants.GPUDeviceType) + assert(learner.datasetCreationParams.split(" ").contains("device_type=gpu")) + } + } + + test("Verify mixed-case canonical and alias CUDA values are rejected") { + Seq("device_type=CUDA", + "device=CuDa", + "device=GPU device_type=CuDa", + "device_type=cUdA device=GPU").foreach { args => + val learner = new DeviceParameterProbe() + .setPassThroughArgs(args) + .setDeviceType(LightGBMConstants.CPUDeviceType) + assert(learner.effectiveDeviceType == LightGBMConstants.CUDADeviceType) + assert(learner.datasetCreationParams.split(" ").contains("device_type=cuda")) + val error = intercept[IllegalArgumentException] { + learner + .setLabelCol(labelCol) + .setFeaturesCol(featuresCol) + .setNumTasks(1) + .fit(deviceTrainingDF) + } + assert(error.getMessage.contains("missing CUDA metadata")) + } + } + + test("Verify mixed-case GPU values resolve safely") { + Seq("device=GpU", + "device=CuDa device_type=GPU", + "device_type=gPu device=CUDA").foreach { args => + val learner = new DeviceParameterProbe() + .setPassThroughArgs(args) + .setDeviceType(LightGBMConstants.CPUDeviceType) + assert(learner.effectiveDeviceType == LightGBMConstants.GPUDeviceType) + assert(learner.datasetCreationParams.split(" ").contains("device_type=gpu")) + } + } + + test("Verify quoted CUDA keys and values are normalized and rejected") { + val quotedCudaArgs = Seq( + """device_type="CUDA"""", + "device_type='CuDa'", + """"device_type"=cUdA""", + "'device_type'=CUDA", + """device="CuDa"""", + "device='CUDA'", + """"device"='cUdA'""", + """"device_type'=CUDA"""", + """device_type='CuDa"""") + quotedCudaArgs.foreach { args => + val learner = new DeviceParameterProbe() + .setPassThroughArgs(args) + .setDeviceType(LightGBMConstants.GPUDeviceType) + assert(learner.getPassThroughArgs == args) + assert(learner.effectiveDeviceType == LightGBMConstants.CUDADeviceType) + assert(learner.datasetCreationParams.split(" ").contains("device_type=cuda")) + } + Seq("""device_type="CUDA"""", "device='CuDa'").foreach { args => + val error = intercept[IllegalArgumentException] { + new DeviceParameterProbe() + .setPassThroughArgs(args) + .setLabelCol(labelCol) + .setFeaturesCol(featuresCol) + .setNumTasks(1) + .fit(deviceTrainingDF) + } + assert(error.getMessage.contains("missing CUDA metadata")) + } + } + + test("Verify quoted canonical device_type retains precedence over quoted device alias") { + Seq( + """"device"='gpu' 'device_type'="CuDa"""", + """'device_type'="CUDA" "device"='gpu'""").foreach { args => + val learner = new DeviceParameterProbe().setPassThroughArgs(args) + assert(learner.getPassThroughArgs == args) + assert(learner.effectiveDeviceType == LightGBMConstants.CUDADeviceType) + assert(learner.datasetCreationParams.split(" ").contains("device_type=cuda")) + } + Seq( + """"device"='CuDa' 'device_type'="GpU"""", + """'device_type'="GPU" "device"='cuda'""").foreach { args => + val learner = new DeviceParameterProbe().setPassThroughArgs(args) + assert(learner.getPassThroughArgs == args) + assert(learner.effectiveDeviceType == LightGBMConstants.GPUDeviceType) + assert(learner.datasetCreationParams.split(" ").contains("device_type=gpu")) + } + } + + test("Verify device-like text inside unrelated values is ignored") { + Seq("note=device_type=cuda", """note="device_type=CUDA"""", "path=device_cuda").foreach { args => + val learner = new DeviceParameterProbe().setPassThroughArgs(args) + assert(learner.getPassThroughArgs == args) + assert(learner.effectiveDeviceType == LightGBMConstants.CPUDeviceType) + assert(!learner.datasetCreationParams.contains("device_type")) + } + } + + test("Verify shared device parser matches native-effective syntax") { + val cases = Seq( + """device_type="CUDA"""" -> Some(LightGBMConstants.CUDADeviceType), + """'device'='GpU'""" -> Some(LightGBMConstants.GPUDeviceType), + """device="cuda" "device_type"='GPU'""" -> Some(LightGBMConstants.GPUDeviceType), + """'device_type'='CuDa' device="gpu"""" -> Some(LightGBMConstants.CUDADeviceType), + "device cuda" -> None, + "device_type gpu" -> None, + "note=device_type=cuda" -> None, + """note="device=CUDA"""" -> None) + cases.foreach { case (parameters, expected) => + assert(LightGBMUtils.effectiveDeviceType(parameters) == expected) + assert(LightGBMUtils.hasDeviceParameter(parameters) == expected.isDefined) + } + } + + test("Verify booster guidance uses the shared effective device parser") { + Seq( + """device_type="CUDA"""" -> "device_type=cuda", + """'device'='GpU'""" -> "device_type=gpu", + """device='cuda' "device_type"="GPU"""" -> "device_type=gpu").foreach { + case (parameters, expectedDevice) => + val guidance = LightGBMUtils.boosterFailureGuidance(parameters) + assert(guidance.contains(s"Requested $expectedDevice")) + } + Seq("device cuda", "device_type gpu", "note=device_type=cuda", "device=cpu").foreach { parameters => + assert(LightGBMUtils.boosterFailureGuidance(parameters).isEmpty) + } + } + + test("Verify deviceType rejects an unsupported device") { + val cudaError = intercept[IllegalArgumentException] { + new LightGBMClassifier().setDeviceType(LightGBMConstants.CUDADeviceType) + } + assert(cudaError.getMessage.contains("can crash Spark executors")) + assertThrows[IllegalArgumentException](new LightGBMClassifier().setDeviceType("tpu")) + } + + test("Verify generic Params set validates deviceType like generated wrappers") { + val learner = new LightGBMClassifier() + learner.set(learner.deviceType, LightGBMConstants.CPUDeviceType) + assert(learner.getDeviceType == LightGBMConstants.CPUDeviceType) + learner.set(learner.deviceType, LightGBMConstants.GPUDeviceType) + assert(learner.getDeviceType == LightGBMConstants.GPUDeviceType) + assertThrows[IllegalArgumentException] { + learner.set(learner.deviceType, LightGBMConstants.CUDADeviceType) + } + } + + private lazy val deviceTrainingDF: DataFrame = { + import spark.implicits._ + Seq((0.0, Vectors.dense(1.0, 2.0, 3.0)), + (1.0, Vectors.dense(4.0, 5.0, 6.0)), + (0.0, Vectors.dense(1.5, 2.5, 3.5)), + (1.0, Vectors.dense(4.5, 5.5, 6.5)), + (0.0, Vectors.dense(1.2, 2.2, 3.2)), + (1.0, Vectors.dense(4.2, 5.2, 6.2))) + .toDF(labelCol, featuresCol) + } + + private def baseDeviceClassifier: LightGBMClassifier = + new LightGBMClassifier() + .setLabelCol(labelCol) + .setFeaturesCol(featuresCol) + .setNumLeaves(2) + .setNumIterations(2) + .setNumTasks(1) + .setNumThreads(1) + .setSeed(42) + + private def deviceClassifier(device: String): LightGBMClassifier = + baseDeviceClassifier.setDeviceType(device) + + private def rootCause(t: Throwable): Throwable = + Iterator.iterate(t)(_.getCause).takeWhile(_ != null).toList.last + + test("Verify the default cpu deviceType leaves training output unchanged") { + val defaultClassifier = baseDeviceClassifier + val explicitCpuClassifier = deviceClassifier(LightGBMConstants.CPUDeviceType) + assert(defaultClassifier.getTrainParams(1, deviceFeaturesSchema, 2).toString == + explicitCpuClassifier.getTrainParams(1, deviceFeaturesSchema, 2).toString) + + val defaultModel = defaultClassifier.fit(deviceTrainingDF) + val explicitCpuModel = explicitCpuClassifier.fit(deviceTrainingDF) + assert(defaultModel.getModel.modelStr == explicitCpuModel.getModel.modelStr) + } + + /** The published lightgbmlib artifact bundles a CPU-only native library, so gpu and cuda cannot + * be trained here. What must hold on any build is that the request reaches LightGBM instead of + * being quietly dropped: a caller who asks for a GPU and unknowingly gets CPU has no way to + * tell, and would take the slowdown as a fact of life. LightGBM refusing by name is proof the + * parameter arrived. If the native library ever does gain GPU support the fit simply succeeds, + * which satisfies the same property. + */ + private def assertDeviceIsHonored(device: String, learner: String): Unit = { + val outcome = Try(deviceClassifier(device).fit(deviceTrainingDF).transform(deviceTrainingDF).count()) + outcome match { + case Success(count) => assert(count == deviceTrainingDF.count()) + case Failure(t) => + val message = rootCause(t).getMessage + assert(message != null && message.contains(learner) && + message.contains(s"Requested device_type=$device") && + message.contains("bundled LightGBM native libraries are CPU-only") && + message.contains("java.library.path"), + s"Requesting device_type=$device did not produce actionable accelerator guidance: $message") + } + } + + test("Verify the gpu deviceType is never silently downgraded to cpu") { + assertDeviceIsHonored(LightGBMConstants.GPUDeviceType, "GPU Tree Learner") + } + + test("Verify the passThroughArgs cuda alias is rejected before native training") { + val error = intercept[IllegalArgumentException] { + baseDeviceClassifier.setPassThroughArgs("device=cuda").fit(deviceTrainingDF) + } + assert(error.getMessage.contains("missing CUDA metadata")) + } } diff --git a/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/split1/WorkerWireFormatSuite.scala b/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/split1/WorkerWireFormatSuite.scala new file mode 100644 index 00000000000..7dc844763e3 --- /dev/null +++ b/lightgbm/src/test/scala/com/microsoft/azure/synapse/ml/lightgbm/split1/WorkerWireFormatSuite.scala @@ -0,0 +1,128 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.lightgbm.split1 + +import com.microsoft.azure.synapse.ml.lightgbm.{LightGBMConstants, NetworkManager, TaskMessageInfo, + WorkerEndpoint, WorkerMessage} +import org.scalatest.funsuite.AnyFunSuite + +import java.io.{BufferedReader, BufferedWriter, InputStreamReader, OutputStreamWriter} +import java.net.{InetSocketAddress, ServerSocket, Socket} +import scala.collection.mutable.ListBuffer +import scala.util.Try + +/** Covers the wire form of every endpoint LightGBM components exchange. */ +class WorkerWireFormatSuite extends AnyFunSuite { + + private val socketTimeoutMillis = 30000 + private val driverHost = "127.0.0.1" + private val timeout = 30.0 + + private class ReportingTask(host: String, port: Int, taskHost: String, listenPort: Int, partitionId: Int) + extends AutoCloseable { + private val socket = new Socket() + socket.connect(new InetSocketAddress(host, port), socketTimeoutMillis) + socket.setSoTimeout(socketTimeoutMillis) + private val reader = new BufferedReader(new InputStreamReader(socket.getInputStream)) + private val writer = new BufferedWriter(new OutputStreamWriter(socket.getOutputStream)) + + def report(): Unit = { + val message = WorkerMessage.format( + TaskMessageInfo(LightGBMConstants.EnabledTask, taskHost, listenPort, partitionId, partitionId.toString), 0) + writer.write(s"$message\n") + writer.flush() + } + + def readMachineList(): String = reader.readLine() + + override def close(): Unit = socket.close() + } + + test("Only an IPv6 host is bracketed on the wire") { + assert(WorkerEndpoint.wireString("10.0.0.4", 12400) == "10.0.0.4:12400") + assert(WorkerEndpoint.wireString("worker-1", 12400) == "worker-1:12400") + assert(WorkerEndpoint.wireString("2001:db8::1", 12400) == "[2001:db8::1]:12400") + // A zone identifier belongs inside the brackets, with the address it scopes. + assert(WorkerEndpoint.wireString("fe80::1%eth0", 12400) == "[fe80::1%eth0]:12400") + // An already bracketed host is not bracketed twice. + assert(WorkerEndpoint.wireString("[2001:db8::1]", 12400) == "[2001:db8::1]:12400") + } + + test("A wire endpoint round trips back to the host it was built from") { + Seq("10.0.0.4", "worker-1", "2001:db8::1", "fe80::1%eth0", "::1").foreach { host => + val parsed = WorkerEndpoint.parse(WorkerEndpoint.wireString(host, 12400)) + assert(parsed.host == host) + assert(parsed.port == 12400) + assert(parsed.wireString == WorkerEndpoint.wireString(host, 12400)) + } + } + + test("A host carrying a control character or a delimiter never reaches the wire") { + // The topology exchange is a line protocol over a comma delimited machine list, so any of these + // would either split one endpoint into two or forge an extra protocol line. + Seq("10.0.0.4\n", "10.0.0.4\r", "10.0.0.4\t", "10.0.0.4\u0000", "10.0.0.4 ", "10.0.0.4,10.0.0.5", + "10.0.0.4]", "[10.0.0.4").foreach { host => + val failure = intercept[IllegalArgumentException](WorkerEndpoint.wireString(host, 12400)) + assert(failure.getMessage.contains("Invalid LightGBM worker endpoint"), s"unexpected error for '$host'") + } + } + + test("A rejected endpoint reports control characters as escapes rather than raw bytes") { + val failure = intercept[IllegalArgumentException](WorkerEndpoint.wireString("10.0.0.4\r\nignore:1", 12400)) + assert(failure.getMessage.contains("\\r\\n")) + assert(!failure.getMessage.contains("\r")) + assert(!failure.getMessage.contains("\n")) + val nullByte = intercept[IllegalArgumentException](WorkerEndpoint.wireString("10.0.0.4\u0000", 12400)) + assert(nullByte.getMessage.contains("\\u0000")) + } + + test("A port outside the valid range never reaches the wire") { + Seq(0, -1, LightGBMConstants.MaxPort + 1).foreach { port => + val failure = intercept[IllegalArgumentException](WorkerEndpoint.wireString("10.0.0.4", port)) + assert(failure.getMessage.contains("Invalid LightGBM worker endpoint")) + } + } + + test("A task message round trips for every host form, including a scoped IPv6 literal") { + Seq("10.0.0.4", "2001:db8::1", "fe80::1%eth0").foreach { host => + val status = TaskMessageInfo(LightGBMConstants.EnabledTask, host, 12400, 3, "executor-2") + val message = WorkerMessage.format(status, 7) + val parsed = WorkerMessage.parse(message) + assert(parsed.taskHost == host) + assert(parsed.localListenPort == 12400) + assert(parsed.partitionId == 3) + assert(parsed.executorId == "executor-2") + assert(parsed.stageAttemptNumber == 7) + } + } + + test("A task message with a forged extra line is rejected before it is sent") { + val status = TaskMessageInfo(LightGBMConstants.EnabledTask, "10.0.0.4\nenabledTask:10.0.0.9", 12400, 0, "e") + assert(intercept[IllegalArgumentException](WorkerMessage.format(status, 0)) + .getMessage.contains("Invalid LightGBM worker endpoint")) + } + + test("The driver publishes a machine list the LightGBM network layer can split on commas") { + val serverSocket = new ServerSocket(0) + serverSocket.setSoTimeout(socketTimeoutMillis) + val manager = NetworkManager(2, serverSocket, driverHost, serverSocket.getLocalPort, timeout, + useBarrierExecutionMode = false) + val tasks = ListBuffer.empty[ReportingTask] + try { + tasks += new ReportingTask(driverHost, serverSocket.getLocalPort, "2001:db8::1", 12400, 0) + tasks += new ReportingTask(driverHost, serverSocket.getLocalPort, "2001:db8::2", 12401, 1) + tasks.foreach(_.report()) + + val machineList = tasks.head.readMachineList() + assert(machineList == "[2001:db8::1]:12400,[2001:db8::2]:12401", + "An unbracketed IPv6 machine list cannot be split into host and port by any peer") + // Every consumer of the machine list has to agree on where the first endpoint ends. + assert(NetworkManager.getMainWorkerPort(machineList, org.slf4j.LoggerFactory.getLogger(getClass)) == 12400) + manager.waitForNetworkCommunicationsDone() + } finally { + manager.closeConnections() + tasks.foreach(task => Try(task.close())) + } + } +} diff --git a/pipeline.yaml b/pipeline.yaml index f831382bb96..48ef8cfbc78 100644 --- a/pipeline.yaml +++ b/pipeline.yaml @@ -1,11 +1,19 @@ resources: -- repo: self + repositories: + - repository: SynapseML-Internal + type: git + name: A365/SynapseML-Internal + # Queue-time default only. `ref` is resolved before any job runs, so it cannot depend on + # the branch under test; the InternalCompat job re-points this checkout at the Internal + # branch matching the OSS target branch. See 'Select matching SynapseML-Internal branch'. + ref: master trigger: branches: include: - master - spark3.5 + - spark4.0 - spark4.1 paths: exclude: @@ -22,6 +30,7 @@ pr: include: - master - spark3.5 + - spark4.0 - spark4.1 paths: exclude: @@ -612,6 +621,7 @@ jobs: condition: and(succeededOrFailed(), eq(variables.runCoverage, true)) - ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/master'), startsWith(variables['Build.SourceBranch'], 'refs/tags/')) }}: - template: templates/codecov.yml + - template: templates/publish_coverage_ado.yml - job: RTests dependsOn: BuildAndCacheSbt timeoutInMinutes: 60 @@ -679,6 +689,7 @@ jobs: condition: and(succeededOrFailed(), eq(variables.runCoverage, true)) - ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/master'), startsWith(variables['Build.SourceBranch'], 'refs/tags/')) }}: - template: templates/codecov.yml + - template: templates/publish_coverage_ado.yml - job: WebsiteSamplesTests dependsOn: BuildAndCacheSbt @@ -715,6 +726,11 @@ jobs: condition: and(succeededOrFailed(), eq(variables.runCoverage, true)) - ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/master'), startsWith(variables['Build.SourceBranch'], 'refs/tags/')) }}: - template: templates/codecov.yml + # testWebsiteDocs drives Spark out-of-process, so scoverage records no measurement + # data for this job; an empty report here is expected rather than a broken glob. + - template: templates/publish_coverage_ado.yml + parameters: + failIfCoverageEmpty: false - job: UnitTests dependsOn: BuildAndCacheSbt @@ -835,6 +851,33 @@ jobs: PACKAGE: "train" vw: PACKAGE: "vw" + # Suites whose packages no other leg claims. PipelineTestCoverageSuite fails the build when + # a new suite lands in a package this matrix never names. + misc: + PACKAGE: "misc" + TEST_CLASSES: >- + com.microsoft.azure.synapse.ml.param.** + com.microsoft.azure.synapse.ml.logging.** + com.microsoft.azure.synapse.ml.fabric.** + com.microsoft.azure.synapse.ml.explainers.VerifyExplainerSharedParams + com.microsoft.azure.synapse.ml.explainers.VerifyFeatureStats + com.microsoft.azure.synapse.ml.explainers.VerifyRowUtils + com.microsoft.azure.synapse.ml.io.binary.VerifyBinaryFileFormat + com.microsoft.azure.synapse.ml.io.http.VerifyClients + com.microsoft.azure.synapse.ml.io.http.VerifyHTTPSchema + com.microsoft.azure.synapse.ml.io.http.VerifyRESTHelpers + com.microsoft.azure.synapse.ml.io.http.VerifySharedVariable + com.microsoft.azure.synapse.ml.io.image.VerifyImageUtils + com.microsoft.azure.synapse.ml.lightgbm.dataset.VerifyDatasetUtils + com.microsoft.azure.synapse.ml.nbtest.FabricTestArtifactTrackerSuite + com.microsoft.azure.synapse.ml.services.CognitiveServiceBaseSuite + com.microsoft.azure.synapse.ml.services.search.AddDocumentsHeaderPersistenceSuite + com.microsoft.azure.synapse.ml.services.search.AzureSearchAuthSuite + com.microsoft.azure.synapse.ml.services.search.AzureSearchGenericParamPersistenceSuite + com.microsoft.azure.synapse.ml.services.search.IndexSchemaLiveRoundTripSuite + com.microsoft.azure.synapse.ml.services.search.IndexSchemaParsingSuite + com.microsoft.azure.synapse.ml.services.search.SearchIndexRetentionSuite + com.microsoft.azure.synapse.ml.services.speech.SpeechToTextSDKSecuritySuite steps: - template: templates/sbt_cache.yml - template: templates/update_cli.yml @@ -877,11 +920,13 @@ jobs: displayName: Load Codecov token condition: succeededOrFailed() retryCountOnTaskFailure: 3 + continueOnError: true inputs: azureSubscription: 'SynapseML Build' keyVaultName: mmlspark-keys SecretsFilter: codecov-token - template: templates/codecov.yml + - template: templates/publish_coverage_ado.yml - job: ReleaseBranchCompat dependsOn: BuildAndCacheSbt @@ -987,8 +1032,288 @@ jobs: "${REPLAY_PATHS[@]}" > "$PATCH_PATH" test -s "$PATCH_PATH" + PREREQUISITES_CONFIG=".pipelines/release-compat-prerequisites.txt" + CONFIGURED_PREREQUISITES=() + CONFIGURED_PREREQUISITE_SCOPES=() + CONFIGURED_PREREQUISITE_IS_SCOPED=() + if git cat-file -e "$PR_MERGE_HEAD:$PREREQUISITES_CONFIG" 2>/dev/null; then + LINE_NUMBER=0 + while IFS= read -r RAW_LINE || [ -n "$RAW_LINE" ]; do + LINE_NUMBER=$((LINE_NUMBER + 1)) + LINE_WITHOUT_CR="${RAW_LINE%$'\r'}" + SCOPED_LINE="${LINE_WITHOUT_CR#"${LINE_WITHOUT_CR%%[![:space:]]*}"}" + TRIMMED_LINE="${SCOPED_LINE%"${SCOPED_LINE##*[![:space:]]}"}" + case "$TRIMMED_LINE" in + ""|\#*) + continue + ;; + esac + if [[ "$SCOPED_LINE" == *$'\t'* ]]; then + PREREQUISITE="${SCOPED_LINE%%$'\t'*}" + PREREQUISITE_SCOPE="${SCOPED_LINE#*$'\t'}" + PREREQUISITE_IS_SCOPED=true + else + PREREQUISITE="$TRIMMED_LINE" + PREREQUISITE_SCOPE="" + PREREQUISITE_IS_SCOPED=false + fi + if [[ ! "$PREREQUISITE" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "##vso[task.logissue type=error]Invalid prerequisite at $PREREQUISITES_CONFIG:$LINE_NUMBER; expected a full 40-character commit SHA optionally followed by tab-separated paths" + exit 1 + fi + if [ "$PREREQUISITE_IS_SCOPED" = true ]; then + REMAINING_PATHS="$PREREQUISITE_SCOPE" + while true; do + if [[ "$REMAINING_PATHS" == *$'\t'* ]]; then + path="${REMAINING_PATHS%%$'\t'*}" + REMAINING_PATHS="${REMAINING_PATHS#*$'\t'}" + HAS_MORE_PATHS=true + else + path="$REMAINING_PATHS" + HAS_MORE_PATHS=false + fi + if [ -z "$path" ]; then + echo "##vso[task.logissue type=error]Scoped prerequisite $PREREQUISITE contains an empty path" + exit 1 + fi + TRIMMED_PATH="${path#"${path%%[![:space:]]*}"}" + TRIMMED_PATH="${TRIMMED_PATH%"${TRIMMED_PATH##*[![:space:]]}"}" + if [ "$path" != "$TRIMMED_PATH" ]; then + echo "##vso[task.logissue type=error]Scoped prerequisite path must not have leading or trailing whitespace: $path" + exit 1 + fi + case "$path" in + /*|.|./*) + echo "##vso[task.logissue type=error]Scoped prerequisite path must be repo-relative: $path" + exit 1 + ;; + esac + case "/$path/" in + */../*) + echo "##vso[task.logissue type=error]Scoped prerequisite path must not contain traversal: $path" + exit 1 + ;; + *//*|*/./*) + echo "##vso[task.logissue type=error]Scoped prerequisite path must be normalized: $path" + exit 1 + ;; + esac + if [ "$HAS_MORE_PATHS" = false ]; then + break + fi + done + else + if ! git cat-file -e "$PREREQUISITE^{commit}" 2>/dev/null; then + echo "##vso[task.logissue type=error]Release compatibility prerequisite $PREREQUISITE is unavailable" + exit 1 + fi + if ! git merge-base --is-ancestor "$PREREQUISITE" "$TARGET_HEAD"; then + echo "##vso[task.logissue type=error]Release compatibility prerequisite $PREREQUISITE is not an ancestor of PR target $TARGET_HEAD" + exit 1 + fi + fi + CONFIGURED_PREREQUISITES+=("$PREREQUISITE") + CONFIGURED_PREREQUISITE_SCOPES+=("$PREREQUISITE_SCOPE") + CONFIGURED_PREREQUISITE_IS_SCOPED+=("$PREREQUISITE_IS_SCOPED") + done < <(git show "$PR_MERGE_HEAD:$PREREQUISITES_CONFIG") + fi + + PREREQUISITE_COMMITS=() + PREREQUISITE_PATCHES=() + for index in "${!CONFIGURED_PREREQUISITES[@]}"; do + PREREQUISITE="${CONFIGURED_PREREQUISITES[$index]}" + PREREQUISITE_SCOPE="${CONFIGURED_PREREQUISITE_SCOPES[$index]}" + PREREQUISITE_IS_SCOPED="${CONFIGURED_PREREQUISITE_IS_SCOPED[$index]}" + PREREQUISITE_PATHS=() + PREREQUISITE_PATHSPECS=() + if [ "$PREREQUISITE_IS_SCOPED" = true ]; then + REMAINING_PATHS="$PREREQUISITE_SCOPE" + while true; do + if [[ "$REMAINING_PATHS" == *$'\t'* ]]; then + path="${REMAINING_PATHS%%$'\t'*}" + REMAINING_PATHS="${REMAINING_PATHS#*$'\t'}" + HAS_MORE_PATHS=true + else + path="$REMAINING_PATHS" + HAS_MORE_PATHS=false + fi + PATH_SELECTED=false + for REPLAY_PATH in "${REPLAY_PATHS[@]}"; do + if [ "$path" = "$REPLAY_PATH" ]; then + PREREQUISITE_PATHS+=("$path") + PATH_SELECTED=true + break + fi + done + if [ "$PATH_SELECTED" = false ]; then + for ((later_index=index + 1; + later_index<${#CONFIGURED_PREREQUISITES[@]}; + later_index++)); do + if [ "${CONFIGURED_PREREQUISITE_IS_SCOPED[$later_index]}" = false ]; then + LATER_PREREQUISITE="${CONFIGURED_PREREQUISITES[$later_index]}" + if ! LATER_PARENT=$(git rev-parse "$LATER_PREREQUISITE^1" 2>/dev/null); then + echo "##vso[task.logissue type=error]Dependent release compatibility prerequisite $LATER_PREREQUISITE has no first parent" + exit 1 + fi + if git diff --quiet "$LATER_PARENT" "$LATER_PREREQUISITE" -- \ + ":(literal)$path"; then + : + else + DIFF_STATUS=$? + if [ "$DIFF_STATUS" -ne 1 ]; then + echo "##vso[task.logissue type=error]Unable to inspect dependent prerequisite $LATER_PREREQUISITE for path: $path" + exit 1 + fi + echo "Scoped prerequisite $PREREQUISITE path $path is required by later prerequisite $LATER_PREREQUISITE" + PREREQUISITE_PATHS+=("$path") + PATH_SELECTED=true + break + fi + fi + done + fi + if [ "$HAS_MORE_PATHS" = false ]; then + break + fi + done + if [ ${#PREREQUISITE_PATHS[@]} -eq 0 ]; then + echo "Scoped prerequisite $PREREQUISITE has no paths in this PR replay; skipping" + continue + fi + if ! git cat-file -e "$PREREQUISITE^{commit}" 2>/dev/null; then + echo "##vso[task.logissue type=error]Release compatibility prerequisite $PREREQUISITE is unavailable" + exit 1 + fi + if ! git merge-base --is-ancestor "$PREREQUISITE" "$TARGET_HEAD"; then + echo "##vso[task.logissue type=error]Release compatibility prerequisite $PREREQUISITE is not an ancestor of PR target $TARGET_HEAD" + exit 1 + fi + fi + if ! PREREQUISITE_PARENT=$(git rev-parse "$PREREQUISITE^1" 2>/dev/null); then + echo "##vso[task.logissue type=error]Release compatibility prerequisite $PREREQUISITE has no first parent" + exit 1 + fi + if [ "$PREREQUISITE_IS_SCOPED" = true ]; then + for path in "${PREREQUISITE_PATHS[@]}"; do + if git diff --quiet "$PREREQUISITE_PARENT" "$PREREQUISITE" -- \ + ":(literal)$path"; then + echo "##vso[task.logissue type=error]Scoped path is not changed by prerequisite $PREREQUISITE: $path" + exit 1 + else + DIFF_STATUS=$? + if [ "$DIFF_STATUS" -ne 1 ]; then + echo "##vso[task.logissue type=error]Unable to validate scoped path for prerequisite $PREREQUISITE: $path" + exit 1 + fi + fi + if git cat-file -e "$PREREQUISITE_PARENT:$path" 2>/dev/null; then + echo "##vso[task.logissue type=error]Scoped path is not an add-only baseline in prerequisite $PREREQUISITE: $path" + exit 1 + fi + if ! PREREQUISITE_BLOB=$(git rev-parse "$PREREQUISITE:$path" 2>/dev/null); then + echo "##vso[task.logissue type=error]Scoped path is absent from prerequisite $PREREQUISITE: $path" + exit 1 + fi + if [ "$(git cat-file -t "$PREREQUISITE_BLOB")" != blob ]; then + echo "##vso[task.logissue type=error]Scoped prerequisite path is not a blob: $path" + exit 1 + fi + BASELINE_COMMIT="$TARGET_HEAD" + for ((later_index=index + 1; + later_index<${#CONFIGURED_PREREQUISITES[@]}; + later_index++)); do + if [ "${CONFIGURED_PREREQUISITE_IS_SCOPED[$later_index]}" = false ]; then + LATER_PREREQUISITE="${CONFIGURED_PREREQUISITES[$later_index]}" + if ! LATER_PARENT=$(git rev-parse "$LATER_PREREQUISITE^1" 2>/dev/null); then + echo "##vso[task.logissue type=error]Dependent release compatibility prerequisite $LATER_PREREQUISITE has no first parent" + exit 1 + fi + if git diff --quiet "$LATER_PARENT" "$LATER_PREREQUISITE" -- \ + ":(literal)$path"; then + : + else + DIFF_STATUS=$? + if [ "$DIFF_STATUS" -ne 1 ]; then + echo "##vso[task.logissue type=error]Unable to inspect dependent prerequisite $LATER_PREREQUISITE for path: $path" + exit 1 + fi + BASELINE_COMMIT="$LATER_PARENT" + break + fi + fi + done + if ! BASELINE_BLOB=$(git rev-parse "$BASELINE_COMMIT:$path" 2>/dev/null); then + echo "##vso[task.logissue type=error]Scoped path is absent from expected baseline $BASELINE_COMMIT: $path" + exit 1 + fi + if [ "$PREREQUISITE_BLOB" != "$BASELINE_BLOB" ]; then + echo "##vso[task.logissue type=error]Scoped prerequisite blob does not match expected baseline $BASELINE_COMMIT: $path" + exit 1 + fi + PREREQUISITE_PATHSPECS+=(":(literal)$path") + done + else + while IFS= read -r -d '' path; do + case "$path" in + .github/*|.pipelines/*|docs/*|templates/*|tools/acr/*|tools/ci/*|tools/docker/*|tools/helm/*|website/*) + ;; + pipeline.yaml|CODEOWNERS|CONTRIBUTING.md|LICENSE|README.md|SECURITY.md) + ;; + *) + PREREQUISITE_PATHS+=("$path") + PREREQUISITE_PATHSPECS+=(":(literal)$path") + ;; + esac + done < <(git diff --name-only -z "$PREREQUISITE_PARENT" "$PREREQUISITE") + fi + + if [ ${#PREREQUISITE_PATHS[@]} -eq 0 ]; then + echo "Prerequisite $PREREQUISITE has no release-relevant paths; skipping" + continue + fi + + PREREQUISITE_PATCH="$(Agent.TempDirectory)/release-compat-prerequisite-$PREREQUISITE.patch" + git diff --binary --full-index "$PREREQUISITE_PARENT" "$PREREQUISITE" -- \ + "${PREREQUISITE_PATHSPECS[@]}" > "$PREREQUISITE_PATCH" + if [ ! -s "$PREREQUISITE_PATCH" ]; then + echo "##vso[task.logissue type=error]Prerequisite $PREREQUISITE produced an empty release patch" + exit 1 + fi + PREREQUISITE_COMMITS+=("$PREREQUISITE") + PREREQUISITE_PATCHES+=("$PREREQUISITE_PATCH") + done + echo "=== Attempting to apply release-relevant PR changes onto $(RELEASE_BRANCH) ===" git checkout --detach $RELEASE_TIP + git reset --hard $RELEASE_TIP + git update-index --refresh + for index in "${!PREREQUISITE_COMMITS[@]}"; do + PREREQUISITE="${PREREQUISITE_COMMITS[$index]}" + PREREQUISITE_PATCH="${PREREQUISITE_PATCHES[$index]}" + echo "=== Applying release compatibility prerequisite $PREREQUISITE ===" + if git apply --reverse --check --index "$PREREQUISITE_PATCH" >/dev/null 2>&1; then + echo "Prerequisite $PREREQUISITE is already present on $(RELEASE_BRANCH); skipping" + continue + fi + if ! APPLY_OUTPUT=$(git apply --3way --index "$PREREQUISITE_PATCH" 2>&1); then + printf '%s\n' "$APPLY_OUTPUT" + CONFLICTING_FILES=$(git diff --name-only --diff-filter=U 2>/dev/null || true) + if [ -n "$CONFLICTING_FILES" ]; then + echo "##vso[task.logissue type=error]Prerequisite $PREREQUISITE conflicts with $(RELEASE_BRANCH)" + echo "" + echo "=== Conflicting files ===" + printf '%s\n' "$CONFLICTING_FILES" + else + echo "##vso[task.logissue type=error]Unable to apply prerequisite $PREREQUISITE before conflict detection" + fi + echo "=== Checkout status after prerequisite failure ===" + git status --short + exit 1 + fi + printf '%s\n' "$APPLY_OUTPUT" + echo "Prerequisite $PREREQUISITE applies cleanly" + done + if ! APPLY_OUTPUT=$(git apply --3way --index "$PATCH_PATH" 2>&1); then printf '%s\n' "$APPLY_OUTPUT" CONFLICTING_FILES=$(git diff --name-only --diff-filter=U 2>/dev/null || true) @@ -1027,3 +1352,391 @@ jobs: testResultsFiles: '**/test-reports/TEST-*.xml' failTaskOnFailedTests: false condition: and(succeededOrFailed(), eq(variables.releaseCompatRequired, 'true')) +- job: InternalCompat + displayName: 'SynapseML-Internal Compatibility Check' + # Scoped to PR builds, matching ReleaseBranchCompat above: this validates that a + # proposed change still compiles Internal, which is only actionable while the + # change can still be revised. System.PullRequest.* is empty on non-PR builds, so + # the IsFork check alone would let this 120-minute, secret-consuming job run on + # every master push and nightly build. + # SynapseML-Internal is a private repo. Fork PRs cannot authenticate to it, so skip the job + # there rather than failing every external contribution. + condition: >- + and( + succeeded(), + eq(variables.isPR, true), + ne(variables['System.PullRequest.IsFork'], 'True') + ) + cancelTimeoutInMinutes: 0 + # Advisory, not a merge gate. The Internal repo moves independently of this one, so a red + # here is as often "Internal's own snapshot did not resolve today" as it is a real break -- + # build 230974877 went red with `unresolved dependency: ...-internal_2.12 ... not found` on a + # PR that changes only pipeline.yaml. Without this, InternalCompat rolls up into the required + # `microsoft.SynapseML` check and a transient Internal problem blocks unrelated OSS merges. + # This still surfaces as its own red check run, so the signal is not lost, matching how + # ReleaseBranchCompat above is configured. + continueOnError: true + # Measured on build 230962458: conda env 6.4m + compile 1.5m + package 1.2m + + # Scala tests ~34m + ExcludeAIFunc. 90 min was not enough and the job was + # killed mid-step; 120 leaves headroom now that the live-service suites are gone. + timeoutInMinutes: 120 + dependsOn: BuildAndCacheSbt + pool: + vmImage: $(UBUNTU_VERSION) + # sbt on Java 17 needs java.util.prefs opened reflectively. Setting it here + # rather than on individual sbt commands covers every JVM the job starts, + # including templates/sbt_cache.yml's prewarm, the Internal test loop, and any + # JVM the tests fork -- per-command flags reach none of those. spark4.0's + # pipeline already uses JAVA_TOOL_OPTIONS this way for the same reason. + # + # Set unconditionally rather than only for Spark 4. `java.prefs` has existed + # since Java 9, so on master's Java 11 the option is genuinely applied rather + # than ignored -- it just opens a package nothing on that path reflects into, + # which is harmless. Setting it for every target keeps a branch conditional + # out of this file. + # + # The one place JVM stderr is parsed is the OSS version read below, and its + # grep is anchored to '^\[info\] ', which the "Picked up JAVA_TOOL_OPTIONS" + # line cannot match. + variables: + JAVA_TOOL_OPTIONS: '--add-opens=java.prefs/java.util.prefs=ALL-UNNAMED' + steps: + # Explicit paths: a multi-repo job otherwise nests every repo under a folder + # named after it, which moves the OSS root off $(Build.SourcesDirectory). That + # would leave templates/sbt_cache.yml resolving its cache keys and + # tools/ci/sbt_retry.sh against a directory with no build definition, so this + # job would silently cold-bootstrap sbt instead of sharing the prewarmed cache. + - checkout: self + path: s + - checkout: SynapseML-Internal + path: s-internal + # Required by 'Select matching SynapseML-Internal branch' below: without persisted + # credentials the follow-up fetch cannot authenticate to this private repo. + persistCredentials: true + # The JDK has to match the branch under test, and the branch already declares it: + # templates/java_setup.yml is this repo's per-branch JDK pin, carrying 11 here and + # 17 on spark4.0/spark4.1. Without it this job compiled both trees on the agent's + # default JDK and failed with "Class java.lang.Record not found" -- on the OSS tree + # for spark4.1 (build 231477746) and on the Internal tree for spark4.0 (build + # 231456784), a type that arrived in Java 16. That was a permanent red on those + # branches rather than a real compatibility signal. + # + # Including the template rather than mapping the target branch to a version here + # keeps this line identical on every branch, so the sync never has to reconcile it, + # and leaves exactly one place per branch that decides the JDK. + - template: templates/java_setup.yml + # Internal mirrors this repo's branch names 1:1 (master, spark3.5, spark4.0, spark4.1), + # so the OSS branch under test decides which Internal branch is the correct comparison + # point: master pairs with Internal master (both Spark 3.5), spark4.1 with Internal + # spark4.1. Validating a Spark 4.1 PR against Internal master would compile a Spark 3.5 + # tree against Spark 4.1 artifacts, which cannot resolve -- a permanent red on that + # branch rather than a real signal. + - bash: | + set -euo pipefail + cd $(Agent.BuildDirectory)/s-internal + + # Read the target branch from the environment rather than a $() macro: macros are + # substituted as raw text into the script, and git permits ';' and quotes in ref + # names, so a crafted branch name would otherwise be executable. + OSS_TARGET="${SYSTEM_PULLREQUEST_TARGETBRANCH:-}" + OSS_TARGET="${OSS_TARGET#refs/heads/}" + if [ -z "$OSS_TARGET" ]; then + echo "##vso[task.logissue type=error]System.PullRequest.TargetBranch is empty; this job is PR-only" + exit 1 + fi + + # Identity mapping: OSS and Internal use the same branch names, so a branch added to + # the pr: trigger list is paired correctly without touching this step. + INTERNAL_BRANCH="$OSS_TARGET" + + # Fall back rather than fail: this job is advisory, and Internal may not have cut a + # counterpart branch yet. The warning keeps the mismatch visible, because a silent + # fallback would report a result that actually validated a different Spark line. + if ! git ls-remote --exit-code --heads origin "$INTERNAL_BRANCH" >/dev/null 2>&1; then + echo "##vso[task.logissue type=warning]SynapseML-Internal has no '$INTERNAL_BRANCH' branch; falling back to master. This PR targets OSS '$OSS_TARGET', so treat any failure here as unverified." + INTERNAL_BRANCH=master + fi + + echo "OSS target branch '$OSS_TARGET' -> SynapseML-Internal '$INTERNAL_BRANCH'" + git fetch --no-tags origin "$INTERNAL_BRANCH" + git checkout --force FETCH_HEAD + echo "##vso[task.setvariable variable=INTERNAL_BRANCH]$INTERNAL_BRANCH" + echo "=== SynapseML-Internal now at $INTERNAL_BRANCH ===" + git log --oneline -1 + displayName: 'Select matching SynapseML-Internal branch' + - template: templates/sbt_cache.yml + - task: AzureCLI@2 + displayName: 'Publish OSS to local Maven' + timeoutInMinutes: 20 + inputs: + azureSubscription: 'SynapseML Build' + scriptLocation: inlineScript + scriptType: bash + inlineScript: | + set -e + cd $(Build.SourcesDirectory) + export SBT_OPTS="-Xmx4G -Xss2M -Duser.timezone=GMT" + # The version string is recomputed on every sbt invocation and embeds an HHmm + # timestamp whenever the tree is dirty, so querying the version and publishing in + # two separate invocations can yield two different versions when they straddle a + # minute boundary. Run both in ONE sbt session so the version is computed once. + if ! SBT_OUT="$(sbt -no-colors "core/version" publishM2 2>&1)"; then + echo "$SBT_OUT" + echo "##vso[task.logissue type=error]sbt core/version publishM2 failed" + exit 1 + fi + echo "$SBT_OUT" + # tools/ci/get_sbt_version.sh is not reused here: it runs its own sbt + # invocation, which is precisely the second invocation this step exists to + # avoid. Its parsing and validation are mirrored inline instead. The regex + # is anchored to the start of the line and requires a version-shaped token, + # so a stray "[info] ..." line elsewhere in the output cannot be + # mistaken for the version and silently retarget Internal at an artifact + # that was never published. + OSS_VERSION=$(echo "$SBT_OUT" | sed 's/\x1b\[[0-9;]*m//g' \ + | grep -m1 -E '^\[info\] ([0-9]+\.[0-9]+|HEAD-)' | cut -d' ' -f2) + echo "OSS version: $OSS_VERSION" + # The grep above already constrains the shape, so an empty result is the only + # way this can go wrong: it means no line matched. + if [ -z "$OSS_VERSION" ]; then + echo "##vso[task.logissue type=error]Failed to determine the OSS version from sbt output" + exit 1 + fi + # Every later step interpolates this value into bash, sed and grep. The grep + # above only anchors the start of the token, so constrain the whole thing to + # characters that are inert in all three contexts: that removes any way for a + # crafted version string in build.sbt to smuggle a command substitution or a + # sed delimiter through the Azure Pipelines macro expander. + case "$OSS_VERSION" in + *[!A-Za-z0-9._+-]*) + echo "##vso[task.logissue type=error]OSS version '$OSS_VERSION' contains unexpected characters" + exit 1 + ;; + esac + echo "##vso[task.setvariable variable=OSS_VERSION]$OSS_VERSION" + echo "=== published OSS versions in ~/.m2 ===" + ls "$HOME/.m2/repository/com/microsoft/azure/" 2>/dev/null || true + - bash: | + set -e + cd $(Agent.BuildDirectory)/s-internal + + # $OSS_VERSION is read as a bash environment variable rather than the + # $(OSS_VERSION) macro: macros are pasted in as raw text before bash parses the + # script, so a version string containing $() or backticks would execute. + echo "=== Retargeting Internal to OSS version $OSS_VERSION ===" + [ -n "${OSS_VERSION:-}" ] || { echo "##vso[task.logissue type=error]OSS_VERSION is not set"; exit 1; } + grep -q '^val synapseMLVersion = ' build.sbt || { + echo "##vso[task.logissue type=error]synapseMLVersion not found in Internal build.sbt"; exit 1; } + grep -q '^resolvers ++= Seq(' build.sbt || { + echo "##vso[task.logissue type=error]resolvers block not found in Internal build.sbt"; exit 1; } + + sed -i "s|val synapseMLVersion = \".*\"|val synapseMLVersion = \"${OSS_VERSION}\"|" build.sbt + sed -i '/^resolvers ++= Seq(/a\ Resolver.mavenLocal,' build.sbt + + # A silent no-op here would compile Internal against the *published* OSS build and + # report a false green, so verify both edits actually landed. Use fixed-string + # matching: the version contains dots, which as a regex would match any character + # and could accept a line the sed above never wrote. + grep -qF "val synapseMLVersion = \"${OSS_VERSION}\"" build.sbt || { + echo "##vso[task.logissue type=error]Failed to retarget synapseMLVersion"; exit 1; } + grep -qF 'Resolver.mavenLocal,' build.sbt || { + echo "##vso[task.logissue type=error]Failed to add Resolver.mavenLocal"; exit 1; } + + echo "=== Modified build.sbt ===" + grep -n 'synapseMLVersion' build.sbt + grep -n -A4 'resolvers ++=' build.sbt + displayName: 'Retarget Internal to this build' + - task: AzureCLI@2 + displayName: 'Compile Internal against OSS' + timeoutInMinutes: 15 + inputs: + azureSubscription: 'SynapseML Build' + scriptLocation: inlineScript + scriptType: bash + inlineScript: | + set -e + cd $(Agent.BuildDirectory)/s-internal + export SBT_OPTS="-Xmx4G -Xss2M -Duser.timezone=GMT" + echo "Compiling SynapseML-Internal against OSS $OSS_VERSION..." + sbt compile Test/compile + - bash: | + echo "##vso[task.prependpath]$CONDA/bin" + displayName: 'Add conda to PATH' + - bash: | + sudo chown -R $(whoami):$(id -ng) $(CONDA_CACHE_DIR) + displayName: 'Fix conda directory permissions' + - task: PipAuthenticate@1 + displayName: 'Private Conda Feed Authentication' + inputs: + artifactFeeds: 'A365/Synapse-Conda' + - bash: | + conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main + conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r + displayName: 'Accept Anaconda TOS' + - bash: | + echo "=== Disk space BEFORE cleanup ===" + df -h / | grep -E 'Filesystem|/$' + echo "Removing unused pre-installed SDKs and runtimes..." + sudo rm -rf /usr/local/lib/android || true + sudo rm -rf /usr/share/dotnet || true + sudo rm -rf /opt/ghc || true + sudo rm -rf /usr/local/share/boost || true + docker system prune -af --volumes 2>/dev/null || true + echo "=== Disk space AFTER cleanup ===" + df -h / | grep -E 'Filesystem|/$' + displayName: 'Free disk space (remove unused SDKs)' + - bash: | + set -e + conda env create --yes -f $(Agent.BuildDirectory)/s-internal/environment.yaml -v || \ + conda env create --yes -f $(Agent.BuildDirectory)/s-internal/environment.yaml -v + conda clean --all -y + pip cache purge + displayName: 'Create Internal conda env' + - task: AzureKeyVault@2 + displayName: 'Fetch AI service secrets' + retryCountOnTaskFailure: 3 + inputs: + azureSubscription: 'SynapseML Build' + keyVaultName: mmlspark-keys + - template: templates/fabric_kv.yml + - task: AzureCLI@2 + displayName: 'Run Internal Scala tests' + timeoutInMinutes: 60 + inputs: + azureSubscription: 'SynapseML Build' + scriptLocation: inlineScript + scriptType: bash + inlineScript: | + set -e + cd $(Agent.BuildDirectory)/s-internal + eval "$(conda shell.bash hook)" + conda activate synapseml-internal + export CREATE_SEMPY_WRITER=false + export SBT_OPTS="-Xmx4G -Xss2M -Duser.timezone=GMT" + echo "Running Internal tests against OSS $OSS_VERSION..." + # Only the reproducible packages run here. They exercise Internal code against + # the OSS API surface, so a failure is most often a genuine break. They are not + # hermetic -- spark.aifunc calls live AI services and takes ~23 min -- which is + # part of why this job is advisory rather than a merge gate. + # + # powerbi and nbtest are deliberately NOT run: they drive live services + # (PowerBI XMLA needs a populated workspace; nbtest submits Spark Job + # Definitions to live Fabric capacity) and fail here for environmental + # reasons regardless of the OSS diff. Running them anyway cost ~4 min and + # produced only ignorable warnings. That live E2E coverage belongs in + # SynapseML-Internal's own nightly pipeline, which has the workspaces, + # sample datasets and capacity provisioned. + FAILURES=0 + for pkg in spark.aifunc ebm predict; do + echo "=== Testing $pkg ===" + if ! sbt "testOnly com.microsoft.azure.synapse.ml.$pkg.**"; then + echo "##vso[task.logissue type=error]$pkg tests failed - OSS change breaks Internal" + FAILURES=$((FAILURES + 1)) + fi + done + if [ $FAILURES -gt 0 ]; then + echo "##vso[task.logissue type=error]$FAILURES test package(s) failed" + exit 1 + fi + + # sbt `testOnly` exits 0 when its filter matches nothing, so an Internal package + # rename would turn this job into a permanently green no-op. The version retarget + # above is guarded against exactly this class of false green; the tests were not. + # Count what actually ran instead of trusting the exit status. grep -o emits one + # match per line, so several elements sharing a line all count. + TEST_COUNT=$(find . -path '*/test-reports/TEST-*.xml' -exec grep -ho 'tests="[0-9]*"' {} + \ + | tr -dc '0-9\n' | awk '{n += $1} END {print n + 0}') + echo "Scala tests executed: $TEST_COUNT" + if [ "${TEST_COUNT:-0}" -lt 150 ]; then + echo "##vso[task.logissue type=error]Only $TEST_COUNT Scala tests ran (expected ~211); the testOnly filter matched little or nothing, so this result is not a real signal" + exit 1 + fi + env: + INTEGRATION_ENV: $(sempy-integration-region) + INTEGRATION_ACCOUNT: $(sempy-integration-account) + INTEGRATION_CERTIFICATE: $(sempy-integration-certificate) + INTEGRATION_WORKSPACE_PREFIX: $(sempy-integration-workspace-prefix) + - task: AzureCLI@2 + displayName: 'Package Internal Python against OSS' + timeoutInMinutes: 15 + inputs: + azureSubscription: 'SynapseML Build' + scriptLocation: inlineScript + scriptType: bash + inlineScript: | + set -e + cd $(Agent.BuildDirectory)/s-internal + eval "$(conda shell.bash hook)" + conda activate synapseml-internal + export CREATE_SEMPY_WRITER=false + export SBT_OPTS="-Xmx4G -Xss2M -Duser.timezone=GMT" + echo "Packaging Internal Python against OSS $OSS_VERSION..." + # packagePython and publishM2 MUST share one sbt session. Internal's version + # embeds an HHmm timestamp, so separate invocations can differ by a minute: + # packagePython bakes the jar coordinate into the generated Python package, + # publishM2 then publishes a *different* version, and the Spark launch inside + # make_mlflow_models.py fails to resolve it - surfacing only as an opaque + # JAVA_GATEWAY_EXITED. Observed: packaged 1022, published 1023. + sbt packagePython publishM2 + echo "=== published Internal versions in ~/.m2 ===" + ls -d "$HOME"/.m2/repository/com/microsoft/azure/*-internal_2.12/*/ 2>/dev/null || true + condition: succeededOrFailed() + # NOTE: 'Run Internal Python tests (AIFunc)' was intentionally removed. All four + # AIFunc suites call live Azure OpenAI endpoints, so they measure service and + # quota availability rather than OSS/Internal compatibility. They took ~23 min + # of the job's budget and were the main reason it hit its timeout before the + # gating ExcludeAIFunc step could finish. They remain covered by + # SynapseML-Internal's own pipeline. + - task: AzureCLI@2 + displayName: 'Run Internal Python tests (ExcludeAIFunc)' + timeoutInMinutes: 30 + inputs: + azureSubscription: 'SynapseML Build' + scriptLocation: inlineScript + scriptType: bash + inlineScript: | + set -e + cd $(Agent.BuildDirectory)/s-internal + eval "$(conda shell.bash hook)" + conda activate synapseml-internal + export CREATE_SEMPY_WRITER=false + export SBT_OPTS="-Xmx4G -Xss2M -Duser.timezone=GMT" + # The fixture script builds several models in-process and then launches a + # JVM for a Spark session. That launch failed once with JAVA_GATEWAY_EXITED, + # which pyspark reports without surfacing any JVM stderr, so record the + # agent state first to make a repeat diagnosable rather than a mystery. + echo "=== agent state before Spark JVM launch ===" + free -m || true + df -h /home/vsts /tmp || true + echo "JAVA_HOME=${JAVA_HOME:-}" + java -version 2>&1 || true + echo "residual JVMs: $(pgrep -c java || echo 0)" + echo "===========================================" + echo "Creating MLflow model fixtures..." + # Launching the gateway is the one step here that is known to fail + # intermittently, so give it a second attempt before failing the job. + if ! python ./src/test/python/make_mlflow_models.py; then + echo "##vso[task.logissue type=warning]MLflow fixture generation failed; retrying once" + free -m || true + sleep 30 + python ./src/test/python/make_mlflow_models.py + fi + echo "Running ExcludeAIFunc Python tests against OSS $OSS_VERSION..." + sbt testPythonExcludeAIFunc + condition: succeededOrFailed() + - task: PublishTestResults@2 + displayName: 'Publish Internal Scala Test Results' + inputs: + testResultsFiles: '**/test-reports/TEST-*.xml' + # Without this the task searches the OSS checkout, which has no Internal + # results, and silently publishes nothing. + searchFolder: '$(Agent.BuildDirectory)/s-internal' + failTaskOnFailedTests: false + condition: succeededOrFailed() + - task: PublishTestResults@2 + displayName: 'Publish Internal Python Test Results' + inputs: + testResultsFiles: '**/python-test-*.xml' + searchFolder: '$(Agent.BuildDirectory)/s-internal' + failTaskOnFailedTests: false + condition: succeededOrFailed() diff --git a/project/OnnxRuntimeDependency.scala b/project/OnnxRuntimeDependency.scala new file mode 100644 index 00000000000..ccfb2f02602 --- /dev/null +++ b/project/OnnxRuntimeDependency.scala @@ -0,0 +1,49 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +import sbt._ +import sbt.Keys._ + +/** + * Rewrites the known ONNX Runtime artifacts (`onnxruntime`, the default CPU-only artifact, and + * `onnxruntime_gpu`, the opt-in CUDA artifact) to a single shared version, as the final deep-learning + * project setting. Keeping the version separate lets the same change replay across Spark branches with + * different adjacent protobuf coordinates, and covers a branch that still declares `onnxruntime_gpu` on + * its own test classpath. + * + * Matching is restricted to `ManagedArtifactIds` (not just the `com.microsoft.onnxruntime` organization): + * a plain org-wide match would silently coerce the version of any future, unrelated artifact published + * under the same organization (for example one with its own independent release cadence) without anyone + * noticing. Any dependency declaration this setting actually changes -- or any unmanaged artifact under + * this organization it deliberately leaves untouched -- is logged so the override is never silent. + * + * GH2417: 1.8.1 fails to load its natives for local Spark 3.5 inference, and the GPU-only artifact never + * ships macOS natives at all (CUDA has no macOS support), so upgrading `onnxruntime_gpu` alone cannot fix + * macOS. 1.17.3 is confirmed to fix local CPU inference and additionally publishes a CPU-only `onnxruntime` + * artifact with macOS x64/aarch64 natives, so it is used as the default cross-platform dependency in + * build.sbt. See docs/Explore Algorithms/Deep Learning/ONNX.md for the CUDA opt-in instructions. + */ +object OnnxRuntimeDependency { + val Version = "1.17.3" + private val Organization = "com.microsoft.onnxruntime" + private val ManagedArtifactIds: Set[String] = Set("onnxruntime", "onnxruntime_gpu") + + val settings: Seq[Setting[_]] = Seq( + libraryDependencies ~= { + _.map { + case dependency if dependency.organization == Organization && ManagedArtifactIds(dependency.name) => + if (dependency.revision != Version) { + println(s"[info] OnnxRuntimeDependency: overriding $Organization:${dependency.name} " + + s"${dependency.revision} -> $Version (see project/OnnxRuntimeDependency.scala).") + } + dependency.withRevision(Version) + case dependency if dependency.organization == Organization => + println(s"[warn] OnnxRuntimeDependency: leaving unmanaged $Organization:${dependency.name}:" + + s"${dependency.revision} at its declared version -- add it to ManagedArtifactIds in " + + s"project/OnnxRuntimeDependency.scala if it should track the shared $Version instead.") + dependency + case dependency => dependency + } + } + ) +} diff --git a/sonatype.sbt b/sonatype.sbt index d49d82c6dbd..96d2d809cbe 100644 --- a/sonatype.sbt +++ b/sonatype.sbt @@ -10,16 +10,14 @@ ThisBuild / scmInfo := Some( ) ) ThisBuild / developers := List( - Developer("mhamilton723", "Mark Hamilton", - "synapseml-support@microsoft.com", url("https://github.com/mhamilton723")), - Developer("imatiach-msft", "Ilya Matiach", - "synapseml-support@microsoft.com", url("https://github.com/imatiach-msft")), - Developer("drdarshan", "Sudarshan Raghunathan", - "synapseml-support@microsoft.com", url("https://github.com/drdarshan")), - Developer("svotaw", "Scott Votaw", - "synapseml-support@microsoft.com", url("https://github.com/svotaw")), + Developer("ranadeepsingh", "Rana Singh", + "synapseml-support@microsoft.com", url("https://github.com/ranadeepsingh")), Developer("BrendanWalsh", "Brendan Walsh", "synapseml-support@microsoft.com", url("https://github.com/BrendanWalsh")), + Developer("svotaw", "Scott Votaw", + "synapseml-support@microsoft.com", url("https://github.com/svotaw")), + Developer("eisber", "Markus Cozowicz", + "synapseml-support@microsoft.com", url("https://github.com/eisber")), Developer("JessicaXYWang", "Jessica Wang", "synapseml-support@microsoft.com", url("https://github.com/JessicaXYWang")) ) diff --git a/src/test/scala/com/microsoft/azure/synapse/ml/core/test/fuzzing/FuzzingTest.scala b/src/test/scala/com/microsoft/azure/synapse/ml/core/test/fuzzing/FuzzingTest.scala index e6ec91917e7..57bc9cdf529 100644 --- a/src/test/scala/com/microsoft/azure/synapse/ml/core/test/fuzzing/FuzzingTest.scala +++ b/src/test/scala/com/microsoft/azure/synapse/ml/core/test/fuzzing/FuzzingTest.scala @@ -12,7 +12,7 @@ import org.apache.spark.ml._ import org.apache.spark.ml.param._ import org.apache.spark.ml.util.{MLReadable, MLWritable} -import java.lang.reflect.ParameterizedType +import java.lang.reflect.{InvocationTargetException, Modifier, ParameterizedType} import scala.language.existentials /** Tests to validate fuzzing of modules. */ @@ -417,7 +417,19 @@ class FuzzingTest extends TestBase { private lazy val readers: List[MLReadable[_]] = JarLoadingUtils.instantiateObjects[MLReadable[_]]() - private lazy val pipelineStages: List[PipelineStage] = JarLoadingUtils.instantiateServices[PipelineStage]() + private lazy val pipelineStages: List[PipelineStage] = { + JarLoadingUtils.AllClasses + .filter(classOf[PipelineStage].isAssignableFrom(_)) + .filter(clazz => !Modifier.isAbstract(clazz.getModifiers)) + .filterNot(clazz => clazz.getName.contains("$") || clazz.getSimpleName.startsWith("Testable")) + .map { clazz => + try { + clazz.getConstructor().newInstance().asInstanceOf[PipelineStage] + } catch { + case e: InvocationTargetException => throw e.getCause + } + } + } private lazy val experimentFuzzers: List[ExperimentFuzzing[_ <: PipelineStage]] = JarLoadingUtils.instantiateServices[ExperimentFuzzing[_ <: PipelineStage]]() diff --git a/templates/codecov.yml b/templates/codecov.yml index 13d921d1191..9a7e3db76d9 100644 --- a/templates/codecov.yml +++ b/templates/codecov.yml @@ -21,3 +21,4 @@ steps: retryCountOnTaskFailure: 1 displayName: Upload Coverage Report To Codecov.io condition: succeededOrFailed() + continueOnError: true diff --git a/templates/publish_coverage_ado.yml b/templates/publish_coverage_ado.yml new file mode 100644 index 00000000000..9f1eee6e850 --- /dev/null +++ b/templates/publish_coverage_ado.yml @@ -0,0 +1,22 @@ +parameters: + # WebsiteSamplesTests drives Spark out-of-process, so scoverage never records + # measurement data there and having nothing to publish is the expected state. + # Every other job that includes this template does produce a report. + - name: failIfCoverageEmpty + type: boolean + default: true + +steps: + - task: PublishCodeCoverageResults@2 + displayName: 'Publish Code Coverage to Azure DevOps' + inputs: + # Cobertura XML, which Azure DevOps understands. + # sbt-scoverage writes it to target/scala-/coverage-report/ + # (scoverage-report/ only ever holds scoverage.xml and the HTML), so glob + # coverage-report explicitly. The glob is deliberately version-agnostic so + # this template is correct on every branch. + summaryFileLocation: '**/coverage-report/cobertura.xml' + pathToSources: '$(Build.SourcesDirectory)' + # Fail loudly if the glob stops matching, rather than silently publishing nothing. + failIfCoverageEmpty: ${{ parameters.failIfCoverageEmpty }} + condition: and(succeededOrFailed(), eq(variables.runCoverage, true)) diff --git a/tools/ci/tests/test_pipeline_yaml.py b/tools/ci/tests/test_pipeline_yaml.py index b21e9125fe5..a918b5d29a8 100644 --- a/tools/ci/tests/test_pipeline_yaml.py +++ b/tools/ci/tests/test_pipeline_yaml.py @@ -6,13 +6,17 @@ shared helper. Run with: ``python -m pytest tools/ci/tests/test_pipeline_yaml.py``. """ +import os +import re +import shutil +import subprocess +import uuid from pathlib import Path +import pytest import yaml REPO_ROOT = Path(__file__).resolve().parents[3] -BUILD_SBT = REPO_ROOT / "build.sbt" -PLUGINS_SBT = REPO_ROOT / "project" / "plugins.sbt" PIPELINE = REPO_ROOT / "pipeline.yaml" SBT_CACHE_TPL = REPO_ROOT / "templates" / "sbt_cache.yml" SBT_RETRY = REPO_ROOT / "tools" / "ci" / "sbt_retry.sh" @@ -21,6 +25,12 @@ DATABRICKS_STEPS_TPL = REPO_ROOT / "templates" / "databricks_e2e_steps.yml" CLEAN_ACR_PIPELINE = REPO_ROOT / ".pipelines" / "clean-acr.yml" PR_VALIDATION = REPO_ROOT / ".github" / "workflows" / "pr-validation.yml" +BUILD_SBT = REPO_ROOT / "build.sbt" +PLUGINS_SBT = REPO_ROOT / "project" / "plugins.sbt" +RELEASE_COMPAT_PREREQUISITES = ( + REPO_ROOT / ".pipelines" / "release-compat-prerequisites.txt" +) +ASCII_WHITESPACE = " \t\r\n\v\f" def _pipeline_text(): @@ -38,6 +48,67 @@ def _jobs(node): yield from _jobs(value) +def _release_compat_script(): + data = yaml.safe_load(_pipeline_text()) + jobs = {j.get("job"): j for j in _jobs(data["jobs"])} + steps = jobs["ReleaseBranchCompat"]["steps"] + return next( + step["bash"] + for step in steps + if isinstance(step, dict) + and step.get("displayName") == "Apply PR changes onto $(RELEASE_BRANCH)" + ) + + +def _git(repo, *args): + result = subprocess.run( + ["git", "-C", str(repo), *args], + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 0, ( + f"git {' '.join(args)} failed\nstdout:\n{result.stdout}\nstderr:\n" + f"{result.stderr}" + ) + return result + + +def _init_release_compat_scratch_repo(repo): + subprocess.run( + ["git", "init", "--initial-branch=master", str(repo)], + check=True, + capture_output=True, + text=True, + ) + for key, value in ( + ("user.name", "Release Compat Test"), + ("user.email", "release-compat@example.test"), + ("core.autocrlf", "false"), + ("core.hooksPath", ".git/hooks-disabled"), + ("commit.gpgsign", "false"), + ("merge.autostash", "false"), + ("rebase.autostash", "false"), + ): + _git(repo, "config", key, value) + + +def _assert_git_clean(repo, context): + status = _git(repo, "status", "--short").stdout.strip() + assert not status, f"{context} left scratch repo dirty:\n{status}" + + +def _is_normalized_prerequisite_path(path): + return ( + bool(path) + and path == path.strip(ASCII_WHITESPACE) + and not any(character in path for character in "\t\r\n") + and not path.startswith("/") + and not path.endswith("/") + and all(part not in {"", ".", ".."} for part in path.split("/")) + ) + + def test_pipeline_and_templates_parse(): assert yaml.safe_load(PIPELINE.read_text()) is not None assert yaml.safe_load(CLEAN_ACR_PIPELINE.read_text()) is not None @@ -271,9 +342,17 @@ def test_spark41_isolation_forest_excludes_spark_runtime_dependencies(): assert exclusion in dependency +def _pr_validation_branches(): + """Parse the pull_request branch filter. YAML 1.1 folds the bare key ``on`` + to boolean True, so accept either spelling.""" + data = yaml.safe_load(PR_VALIDATION.read_text()) + triggers = data.get("on", data.get(True)) + return triggers["pull_request"]["branches"] + + def test_github_pr_validation_supports_spark41(): workflow = PR_VALIDATION.read_text() - assert 'branches: [ "master", "spark4.1" ]' in workflow + assert "spark4.1" in _pr_validation_branches() assert 'python-version: "3.13"' in workflow assert "Set up JDK 17" in workflow assert "java-version: 17" in workflow @@ -338,12 +417,83 @@ def test_release_compat_accepts_github_target_and_uses_one_sbt_process(): assert "Skipping deletion already absent on $(RELEASE_BRANCH)" in rebase_script assert "[ ${#REPLAY_PATHS[@]} -eq 0 ]" in rebase_script assert '"${REPLAY_PATHS[@]}" > "$PATCH_PATH"' in rebase_script + assert 'PREREQUISITES_CONFIG=".pipelines/release-compat-prerequisites.txt"' in ( + rebase_script + ) + assert 'git show "$PR_MERGE_HEAD:$PREREQUISITES_CONFIG"' in rebase_script + assert '[[ ! "$PREREQUISITE" =~ ^[0-9a-fA-F]{40}$ ]]' in rebase_script + assert "LINE_WITHOUT_CR=\"${RAW_LINE%$'\\r'}\"" in rebase_script + assert r"""[[ "$SCOPED_LINE" == *$'\t'* ]]""" in rebase_script + assert r"""[[ "$RAW_LINE" == *$'\t'* ]]""" not in rebase_script + assert "PREREQUISITE=\"${SCOPED_LINE%%$'\\t'*}\"" in rebase_script + assert "PREREQUISITE_SCOPE=\"${SCOPED_LINE#*$'\\t'}\"" in rebase_script + assert "CONFIGURED_PREREQUISITE_SCOPES=()" in rebase_script + assert ( + 'git merge-base --is-ancestor "$PREREQUISITE" "$TARGET_HEAD"' in rebase_script + ) + assert 'case "/$path/" in' in rebase_script + assert ( + "Scoped prerequisite path must not have leading or trailing whitespace: $path" + in rebase_script + ) + assert "Scoped prerequisite path must be normalized: $path" in rebase_script + assert 'for REPLAY_PATH in "${REPLAY_PATHS[@]}"' in rebase_script + assert '[ "$path" = "$REPLAY_PATH" ]' in rebase_script + assert 'PREREQUISITE_PATHS+=("$path")' in rebase_script + assert "has no paths in this PR replay; skipping" in rebase_script + assert rebase_script.index('case "/$path/" in') < rebase_script.index( + 'for REPLAY_PATH in "${REPLAY_PATHS[@]}"' + ) + assert 'git diff --quiet "$PREREQUISITE_PARENT" "$PREREQUISITE" --' in rebase_script + assert 'git cat-file -e "$PREREQUISITE_PARENT:$path"' in rebase_script + assert 'git rev-parse "$PREREQUISITE:$path"' in rebase_script + assert 'BASELINE_COMMIT="$TARGET_HEAD"' in rebase_script + assert 'git rev-parse "$BASELINE_COMMIT:$path"' in rebase_script + assert '[ "$PREREQUISITE_BLOB" != "$BASELINE_BLOB" ]' in rebase_script + assert '":(literal)$path"' in rebase_script + assert '"${PREREQUISITE_PATHSPECS[@]}"' in rebase_script + assert "eval " not in rebase_script + assert 'git rev-parse "$PREREQUISITE^1"' in rebase_script + guarded_dependent_parent = ( + 'if ! LATER_PARENT=$(git rev-parse "$LATER_PREREQUISITE^1" ' + "2>/dev/null); then" + ) + assert rebase_script.count(guarded_dependent_parent) == 2 + assert ( + rebase_script.count( + "Dependent release compatibility prerequisite " + "$LATER_PREREQUISITE has no first parent" + ) + == 2 + ) + assert ( + 'git diff --name-only -z "$PREREQUISITE_PARENT" "$PREREQUISITE"' + in rebase_script + ) + release_exclusions = ( + ".github/*|.pipelines/*|docs/*|templates/*|tools/acr/*|tools/ci/*|" + "tools/docker/*|tools/helm/*|website/*" + ) + assert rebase_script.count(release_exclusions) == 2 + assert ( + 'git diff --binary --full-index "$PREREQUISITE_PARENT" "$PREREQUISITE"' + in rebase_script + ) assert "git checkout --detach $RELEASE_TIP" in rebase_script + assert "git reset --hard $RELEASE_TIP" in rebase_script + assert 'git apply --reverse --check --index "$PREREQUISITE_PATCH"' in rebase_script + assert 'git apply --3way --index "$PREREQUISITE_PATCH"' in rebase_script assert 'git apply --3way --index "$PATCH_PATH"' in rebase_script + assert rebase_script.index( + 'git show "$PR_MERGE_HEAD:$PREREQUISITES_CONFIG"' + ) < rebase_script.index("git checkout --detach $RELEASE_TIP") + assert rebase_script.index( + 'git apply --3way --index "$PREREQUISITE_PATCH"' + ) < rebase_script.index('git apply --3way --index "$PATCH_PATH"') assert "git rebase" not in rebase_script assert "CONFLICTING_FILES=$(git diff --name-only --diff-filter=U" in rebase_script assert "before conflict detection" in rebase_script - assert rebase_script.count("printf '%s\\n' \"$APPLY_OUTPUT\"") == 2 + assert rebase_script.count("printf '%s\\n' \"$APPLY_OUTPUT\"") == 4 cache_step = next( step @@ -386,6 +536,677 @@ def test_release_compat_accepts_github_target_and_uses_one_sbt_process(): assert "releaseCompatRequired" in result_steps[0]["condition"] +def test_release_compat_prerequisites_have_valid_format(): + lines = [ + line + for line in RELEASE_COMPAT_PREREQUISITES.read_text().splitlines() + if line.strip() and not line.lstrip().startswith("#") + ] + # The list is meant to drain to empty once every validated release branch carries the + # backports, and the replay script already treats an absent or empty list as a no-op. + entries = [line.split("\t") for line in lines] + assert all(re.fullmatch(r"[0-9a-fA-F]{40}", fields[0]) for fields in entries) + assert all(all(fields[1:]) for fields in entries) + for _, *paths in entries: + assert all(_is_normalized_prerequisite_path(path) for path in paths) + shas = [fields[0] for fields in entries] + assert len(shas) == len(set(shas)), "prerequisite commits must be unique" + + +@pytest.mark.parametrize( + "path", + [ + "src/file.txt", + "src/file with spaces.txt", + "src/with repeated internal spaces.txt", + ".pipelines/release-compat-prerequisites.txt", + "src/.hidden/file", + "src/.../file", + ], +) +def test_release_compat_prerequisite_path_normalization_accepts_valid_paths(path): + assert _is_normalized_prerequisite_path(path) + + +@pytest.mark.parametrize( + "path", + [ + "", + ".", + "./src/file", + "/src/file", + "src/../file", + "src/..", + "src/./file", + "src/.", + "src//file", + "src/file/", + " src/file", + "src/file ", + "\tsrc/file", + "src/file\t", + "\rsrc/file", + "src/file\r", + "\nsrc/file", + "src/file\n", + ], +) +def test_release_compat_prerequisite_path_normalization_rejects_invalid_paths(path): + assert not _is_normalized_prerequisite_path(path) + + +@pytest.mark.skipif(os.name != "posix", reason="release replay script requires Bash") +def test_release_compat_replays_prerequisite_before_pr_patch(): + scratch_root = REPO_ROOT / "target" / f"release-compat-replay-{uuid.uuid4().hex}" + repo = scratch_root / "repo" + origin = scratch_root / "origin.git" + agent_temp = scratch_root / "agent" + + try: + repo.mkdir(parents=True) + agent_temp.mkdir() + _init_release_compat_scratch_repo(repo) + + source_file = repo / "src" / "value.txt" + source_file.parent.mkdir() + source_file.write_text("base\n") + _git(repo, "add", ".") + _git(repo, "commit", "-m", "base") + base = _git(repo, "rev-parse", "HEAD").stdout.strip() + _git(repo, "branch", "release", base) + + source_file.write_text("aad\n") + _git(repo, "commit", "-am", "prerequisite") + prerequisite = _git(repo, "rev-parse", "HEAD").stdout.strip() + + _git(repo, "checkout", "-b", "source") + prerequisite_config = repo / ".pipelines" / "release-compat-prerequisites.txt" + prerequisite_config.parent.mkdir() + prerequisite_config.write_text(f"# prerequisite\n{prerequisite}\n") + source_file.write_text("aad\nsearch\n") + _git(repo, "add", ".") + _git(repo, "commit", "-m", "feature") + + _git(repo, "checkout", "master") + _assert_git_clean(repo, "checkout before synthetic PR merge") + _git(repo, "merge", "--no-ff", "source", "-m", "merge feature") + + subprocess.run( + ["git", "init", "--bare", str(origin)], + check=True, + capture_output=True, + text=True, + ) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "push", "origin", "master", "source", "release") + + script = _release_compat_script() + script = script.replace("$(Agent.TempDirectory)", str(agent_temp)) + script = script.replace("$(RELEASE_BRANCH)", "release") + result = subprocess.run( + ["bash", "-c", script], + cwd=repo, + check=False, + capture_output=True, + text=True, + ) + + assert ( + result.returncode == 0 + ), f"release replay failed\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}" + assert source_file.read_text() == "aad\nsearch\n" + assert _git(repo, "diff", "--cached", "--name-only").stdout.splitlines() == [ + "src/value.txt" + ] + assert f"Prerequisite {prerequisite} applies cleanly" in result.stdout + assert "PR changes apply cleanly onto release" in result.stdout + finally: + shutil.rmtree(scratch_root, ignore_errors=True) + + +def _run_release_compat_with_scoped_config(scope): + scratch_root = ( + REPO_ROOT / "target" / f"release-compat-invalid-path-{uuid.uuid4().hex}" + ) + repo = scratch_root / "repo" + origin = scratch_root / "origin.git" + agent_temp = scratch_root / "agent" + + try: + repo.mkdir(parents=True) + agent_temp.mkdir() + _init_release_compat_scratch_repo(repo) + + pr_file = repo / "src" / "pr.txt" + pr_file.parent.mkdir() + pr_file.write_text("release base\n") + _git(repo, "add", ".") + _git(repo, "commit", "-m", "base") + base = _git(repo, "rev-parse", "HEAD").stdout.strip() + _git(repo, "branch", "release", base) + + _git(repo, "commit", "--allow-empty", "-m", "prerequisite marker") + prerequisite = _git(repo, "rev-parse", "HEAD").stdout.strip() + + _git(repo, "checkout", "-b", "source") + prerequisite_config = repo / ".pipelines" / "release-compat-prerequisites.txt" + prerequisite_config.parent.mkdir() + prerequisite_config.write_bytes(f"{prerequisite}\t{scope}".encode()) + pr_file.write_text("pull request change\n") + _git(repo, "add", ".") + _git(repo, "commit", "-m", "feature with invalid prerequisite path") + + target = _git(repo, "rev-parse", "master").stdout.strip() + source = _git(repo, "rev-parse", "source").stdout.strip() + source_tree = _git(repo, "rev-parse", f"{source}^{{tree}}").stdout.strip() + merge_commit = _git( + repo, + "commit-tree", + source_tree, + "-p", + target, + "-p", + source, + "-m", + "merge feature", + ).stdout.strip() + _git(repo, "checkout", "--detach", merge_commit) + _assert_git_clean(repo, "synthetic PR merge checkout") + + subprocess.run( + ["git", "init", "--bare", str(origin)], + check=True, + capture_output=True, + text=True, + ) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "push", "origin", "master", "source", "release") + + script = _release_compat_script() + script = script.replace("$(Agent.TempDirectory)", str(agent_temp)) + script = script.replace("$(RELEASE_BRANCH)", "release") + result = subprocess.run( + ["bash", "-c", script], + cwd=repo, + check=False, + capture_output=True, + text=True, + ) + + return result + finally: + shutil.rmtree(scratch_root, ignore_errors=True) + + +@pytest.mark.skipif(os.name != "posix", reason="release replay script requires Bash") +@pytest.mark.parametrize( + "invalid_path", + [ + "src/./scoped.txt", + "src//scoped.txt", + "src/scoped.txt/", + "src/scoped.txt/.", + ], + ids=["dot-segment", "repeated-slash", "trailing-slash", "terminal-dot"], +) +def test_release_compat_rejects_non_normalized_scoped_paths(invalid_path): + result = _run_release_compat_with_scoped_config(f"{invalid_path}\n") + + assert result.returncode != 0 + assert ( + f"Scoped prerequisite path must be normalized: {invalid_path}" in result.stdout + ) + + +@pytest.mark.skipif(os.name != "posix", reason="release replay script requires Bash") +@pytest.mark.parametrize( + ("scope", "expected_error"), + [ + ( + " src/scoped.txt\n", + "Scoped prerequisite path must not have leading or trailing whitespace", + ), + ( + "src/scoped.txt \n", + "Scoped prerequisite path must not have leading or trailing whitespace", + ), + ( + "src/scoped.txt \r\n", + "Scoped prerequisite path must not have leading or trailing whitespace", + ), + ( + "src/scoped.txt \tsrc/other.txt\r\n", + "Scoped prerequisite path must not have leading or trailing whitespace", + ), + ( + "src/scoped.txt\t src/other.txt\r\n", + "Scoped prerequisite path must not have leading or trailing whitespace", + ), + ("\tsrc/scoped.txt\r\n", "contains an empty path"), + ("src/scoped.txt\t\r\n", "contains an empty path"), + ], + ids=[ + "leading-space", + "trailing-space", + "trailing-space-crlf", + "first-path-trailing-space", + "second-path-leading-space", + "leading-tab", + "trailing-tab-crlf", + ], +) +def test_release_compat_rejects_scoped_path_field_whitespace(scope, expected_error): + result = _run_release_compat_with_scoped_config(scope) + + assert result.returncode != 0 + assert expected_error in result.stdout + + +@pytest.mark.skipif(os.name != "posix", reason="release replay script requires Bash") +@pytest.mark.parametrize( + ("config_prefix", "scoped_path", "config_suffix"), + [ + ("", "src/scoped.txt", "\n"), + ("", "src/scoped.txt", "\r\n"), + (" ", "src/scoped.txt", "\r\n"), + ("", "src/scoped file.txt", "\n"), + ], + ids=["normalized", "crlf", "leading-sha-whitespace", "internal-space"], +) +def test_release_compat_replays_scoped_missing_file_prerequisite_full_overlap( + config_prefix, scoped_path, config_suffix +): + scratch_root = REPO_ROOT / "target" / f"release-compat-scoped-{uuid.uuid4().hex}" + repo = scratch_root / "repo" + origin = scratch_root / "origin.git" + agent_temp = scratch_root / "agent" + + try: + repo.mkdir(parents=True) + agent_temp.mkdir() + _init_release_compat_scratch_repo(repo) + + base_file = repo / "base.txt" + base_file.write_text("release base\n") + _git(repo, "add", ".") + _git(repo, "commit", "-m", "base") + base = _git(repo, "rev-parse", "HEAD").stdout.strip() + _git(repo, "branch", "release", base) + + scoped_file = repo / scoped_path + unrelated_file = repo / "src" / "unrelated.txt" + scoped_file.parent.mkdir() + scoped_file.write_text("target baseline\n") + unrelated_file.write_text("must not be replayed\n") + _git(repo, "add", ".") + _git(repo, "commit", "-m", "prerequisite adds target files") + prerequisite = _git(repo, "rev-parse", "HEAD").stdout.strip() + + _git(repo, "checkout", "-b", "source") + prerequisite_config = repo / ".pipelines" / "release-compat-prerequisites.txt" + prerequisite_config.parent.mkdir() + prerequisite_config.write_bytes( + (f"{config_prefix}{prerequisite}\t{scoped_path}{config_suffix}").encode() + ) + scoped_file.write_text("pull request change\n") + _git(repo, "add", ".") + _git(repo, "commit", "-m", "feature modifies scoped file") + + _git(repo, "checkout", "master") + _assert_git_clean(repo, "checkout before synthetic PR merge") + _git(repo, "merge", "--no-ff", "source", "-m", "merge feature") + + subprocess.run( + ["git", "init", "--bare", str(origin)], + check=True, + capture_output=True, + text=True, + ) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "push", "origin", "master", "source", "release") + + script = _release_compat_script() + script = script.replace("$(Agent.TempDirectory)", str(agent_temp)) + script = script.replace("$(RELEASE_BRANCH)", "release") + result = subprocess.run( + ["bash", "-c", script], + cwd=repo, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, ( + f"scoped release replay failed\nstdout:\n{result.stdout}\n" + f"stderr:\n{result.stderr}" + ) + assert scoped_file.read_text() == "pull request change\n" + assert not unrelated_file.exists() + assert _git(repo, "diff", "--cached", "--name-only").stdout.splitlines() == [ + scoped_path + ] + assert f"Prerequisite {prerequisite} applies cleanly" in result.stdout + assert "PR changes apply cleanly onto release" in result.stdout + finally: + shutil.rmtree(scratch_root, ignore_errors=True) + + +@pytest.mark.skipif(os.name != "posix", reason="release replay script requires Bash") +def test_release_compat_skips_scoped_prerequisite_for_unrelated_pr_path(): + scratch_root = REPO_ROOT / "target" / f"release-compat-unrelated-{uuid.uuid4().hex}" + repo = scratch_root / "repo" + origin = scratch_root / "origin.git" + agent_temp = scratch_root / "agent" + + try: + repo.mkdir(parents=True) + agent_temp.mkdir() + _init_release_compat_scratch_repo(repo) + + pr_file = repo / "src" / "pr.txt" + pr_file.parent.mkdir() + pr_file.write_text("release base\n") + _git(repo, "add", ".") + _git(repo, "commit", "-m", "base") + base = _git(repo, "rev-parse", "HEAD").stdout.strip() + _git(repo, "branch", "release", base) + + scoped_file = repo / "src" / "scoped.txt" + _git(repo, "commit", "--allow-empty", "-m", "prerequisite marker") + prerequisite = _git(repo, "rev-parse", "HEAD").stdout.strip() + + _git(repo, "checkout", "-b", "source") + prerequisite_config = repo / ".pipelines" / "release-compat-prerequisites.txt" + prerequisite_config.parent.mkdir() + prerequisite_config.write_text(f"{prerequisite}\tsrc/scoped.txt\n") + pr_file.write_text("pull request change\n") + _git(repo, "add", ".") + _git(repo, "commit", "-m", "feature modifies unrelated file") + + _git(repo, "checkout", "master") + _assert_git_clean(repo, "checkout before synthetic PR merge") + _git(repo, "merge", "--no-ff", "source", "-m", "merge feature") + + subprocess.run( + ["git", "init", "--bare", str(origin)], + check=True, + capture_output=True, + text=True, + ) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "push", "origin", "master", "source", "release") + + script = _release_compat_script() + script = script.replace("$(Agent.TempDirectory)", str(agent_temp)) + script = script.replace("$(RELEASE_BRANCH)", "release") + result = subprocess.run( + ["bash", "-c", script], + cwd=repo, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, ( + f"unrelated-path release replay failed\nstdout:\n{result.stdout}\n" + f"stderr:\n{result.stderr}" + ) + assert pr_file.read_text() == "pull request change\n" + assert not scoped_file.exists() + assert _git(repo, "diff", "--cached", "--name-only").stdout.splitlines() == [ + "src/pr.txt" + ] + assert ( + f"Scoped prerequisite {prerequisite} has no paths in this PR replay; skipping" + in result.stdout + ) + assert f"Prerequisite {prerequisite} applies cleanly" not in result.stdout + assert "PR changes apply cleanly onto release" in result.stdout + finally: + shutil.rmtree(scratch_root, ignore_errors=True) + + +@pytest.mark.skipif(os.name != "posix", reason="release replay script requires Bash") +def test_release_compat_replays_scoped_baseline_for_later_prerequisite(): + scratch_root = REPO_ROOT / "target" / f"release-compat-dependent-{uuid.uuid4().hex}" + repo = scratch_root / "repo" + origin = scratch_root / "origin.git" + agent_temp = scratch_root / "agent" + + try: + repo.mkdir(parents=True) + agent_temp.mkdir() + _init_release_compat_scratch_repo(repo) + + pr_file = repo / "src" / "pr.txt" + pr_file.parent.mkdir() + pr_file.write_text("release base\n") + _git(repo, "add", ".") + _git(repo, "commit", "-m", "base") + base = _git(repo, "rev-parse", "HEAD").stdout.strip() + _git(repo, "branch", "release", base) + + scoped_file = repo / "src" / "scoped.txt" + scoped_file.write_text("target baseline\n") + _git(repo, "add", ".") + _git(repo, "commit", "-m", "add scoped baseline") + scoped_prerequisite = _git(repo, "rev-parse", "HEAD").stdout.strip() + + scoped_file.write_text("dependent prerequisite\n") + _git(repo, "commit", "-am", "modify scoped baseline") + dependent_prerequisite = _git(repo, "rev-parse", "HEAD").stdout.strip() + + _git(repo, "checkout", "-b", "source") + prerequisite_config = repo / ".pipelines" / "release-compat-prerequisites.txt" + prerequisite_config.parent.mkdir() + prerequisite_config.write_text( + f"{scoped_prerequisite}\tsrc/scoped.txt\n{dependent_prerequisite}\n" + ) + pr_file.write_text("pull request change\n") + _git(repo, "add", ".") + _git(repo, "commit", "-m", "feature modifies unrelated file") + + _git(repo, "checkout", "master") + _assert_git_clean(repo, "checkout before synthetic PR merge") + _git(repo, "merge", "--no-ff", "source", "-m", "merge feature") + + subprocess.run( + ["git", "init", "--bare", str(origin)], + check=True, + capture_output=True, + text=True, + ) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "push", "origin", "master", "source", "release") + + script = _release_compat_script() + script = script.replace("$(Agent.TempDirectory)", str(agent_temp)) + script = script.replace("$(RELEASE_BRANCH)", "release") + result = subprocess.run( + ["bash", "-c", script], + cwd=repo, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, ( + f"dependent release replay failed\nstdout:\n{result.stdout}\n" + f"stderr:\n{result.stderr}" + ) + assert pr_file.read_text() == "pull request change\n" + assert scoped_file.read_text() == "dependent prerequisite\n" + assert _git(repo, "diff", "--cached", "--name-only").stdout.splitlines() == [ + "src/pr.txt", + "src/scoped.txt", + ] + assert ( + f"Scoped prerequisite {scoped_prerequisite} path src/scoped.txt " + f"is required by later prerequisite {dependent_prerequisite}" + in result.stdout + ) + assert "PR changes apply cleanly onto release" in result.stdout + finally: + shutil.rmtree(scratch_root, ignore_errors=True) + + +@pytest.mark.parametrize("modify_scoped_path", [False, True]) +@pytest.mark.skipif(os.name != "posix", reason="release replay script requires Bash") +def test_release_compat_reports_dependent_prerequisite_without_parent( + modify_scoped_path, +): + scratch_root = ( + REPO_ROOT + / "target" + / (f"release-compat-parent-{modify_scoped_path}-{uuid.uuid4().hex}") + ) + repo = scratch_root / "repo" + origin = scratch_root / "origin.git" + agent_temp = scratch_root / "agent" + + try: + repo.mkdir(parents=True) + agent_temp.mkdir() + _init_release_compat_scratch_repo(repo) + + base_file = repo / "src" / "base.txt" + base_file.parent.mkdir() + base_file.write_text("release base\n") + _git(repo, "add", ".") + _git(repo, "commit", "-m", "root base") + root_prerequisite = _git(repo, "rev-parse", "HEAD").stdout.strip() + _git(repo, "branch", "release", root_prerequisite) + + scoped_file = repo / "src" / "scoped.txt" + scoped_file.write_text("scoped baseline\n") + _git(repo, "add", ".") + _git(repo, "commit", "-m", "add scoped baseline") + scoped_prerequisite = _git(repo, "rev-parse", "HEAD").stdout.strip() + + _git(repo, "checkout", "-b", "source") + prerequisite_config = repo / ".pipelines" / "release-compat-prerequisites.txt" + prerequisite_config.parent.mkdir() + prerequisite_config.write_text( + f"{scoped_prerequisite}\tsrc/scoped.txt\n{root_prerequisite}\n" + ) + if modify_scoped_path: + scoped_file.write_text("pull request change\n") + else: + pr_file = repo / "src" / "pr.txt" + pr_file.write_text("pull request change\n") + _git(repo, "add", ".") + _git(repo, "commit", "-m", "feature change") + + _git(repo, "checkout", "master") + _assert_git_clean(repo, "checkout before synthetic PR merge") + _git(repo, "merge", "--no-ff", "source", "-m", "merge feature") + + subprocess.run( + ["git", "init", "--bare", str(origin)], + check=True, + capture_output=True, + text=True, + ) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "push", "origin", "master", "source", "release") + + script = _release_compat_script() + script = script.replace("$(Agent.TempDirectory)", str(agent_temp)) + script = script.replace("$(RELEASE_BRANCH)", "release") + result = subprocess.run( + ["bash", "-c", script], + cwd=repo, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 1 + assert ( + "##vso[task.logissue type=error]Dependent release compatibility " + f"prerequisite {root_prerequisite} has no first parent" in result.stdout + ) + assert "fatal:" not in result.stderr + finally: + shutil.rmtree(scratch_root, ignore_errors=True) + + +@pytest.mark.skipif(os.name != "posix", reason="release replay script requires Bash") +def test_release_compat_replays_only_partial_scoped_path_intersection(): + scratch_root = REPO_ROOT / "target" / f"release-compat-partial-{uuid.uuid4().hex}" + repo = scratch_root / "repo" + origin = scratch_root / "origin.git" + agent_temp = scratch_root / "agent" + + try: + repo.mkdir(parents=True) + agent_temp.mkdir() + _init_release_compat_scratch_repo(repo) + + base_file = repo / "base.txt" + base_file.write_text("release base\n") + _git(repo, "add", ".") + _git(repo, "commit", "-m", "base") + base = _git(repo, "rev-parse", "HEAD").stdout.strip() + _git(repo, "branch", "release", base) + + intersecting_file = repo / "src" / "intersecting.txt" + nonintersecting_file = repo / "src" / "nonintersecting.txt" + intersecting_file.parent.mkdir() + intersecting_file.write_text("target baseline\n") + _git(repo, "add", ".") + _git(repo, "commit", "-m", "prerequisite adds intersecting file") + prerequisite = _git(repo, "rev-parse", "HEAD").stdout.strip() + + _git(repo, "checkout", "-b", "source") + prerequisite_config = repo / ".pipelines" / "release-compat-prerequisites.txt" + prerequisite_config.parent.mkdir() + prerequisite_config.write_text( + f"{prerequisite}\tsrc/intersecting.txt\tsrc/nonintersecting.txt\n" + ) + intersecting_file.write_text("pull request change\n") + _git(repo, "add", ".") + _git(repo, "commit", "-m", "feature modifies one scoped file") + + _git(repo, "checkout", "master") + _assert_git_clean(repo, "checkout before synthetic PR merge") + _git(repo, "merge", "--no-ff", "source", "-m", "merge feature") + + subprocess.run( + ["git", "init", "--bare", str(origin)], + check=True, + capture_output=True, + text=True, + ) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "push", "origin", "master", "source", "release") + + script = _release_compat_script() + script = script.replace("$(Agent.TempDirectory)", str(agent_temp)) + script = script.replace("$(RELEASE_BRANCH)", "release") + result = subprocess.run( + ["bash", "-c", script], + cwd=repo, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, ( + f"partial scoped release replay failed\nstdout:\n{result.stdout}\n" + f"stderr:\n{result.stderr}" + ) + assert intersecting_file.read_text() == "pull request change\n" + assert not nonintersecting_file.exists() + assert _git(repo, "diff", "--cached", "--name-only").stdout.splitlines() == [ + "src/intersecting.txt" + ] + assert f"Prerequisite {prerequisite} applies cleanly" in result.stdout + assert "PR changes apply cleanly onto release" in result.stdout + finally: + shutil.rmtree(scratch_root, ignore_errors=True) + + def test_acr_cleanup_is_schedule_only_and_uses_dedicated_identity(): data = yaml.safe_load(CLEAN_ACR_PIPELINE.read_text()) assert data["trigger"] == "none" diff --git a/tools/docker/demo/Dockerfile b/tools/docker/demo/Dockerfile index bb5f4e89d6c..a4cc01e8fd7 100644 --- a/tools/docker/demo/Dockerfile +++ b/tools/docker/demo/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/mirror/docker/library/ubuntu:22.04 +FROM mcr.microsoft.com/mirror/docker/library/ubuntu:22.04@sha256:104ae83764a5119017b8e8d6218fa0832b09df65aae7d5a6de29a85d813da2fb ARG SYNAPSEML_VERSION=1.1.3 ARG DEBIAN_FRONTEND=noninteractive diff --git a/tools/docker/minimal/Dockerfile b/tools/docker/minimal/Dockerfile index a27b72d2131..b3f0e6c3c7e 100644 --- a/tools/docker/minimal/Dockerfile +++ b/tools/docker/minimal/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/mirror/docker/library/ubuntu:22.04 +FROM mcr.microsoft.com/mirror/docker/library/ubuntu:22.04@sha256:104ae83764a5119017b8e8d6218fa0832b09df65aae7d5a6de29a85d813da2fb ARG SYNAPSEML_VERSION=1.1.3 ARG DEBIAN_FRONTEND=noninteractive diff --git a/tools/helm/livy/Dockerfile b/tools/helm/livy/Dockerfile index 19c4fffaac3..c08995c168c 100644 --- a/tools/helm/livy/Dockerfile +++ b/tools/helm/livy/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/openjdk/jdk:11-mariner +FROM mcr.microsoft.com/openjdk/jdk:11-mariner@sha256:eea2eae2b8dc62991a43018ca65f895238f43ee0d94a55a7929dbaf7d4bfc7c7 LABEL maintainer="Dalitso Banda dalitsohb@gmail.com" # Get Spark from US Apache mirror. diff --git a/tools/helm/livy/mini.Dockerfile b/tools/helm/livy/mini.Dockerfile index 07caa82f0e6..fecd622880c 100644 --- a/tools/helm/livy/mini.Dockerfile +++ b/tools/helm/livy/mini.Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/mmlspark/spark2.4:v4_mini +FROM mcr.microsoft.com/mmlspark/spark2.4:v4_mini@sha256:a7da0d7cd86ab374d1f0dc7ae4cd35260f8798f8e40a4e4e818748f61a389279 MAINTAINER Dalitso Banda ENV LIVY_VERSION="git_master" diff --git a/tools/helm/spark/Dockerfile b/tools/helm/spark/Dockerfile index d5200fc15a0..b23729c93b6 100644 --- a/tools/helm/spark/Dockerfile +++ b/tools/helm/spark/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/openjdk/jdk:11-mariner +FROM mcr.microsoft.com/openjdk/jdk:11-mariner@sha256:eea2eae2b8dc62991a43018ca65f895238f43ee0d94a55a7929dbaf7d4bfc7c7 LABEL maintainer="Dalitso Banda dalitsohb@gmail.com" # Get Spark from US Apache mirror. diff --git a/tools/helm/spark/mini.Dockerfile b/tools/helm/spark/mini.Dockerfile index 05913f4b0b0..6d54cf163da 100644 --- a/tools/helm/spark/mini.Dockerfile +++ b/tools/helm/spark/mini.Dockerfile @@ -15,7 +15,7 @@ # limitations under the License. # -FROM mcr.microsoft.com/openjdk/jdk:11-mariner +FROM mcr.microsoft.com/openjdk/jdk:11-mariner@sha256:eea2eae2b8dc62991a43018ca65f895238f43ee0d94a55a7929dbaf7d4bfc7c7 ARG spark_jars=jars ARG img_path=kubernetes/dockerfiles diff --git a/tools/helm/zeppelin/Dockerfile b/tools/helm/zeppelin/Dockerfile index 6f92ed02039..0eb6ecd8eb4 100644 --- a/tools/helm/zeppelin/Dockerfile +++ b/tools/helm/zeppelin/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/openjdk/jdk:11-mariner +FROM mcr.microsoft.com/openjdk/jdk:11-mariner@sha256:eea2eae2b8dc62991a43018ca65f895238f43ee0d94a55a7929dbaf7d4bfc7c7 LABEL maintainer="Dalitso Banda dalitsohb@gmail.com" # Get Spark from US Apache mirror. diff --git a/tools/helm/zeppelin/mini.Dockerfile b/tools/helm/zeppelin/mini.Dockerfile index 6b126a81543..b0f751a4bd5 100644 --- a/tools/helm/zeppelin/mini.Dockerfile +++ b/tools/helm/zeppelin/mini.Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/mmlspark/spark2.4:v4_mini +FROM mcr.microsoft.com/mmlspark/spark2.4:v4_mini@sha256:a7da0d7cd86ab374d1f0dc7ae4cd35260f8798f8e40a4e4e818748f61a389279 MAINTAINER Dalitso Banda ADD patch_beam.patch /tmp/patch_beam.patch diff --git a/vw/src/test/scala/com/microsoft/azure/synapse/ml/vw/VerifyVectorUtils.scala b/vw/src/test/scala/com/microsoft/azure/synapse/ml/vw/VerifyVectorUtils.scala new file mode 100644 index 00000000000..13c5365c301 --- /dev/null +++ b/vw/src/test/scala/com/microsoft/azure/synapse/ml/vw/VerifyVectorUtils.scala @@ -0,0 +1,73 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in project root for information. + +package com.microsoft.azure.synapse.ml.vw + +import com.microsoft.azure.synapse.ml.core.test.base.TestBase + +// scalastyle:off magic.number +class VerifyVectorUtils extends TestBase { + + test("sortAndDistinct with empty arrays returns empty arrays") { + val (indices, values) = VectorUtils.sortAndDistinct(Array[Int](), Array[Double]()) + assert(indices.isEmpty) + assert(values.isEmpty) + } + + test("sortAndDistinct sorts indices and values together") { + val (indices, values) = VectorUtils.sortAndDistinct( + Array(3, 1, 2), Array(30.0, 10.0, 20.0)) + assert(indices === Array(1, 2, 3)) + assert(values === Array(10.0, 20.0, 30.0)) + } + + test("sortAndDistinct deduplicates and sums collisions by default") { + val (indices, values) = VectorUtils.sortAndDistinct( + Array(1, 2, 1), Array(10.0, 20.0, 5.0)) + assert(indices === Array(1, 2)) + assert(values === Array(15.0, 20.0)) + } + + test("sortAndDistinct deduplicates without summing when sumCollisions is false") { + val (indices, values) = VectorUtils.sortAndDistinct( + Array(1, 2, 1), Array(10.0, 20.0, 10.0), sumCollisions = false) + assert(indices === Array(1, 2)) + assert(values === Array(10.0, 20.0)) + } + + test("sortAndDistinct with single element") { + val (indices, values) = VectorUtils.sortAndDistinct( + Array(5), Array(42.0)) + assert(indices === Array(5)) + assert(values === Array(42.0)) + } + + test("sortAndDistinct with already sorted no-duplicate input") { + val (indices, values) = VectorUtils.sortAndDistinct( + Array(1, 2, 3, 4), Array(1.0, 2.0, 3.0, 4.0)) + assert(indices === Array(1, 2, 3, 4)) + assert(values === Array(1.0, 2.0, 3.0, 4.0)) + } + + test("sortAndDistinct with all duplicate indices sums to single element") { + val (indices, values) = VectorUtils.sortAndDistinct( + Array(5, 5, 5), Array(1.0, 2.0, 3.0)) + assert(indices === Array(5)) + assert(values === Array(6.0)) + } + + test("sortAndDistinct with multiple collision groups") { + val (indices, values) = VectorUtils.sortAndDistinct( + Array(3, 1, 3, 1, 2), Array(1.0, 2.0, 3.0, 4.0, 5.0)) + assert(indices === Array(1, 2, 3)) + assert(values === Array(6.0, 5.0, 4.0)) + } + + test("sortAndDistinct preserves negative values") { + val (indices, values) = VectorUtils.sortAndDistinct( + Array(2, 1), Array(-5.0, -3.0)) + assert(indices === Array(1, 2)) + assert(values === Array(-3.0, -5.0)) + } +} +// scalastyle:on magic.number diff --git a/website/package-lock.json b/website/package-lock.json index f07c9227a17..aef7b374391 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -5973,8 +5973,8 @@ } }, "node_modules/brace-expansion": { - "version": "1.1.16", - "integrity": "sha512-IDw48K2/2kRkg9LdJxurvq3lV3aBgq0REY89duEqFRthjlPdXHKMj7EnQOXVckxzgisinf3nHfrcE2FufFLXMw==", + "version": "1.1.18", + "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", @@ -8044,8 +8044,8 @@ "license": "MIT" }, "node_modules/fast-uri": { - "version": "3.1.4", - "integrity": "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw==", + "version": "3.1.5", + "integrity": "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==", "funding": [ { "type": "github", @@ -9618,8 +9618,8 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "4.3.0", - "integrity": "sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q==", + "version": "4.3.1", + "integrity": "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==", "funding": [ { "type": "github", @@ -12368,8 +12368,8 @@ } }, "node_modules/nanoid": { - "version": "3.3.16", - "integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==", + "version": "3.3.18", + "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", "funding": [ { "type": "github", diff --git a/website/package.json b/website/package.json index 4147b10186f..4ffa13c705f 100644 --- a/website/package.json +++ b/website/package.json @@ -34,13 +34,14 @@ "overrides": { "ajv@<6.14.0": "6.14.0", "ajv@>=7.0.0-alpha.0 <8.18.0": "8.20.0", - "brace-expansion@<1.1.16": "1.1.16", + "brace-expansion@<1.1.18": "1.1.18", "cross-spawn@>=7.0.0 <7.0.5": "7.0.6", - "fast-uri@<3.1.4": "3.1.4", + "fast-uri@<3.1.5": "3.1.5", "follow-redirects@<=1.15.11": "1.16.0", - "js-yaml@>=4.0.0 <4.3.0": "4.3.0", + "js-yaml@>=4.0.0 <4.3.1": "4.3.1", "lodash@<=4.17.23": "4.18.1", "micromatch@<4.0.8": "4.0.8", + "nanoid@<3.3.18": "3.3.18", "path-to-regexp@>=0.2.0 <1.9.0": "1.9.0", "serialize-javascript": "7.0.7", "shell-quote@<1.10.0": "1.10.0",