fix(datasets): attribute subpackage exports per module, matching tasks/

scripts/sync_package_stubs.py

@@ -60,14 +60,17 @@ def _register_export(
 
 def _discover_task_classes(
     module_paths: list[Path],
+    prefix: str = "",
 ) -> dict[str, str]:
     """Scan *module_paths* for public ``*Task`` class definitions.
 
-    Returns ``{ClassName: module_stem}`` mapping.
+    Returns ``{ClassName: module_stem}``, or ``{ClassName: "prefix.module_stem"}`` when
+    *prefix* is given, so the duplicate-export error names the module the caller
+    registers rather than a bare stem.
     """
     export_to_module: dict[str, str] = {}
     for module_path in module_paths:
-        module_name = module_path.stem
+        module_name = f"{prefix}.{module_path.stem}" if prefix else module_path.stem
         module_ast = ast.parse(
             module_path.read_text(encoding="utf-8"),
             filename=str(module_path),
@@ -90,36 +93,31 @@ def discover_tasks(package_dir: Path) -> dict[str, str]:
     for name, mod in _discover_task_classes(_iter_module_paths(package_dir)).items():
         _register_export(export_to_module, name, mod, "task")
 
-    # 2) Subpackage task modules — scan .py files inside each subpackage
+    # 2) Subpackage task modules — "subpkg.module_stem", matching the runtime
+    # registry in sieval/tasks/__init__.py. Must stay in sync with it.
     for subpkg_dir in _iter_subpackage_dirs(package_dir):
-        subpkg_name = subpkg_dir.name
-        for name, _mod in _discover_task_classes(
-            _iter_module_paths(subpkg_dir)
+        for name, mod in _discover_task_classes(
+            _iter_module_paths(subpkg_dir), prefix=subpkg_dir.name
         ).items():
-            _register_export(export_to_module, name, subpkg_name, "task")
+            _register_export(export_to_module, name, mod, "task")
 
     return export_to_module
 
 
-def discover_subpackage_tasks(subpkg_dir: Path) -> dict[str, str]:
-    """Discover task exports within a single subpackage directory.
-
-    Returns ``{ClassName: module_stem}`` mapping (module-level, not
-    prefixed with the subpackage name).
-    """
-    return _discover_task_classes(_iter_module_paths(subpkg_dir))
-
-
-def _discover_dataset_classes(module_paths: list[Path]) -> dict[str, str]:
+def _discover_dataset_classes(
+    module_paths: list[Path],
+    prefix: str = "",
+) -> dict[str, str]:
     """Scan *module_paths* for public Dataset/DatasetSample/CSVSample class definitions.
 
-    Returns ``{ClassName: module_stem}`` mapping.
+    Returns ``{ClassName: module_stem}``, or ``{ClassName: "prefix.module_stem"}`` when
+    *prefix* is given, as for ``_discover_task_classes``.
     """
     export_to_module: dict[str, str] = {}
     suffixes = ("Dataset", "DatasetSample", "CSVSample")
 
     for module_path in module_paths:
-        module_name = module_path.stem
+        module_name = f"{prefix}.{module_path.stem}" if prefix else module_path.stem
         module_ast = ast.parse(
             module_path.read_text(encoding="utf-8"),
             filename=str(module_path),
@@ -160,13 +158,12 @@ def discover_datasets(package_dir: Path) -> dict[str, str]:
     for name, mod in _discover_dataset_classes(_iter_module_paths(package_dir)).items():
         _register_export(export_to_module, name, mod, "dataset")
 
-    # 2) Subpackage dataset modules — scan .py files inside each subpackage
+    # 2) Subpackage dataset modules — "subpkg.module_stem", as for tasks above.
     for subpkg_dir in _iter_subpackage_dirs(package_dir):
-        subpkg_name = subpkg_dir.name
-        for name, _mod in _discover_dataset_classes(
-            _iter_module_paths(subpkg_dir)
+        for name, mod in _discover_dataset_classes(
+            _iter_module_paths(subpkg_dir), prefix=subpkg_dir.name
         ).items():
-            _register_export(export_to_module, name, subpkg_name, "dataset")
+            _register_export(export_to_module, name, mod, "dataset")
 
     return export_to_module
 
@@ -244,14 +241,13 @@ def main() -> int:
         check=args.check,
     )
 
-    # Task subpackages
-    for subpkg_dir in _iter_subpackage_dirs(TASKS_DIR):
-        ok &= sync_stub(
-            subpkg_dir / "__init__.pyi",
-            render_stub(discover_subpackage_tasks(subpkg_dir)),
-            check=args.check,
-        )
-
+    # Only the two lazy packages get a stub, because only they have a runtime export
+    # map for one to mirror. Subpackages are ordinary packages: their own __init__.py
+    # is their type surface, and a generated .pyi would *shadow* it rather than
+    # supplement it — hiding whatever suffix-based discovery cannot see (ruler/'s
+    # helpers re-exported from _shared.py) or emptying it outright (downloaders/,
+    # which is infrastructure and carries no dataset suffix at all). Don't add a
+    # per-subpackage loop for either package.
     if ok:
         return 0
 

sieval/datasets/CLAUDE.md

@@ -13,3 +13,15 @@
 - `deps_group` here is **loader-side** deps; evaluator-side deps stay on the Task.
 - `hf:` sources are revision-pinned; `url:` sources carry per-file `checksums` (sha256).
   Regenerate the meta index (`scripts/sync_meta_index.py`) after editing either.
+
+## Subpackages
+
+A multi-module benchmark gets a subdirectory; `datasets/__init__.py` lazy-loads it.
+
+- Registered as `subpkg.module_stem`, so the registry never needs the subpackage's
+  `__init__.py` to re-export. Re-export only what other packages import directly.
+- Discovery skips `_*.py`, and the duplicate-export guard applies *within* a
+  subpackage — two modules exporting one name is an error, not last-one-wins.
+- No generated `__init__.pyi` — the hand-written `__init__.py` is the type surface, and
+  a stub would shadow it rather than supplement it. Only `datasets/__init__.py` itself
+  gets one, because only it has a runtime export map to mirror.

sieval/datasets/__init__.py

@@ -56,11 +56,8 @@ def _is_typeddict_call(node: ast.AST) -> bool:
 
 
 def _scan_dataset_exports(module_path: Path) -> list[str]:
-    """Return public dataset export names defined in *module_path* (AST only).
-
-    Classes whose name ends in one of ``_DATASET_EXPORT_SUFFIXES``, plus
-    module-level ``X = TypedDict(...)`` assignments with such a name.
-    """
+    """Return public ``_DATASET_EXPORT_SUFFIXES``-suffixed classes and ``TypedDict``
+    assignments defined in *module_path* (AST only)."""
     tree = ast.parse(module_path.read_text(encoding="utf-8"), filename=str(module_path))
 
     names: list[str] = []
@@ -81,11 +78,9 @@ def _scan_dataset_exports(module_path: Path) -> list[str]:
 def _discover_dataset_exports() -> dict[str, str]:
     """Map each export name to the module ``__getattr__`` should import it from.
 
-    Both passes share one scanner, so they cannot drift apart on what counts as an
-    export. ``scripts/sync_package_stubs.py`` reimplements this scan deliberately —
-    the stub generator must not import the package it generates stubs for — so the
-    two implementations have to agree for the generated ``.pyi`` to match runtime
-    resolution.
+    ``scripts/sync_package_stubs.py`` reimplements this scan deliberately — the stub
+    generator must not import the package it generates stubs for — so the two have to
+    agree for the generated ``.pyi`` to match runtime resolution.
     """
     export_to_module: dict[str, str] = {}
 
@@ -103,13 +98,15 @@ def _register(export_name: str, module_name: str) -> None:
         for name in _scan_dataset_exports(module_path):
             _register(name, module_path.stem)
 
-    # 2) Subpackage .py modules — mapped to the subpackage, which re-exports them
-    # from its __init__. (The task registry instead maps to "subpkg.module_stem";
-    # the two conventions differ, see sieval/tasks/__init__.py.)
+    # 2) Subpackage .py modules — "subpkg.module_stem", matching sieval/tasks/
+    # __init__.py. Resolving to the defining module keeps the registry independent of
+    # what the subpackage's __init__ re-exports, and keeps _register's guard effective
+    # within a subpackage (the bare subpackage name made same-named exports collide
+    # silently).
     for subpkg_dir in _iter_subpackage_dirs():
         for module_path in _iter_module_paths_in(subpkg_dir):
             for name in _scan_dataset_exports(module_path):
-                _register(name, subpkg_dir.name)
+                _register(name, f"{subpkg_dir.name}.{module_path.stem}")
 
     return export_to_module
 

sieval/datasets/__init__.pyi

@@ -117,7 +117,7 @@ from .openbookqa import (
     OpenBookQADataset,
     OpenBookQADatasetSample,
 )
-from .ruler import (
+from .ruler.ruler import (
     RulerDataset,
     RulerDatasetSample,
 )

sieval/datasets/ruler/__init__.py

@@ -1,18 +1,8 @@
-from ._shared import (
-    RulerTaskSpec,
-    len_tag,
-    ruler_task,
-    thinking_prefill,
-    tokens_to_generate,
-)
-from .ruler import RulerDataset, RulerDatasetSample
+from ._shared import len_tag, thinking_prefill
+from .ruler import RulerDatasetSample
 
 __all__ = [
-    "RulerDataset",
     "RulerDatasetSample",
-    "RulerTaskSpec",
     "len_tag",
-    "ruler_task",
-    "tokens_to_generate",
     "thinking_prefill",
 ]

sieval/tasks/CLAUDE.md

@@ -23,7 +23,7 @@ Class: `<Benchmark><ShotType><Mode>Task` — words for shot count (`ZeroShot`, `
 
 ## Key Rules
 
-- ≥ 5 task files per benchmark → subdirectory with an empty `__init__.py` (lazy loading is handled by the top-level `tasks/__init__.py`).
+- ≥ 5 task files per benchmark → subdirectory with an empty `__init__.py` (lazy loading is handled by the top-level `tasks/__init__.py`). Empty on purpose: nothing imports a task class by name — the registry resolves it through the top-level package — so the subpackage needs no type surface of its own, and leaving it empty keeps exactly one import path per task. Contrast `sieval/datasets/`, whose subpackages do re-export, because tasks import them directly.
 - Subpackage shared base module: file named `_base.py` (private module), classes inside without underscore prefix (package-internal public API, e.g. `from ._base import XxxTask`).
 - General code-quality + layer rules live in `.claude/rules/tasks.md`.
 

tests/unit/datasets/test_ruler.py

@@ -7,7 +7,7 @@
 
 import pytest
 
-from sieval.datasets.ruler import tokens_to_generate
+from sieval.datasets.ruler._shared import tokens_to_generate
 
 try:
     import tiktoken as _tiktoken  # noqa: F401
@@ -21,8 +21,12 @@
 )
 
 if _ruler_deps:
-    from sieval.datasets.ruler import RulerDataset, RulerDatasetSample
-    from sieval.datasets.ruler.ruler import _stamp, _subtask_data_path
+    from sieval.datasets.ruler.ruler import (
+        RulerDataset,
+        RulerDatasetSample,
+        _stamp,
+        _subtask_data_path,
+    )
 
 
 # ---------------------------------------------------------------------------

tests/unit/test_lazy_exports.py

@@ -146,10 +146,46 @@ def fake_import(module_name: str) -> ModuleType:
     assert package.__dict__[sample_export] is dummy_value
 
 
+def _read_stub_import_map(stub_path: Path) -> dict[str, str]:
+    """Map each name the stub imports to the relative module it comes from."""
+    module_ast = ast.parse(
+        stub_path.read_text(encoding="utf-8"),
+        filename=str(stub_path),
+    )
+    import_map: dict[str, str] = {}
+    for node in module_ast.body:
+        if isinstance(node, ast.ImportFrom) and node.level == 1 and node.module:
+            for alias in node.names:
+                import_map[alias.asname or alias.name] = node.module
+    if not import_map:
+        raise AssertionError(f"{stub_path} declares no relative imports")
+    return import_map
+
+
 @pytest.mark.parametrize("package_name", ["sieval.tasks", "sieval.datasets"])
 def test_stub_exports_match_runtime_exports(package_name: str) -> None:
     package = importlib.import_module(package_name)
     package_file = package.__file__
     assert package_file is not None
     stub_exports = _read_stub_all(Path(package_file).with_suffix(".pyi"))
     assert stub_exports == package.__all__
+
+
+@pytest.mark.parametrize("package_name", ["sieval.tasks", "sieval.datasets"])
+def test_stub_import_targets_match_runtime_module_map(package_name: str) -> None:
+    """The stub's per-name import target must match the runtime map's values.
+
+    Nothing else compares them: the test above checks only export *names*, and
+    preflight builds its fqn from the runtime map alone. So a stub pointing an
+    export at the wrong module ships silently, and ty resolves
+    ``from sieval.datasets import X`` through a module that never defines it.
+    """
+    package = importlib.import_module(package_name)
+    package_file = package.__file__
+    assert package_file is not None
+
+    stub_map = _read_stub_import_map(Path(package_file).with_suffix(".pyi"))
+    # Via __dict__ because the .pyi deliberately does not declare this global.
+    runtime_map = dict(package.__dict__["_EXPORT_TO_MODULE"])
+
+    assert stub_map == runtime_map