diff --git a/python/private/pypi/whl_installer/BUILD.bazel b/python/private/pypi/whl_installer/BUILD.bazel index b820b843d4..1912b34af8 100644 --- a/python/private/pypi/whl_installer/BUILD.bazel +++ b/python/private/pypi/whl_installer/BUILD.bazel @@ -5,7 +5,6 @@ py_library( name = "lib", srcs = [ "arguments.py", - "wheel.py", "wheel_installer.py", ], visibility = [ diff --git a/python/private/pypi/whl_installer/arguments.py b/python/private/pypi/whl_installer/arguments.py index 0d31ab7c4d..9122654a11 100644 --- a/python/private/pypi/whl_installer/arguments.py +++ b/python/private/pypi/whl_installer/arguments.py @@ -55,11 +55,6 @@ def parser(**kwargs: Any) -> argparse.ArgumentParser: help="Use 'pip download' instead of 'pip wheel'. Disables building wheels from source, but allows use of " "--platform, --python-version, --implementation, and --abi in --extra_pip_args.", ) - parser.add_argument( - "--whl-file", - type=pathlib.Path, - help="Extract a whl file to be used within Bazel.", - ) return parser diff --git a/python/private/pypi/whl_installer/wheel.py b/python/private/pypi/whl_installer/wheel.py deleted file mode 100644 index 801fd0f3b9..0000000000 --- a/python/private/pypi/whl_installer/wheel.py +++ /dev/null @@ -1,105 +0,0 @@ -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Utility class to inspect an extracted wheel directory""" - -import email -from pathlib import Path - -import installer - - -class DoNothingCm: - """A context manager that does nothing when written to.""" - - def __enter__(self): - return self - - def __exit__(self, *args): - pass - - def write(self, data): - pass - - -class NoEntryPointsSchemeDictionaryDestination( - installer.destinations.SchemeDictionaryDestination -): - """ - A custom destination that prevents the `installer` package from automatically - generating scripts for `console_scripts` entry points. - - rules_python handles entry points via its own `venv_entry_point` targets. - If `installer` also generates these scripts in the `bin/` directory, it - causes a target naming collision because `whl_library_targets.bzl` will - try to create a `venv_rewrite_shebang` target with the same name. - - By overriding `for_script` to return a no-op dummy writer, we silently - discard the generated entry point scripts while still allowing `installer` - to process the rest of the wheel normally (including `.data/scripts` which - we do want to keep). - """ - - def for_script(self, name, module, attribute): - return DoNothingCm() - - -class Wheel: - """Representation of the compressed .whl file""" - - def __init__(self, path: Path): - self._path = path - - @property - def path(self) -> Path: - return self._path - - @property - def metadata(self) -> email.message.Message: - with installer.sources.WheelFile.open(self.path) as wheel_source: - metadata_contents = wheel_source.read_dist_info("METADATA") - metadata = installer.utils.parse_metadata_file(metadata_contents) - return metadata - - @property - def version(self) -> str: - # TODO Also available as installer.sources.WheelSource.version - return str(self.metadata["Version"]) - - def unzip(self, directory: str) -> None: - installation_schemes = { - "purelib": "/site-packages", - "platlib": "/site-packages", - "headers": "/include", - "scripts": "/bin", - "data": "/data", - } - - destination = NoEntryPointsSchemeDictionaryDestination( - installation_schemes, - # TODO Should entry_point scripts also be handled by installer rather than custom code? - interpreter="python", - script_kind="posix", - destdir=directory, - bytecode_optimization_levels=[], - ) - - with installer.sources.WheelFile.open(self.path) as wheel_source: - installer.install( - source=wheel_source, - destination=destination, - additional_metadata={ - "INSTALLER": b"https://github.com/bazel-contrib/rules_python", - }, - ) diff --git a/python/private/pypi/whl_installer/wheel_installer.py b/python/private/pypi/whl_installer/wheel_installer.py index 1f00068060..81dd3995db 100644 --- a/python/private/pypi/whl_installer/wheel_installer.py +++ b/python/private/pypi/whl_installer/wheel_installer.py @@ -25,9 +25,7 @@ from tempfile import NamedTemporaryFile from typing import Dict, List, Optional, Set, Tuple -from pip._vendor.packaging.utils import canonicalize_name - -from python.private.pypi.whl_installer import arguments, wheel +from python.private.pypi.whl_installer import arguments def _configure_reproducible_wheels() -> None: @@ -55,45 +53,6 @@ def _configure_reproducible_wheels() -> None: os.environ["PYTHONHASHSEED"] = "0" -def _parse_requirement_for_extra( - requirement: str, -) -> Tuple[Optional[str], Optional[Set[str]]]: - """Given a requirement string, returns the requirement name and set of extras, if extras specified. - Else, returns (None, None) - """ - - # https://www.python.org/dev/peps/pep-0508/#grammar - extras_pattern = re.compile( - r"^\s*([0-9A-Za-z][0-9A-Za-z_.\-]*)\s*\[\s*([0-9A-Za-z][0-9A-Za-z_.\-]*(?:\s*,\s*[0-9A-Za-z][0-9A-Za-z_.\-]*)*)\s*\]" - ) - - matches = extras_pattern.match(requirement) - if matches: - return ( - canonicalize_name(matches.group(1)), - {extra.strip() for extra in matches.group(2).split(",")}, - ) - - return None, None - - -def _extract_wheel( - wheel_file: str, - extras: Dict[str, Set[str]], - installation_dir: Path = Path("."), -) -> None: - """Extracts wheel into given directory and creates py_library and filegroup targets. - - Args: - wheel_file: the filepath of the .whl - installation_dir: the destination directory for installation of the wheel. - extras: a list of extras to add as dependencies for the installed wheel - """ - - whl = wheel.Wheel(wheel_file) - whl.unzip(installation_dir) - - def main() -> None: args = arguments.parser(description=__doc__).parse_args() deserialized_args = dict(vars(args)) @@ -101,17 +60,6 @@ def main() -> None: _configure_reproducible_wheels() - if args.whl_file: - whl = Path(args.whl_file) - - name, extras_for_pkg = _parse_requirement_for_extra(args.requirement) - extras = {name: extras_for_pkg} if extras_for_pkg and name else dict() - _extract_wheel( - wheel_file=whl, - extras=extras, - ) - return - pip_args = ( [sys.executable, "-m", "pip"] + (["--isolated"] if args.isolated else []) diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index 529514578d..966d25d04d 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -14,7 +14,6 @@ "" -load("@rules_python_internal//:rules_python_config.bzl", rp_config = "config") load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth") load("//python/private:envsubst.bzl", "envsubst") load("//python/private:is_standalone_interpreter.bzl", "is_standalone_interpreter") @@ -261,22 +260,6 @@ def _create_repository_execution_environment(rctx, python_interpreter, logger = env[_CPPFLAGS] = " ".join(cppflags) return env -def _extract_whl_py(rctx, *, python_interpreter, args, whl_path, environment, logger): - pypi_repo_utils.execute_checked( - rctx, - op = "whl_library.ExtractWheel({}, {})".format(rctx.attr.name, whl_path), - python = python_interpreter, - arguments = args + [ - "--whl-file", - whl_path, - ], - srcs = rctx.attr._python_srcs, - environment = environment, - quiet = rctx.attr.quiet, - timeout = rctx.attr.timeout, - logger = logger, - ) - def _get_entry_points(rctx, install_dir_path, metadata): dist_info_dir = "{}-{}.dist-info".format( metadata.name.replace("-", "_"), @@ -376,11 +359,10 @@ def _whl_library_impl(rctx): # build deps from PyPI (e.g. `flit_core`) if they are missing. extra_pip_args.extend(["--find-links", "."]) - enable_pipstar_extract = rp_config.bazel_8_or_later - - # When pipstar is enabled, Python isn't used, so there's no need - # to setup env vars to run Python, unless we need to build an sdist - if enable_pipstar_extract and whl_path and not rctx.attr.whl_patches: + # When we already have a wheel and there are no patches, Python isn't used, + # so there's no need to setup env vars to run Python, unless we need to + # build an sdist or resolve a requirement. + if whl_path and not rctx.attr.whl_patches: environment = {} args = [] python_interpreter = None @@ -446,17 +428,7 @@ def _whl_library_impl(rctx): timeout = rctx.attr.timeout, ) - if enable_pipstar_extract: - whl_extract(rctx, whl_path = whl_path, logger = logger) - else: - _extract_whl_py( - rctx, - python_interpreter = python_interpreter, - args = args, - whl_path = whl_path, - environment = environment, - logger = logger, - ) + whl_extract(rctx, whl_path = whl_path, logger = logger) install_dir_path = whl_path.dirname.get_child("site-packages") metadata = whl_metadata( @@ -513,9 +485,8 @@ repo( _remove_files(rctx, "BUILD", "BUILD.bazel") rctx.file("BUILD.bazel", build_file_contents) - if enable_pipstar_extract: - if hasattr(rctx, "repo_metadata"): - return rctx.repo_metadata(reproducible = True) + if hasattr(rctx, "repo_metadata"): + return rctx.repo_metadata(reproducible = True) return None @@ -632,7 +603,6 @@ way to define whl_library and move whl patching to a separate place. INTERNAL US "_python_srcs": attr.label_list( # Used as a default value in a rule to ensure we fetch the dependencies. default = [ - Label("//python/private/pypi/whl_installer:wheel.py"), Label("//python/private/pypi/whl_installer:wheel_installer.py"), Label("//python/private/pypi/whl_installer:arguments.py"), ] + record_files.values(), diff --git a/tests/pypi/whl_installer/BUILD.bazel b/tests/pypi/whl_installer/BUILD.bazel index 0a859c0e4d..5a2efb1260 100644 --- a/tests/pypi/whl_installer/BUILD.bazel +++ b/tests/pypi/whl_installer/BUILD.bazel @@ -1,10 +1,5 @@ load("//python:py_test.bzl", "py_test") -alias( - name = "lib", - actual = "//python/private/pypi/whl_installer:lib", -) - py_test( name = "arguments_test", size = "small", @@ -12,18 +7,6 @@ py_test( "arguments_test.py", ], deps = [ - ":lib", - ], -) - -py_test( - name = "wheel_installer_test", - size = "small", - srcs = [ - "wheel_installer_test.py", - ], - data = ["//examples/wheel:minimal_with_py_package"], - deps = [ - ":lib", + "//python/private/pypi/whl_installer:lib", ], ) diff --git a/tests/pypi/whl_installer/arguments_test.py b/tests/pypi/whl_installer/arguments_test.py index 9e26849db0..c874445029 100644 --- a/tests/pypi/whl_installer/arguments_test.py +++ b/tests/pypi/whl_installer/arguments_test.py @@ -15,7 +15,7 @@ import json import unittest -from python.private.pypi.whl_installer import arguments, wheel +from python.private.pypi.whl_installer import arguments class ArgumentsTestCase(unittest.TestCase): diff --git a/tests/pypi/whl_installer/wheel_installer_test.py b/tests/pypi/whl_installer/wheel_installer_test.py deleted file mode 100644 index 52c44cf2de..0000000000 --- a/tests/pypi/whl_installer/wheel_installer_test.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import os -import shutil -import tempfile -import unittest -from pathlib import Path - -from python.private.pypi.whl_installer import wheel_installer - - -class TestRequirementExtrasParsing(unittest.TestCase): - def test_parses_requirement_for_extra(self) -> None: - cases = [ - ("name[foo]", ("name", frozenset(["foo"]))), - ("name[ Foo123 ]", ("name", frozenset(["Foo123"]))), - (" name1[ foo ] ", ("name1", frozenset(["foo"]))), - ("Name[foo]", ("name", frozenset(["foo"]))), - ("name_foo[bar]", ("name-foo", frozenset(["bar"]))), - ( - "name [fred,bar] @ http://foo.com ; python_version=='2.7'", - ("name", frozenset(["fred", "bar"])), - ), - ( - "name[quux, strange];python_version<'2.7' and platform_version=='2'", - ("name", frozenset(["quux", "strange"])), - ), - ( - "name; (os_name=='a' or os_name=='b') and os_name=='c'", - (None, None), - ), - ( - "name@http://foo.com", - (None, None), - ), - ] - - for case, expected in cases: - with self.subTest(): - self.assertTupleEqual( - wheel_installer._parse_requirement_for_extra(case), expected - ) - - -class TestWhlFilegroup(unittest.TestCase): - def setUp(self) -> None: - self.wheel_name = "example_minimal_package-0.0.1-py3-none-any.whl" - self.wheel_dir = tempfile.mkdtemp() - self.wheel_path = os.path.join(self.wheel_dir, self.wheel_name) - shutil.copy(os.path.join("examples", "wheel", self.wheel_name), self.wheel_dir) - - def tearDown(self): - # On windows, the wheel file remains open, so gives an error upon - # deletion for some reason. - shutil.rmtree(self.wheel_dir, ignore_errors=True) - - def test_wheel_exists(self) -> None: - wheel_installer._extract_wheel( - Path(self.wheel_path), - installation_dir=Path(self.wheel_dir), - extras={}, - ) - - want_files = [ - "site-packages", - self.wheel_name, - ] - self.assertEqual( - sorted(want_files), - sorted( - [ - str(p.relative_to(self.wheel_dir)) - for p in Path(self.wheel_dir).glob("*") - ] - ), - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/venv_site_packages_libs/BUILD.bazel b/tests/venv_site_packages_libs/BUILD.bazel index 256c4f24b5..d44bbcbb63 100644 --- a/tests/venv_site_packages_libs/BUILD.bazel +++ b/tests/venv_site_packages_libs/BUILD.bazel @@ -1,4 +1,3 @@ -load("@rules_python_internal//:rules_python_config.bzl", rp_config = "config") load("@rules_shell//shell:sh_test.bzl", "sh_test") load("//python:py_library.bzl", "py_library") load("//tests/support:py_reconfig.bzl", "py_reconfig_test") @@ -79,9 +78,6 @@ py_reconfig_test( "@platforms//os:windows": "system_python", "//conditions:default": "script", }), - env = { - "BAZEL_8_OR_LATER": "1" if rp_config.bazel_8_or_later else "0", - }, main = "whl_scripts_runnable_test.py", venvs_site_packages = "yes", deps = [ diff --git a/tests/venv_site_packages_libs/whl_scripts_runnable_test.py b/tests/venv_site_packages_libs/whl_scripts_runnable_test.py index 61b477f493..b62b5a5fce 100644 --- a/tests/venv_site_packages_libs/whl_scripts_runnable_test.py +++ b/tests/venv_site_packages_libs/whl_scripts_runnable_test.py @@ -63,12 +63,6 @@ def test_entry_point_is_runnable(self): script_executable = output[-1].strip() self.assertEqual(script_executable, sys.executable) - # This should really check for 8.5 instead of 8+, but we test with 8.6 - # so it's close enough for our purposes. - @unittest.skipUnless( - BAZEL_8_OR_LATER, - "bazel 8.5 and lower uses wheel.py, which rewrites #!pythonw to #!python", - ) def test_pythonw_script(self): script_path = self._get_script_path("whl_with_data1_pythonw") self.assertTrue(script_path.exists(), f"Script not found at {script_path}")