From 40b63f5305008a10b5baf0d3602fe40baa61c425 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Sun, 31 Jan 2021 00:07:37 +0100 Subject: [PATCH 1/8] Test for the creation of an already existing output wheel, and print a helpful warning about building for Python's limited API --- cibuildwheel/linux.py | 12 ++++++++++++ cibuildwheel/macos.py | 10 ++++++++++ cibuildwheel/windows.py | 10 ++++++++++ 3 files changed, 32 insertions(+) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 5d19dbd20..60d5e87b7 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -218,6 +218,18 @@ def build(options: BuildOptions) -> None: # clean up test environment docker.call(['rm', '-rf', venv_dir]) + existing_output_files = docker.call(['ls'], cwd=container_output_dir).split('\n') + for repaired_wheel in repaired_wheels: + if repaired_wheel.name in existing_output_files: + message = f'Created wheel ({repaired_wheel}) already exists in output directory.' + if 'abi3' in repaired_wheel.name: + message += ('\n' + "It looks like you are building wheels against Python's limited API/stable ABI;\n" + 'please limit your Python selection to a single version when building limited API\n' + 'wheels, with CIBW_BUILD.') + log.error(message) + sys.exit(1) + # move repaired wheels to output docker.call(['mkdir', '-p', container_output_dir]) docker.call(['mv', *repaired_wheels, container_output_dir]) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index e6395449f..60b9bceb0 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -456,6 +456,16 @@ def call_with_arch(args: Sequence[PathOrStr], **kwargs: Any) -> int: # clean up shutil.rmtree(venv_dir) + if (options.output_dir / repaired_wheel.name).exists(): + message = f'Created wheel ({repaired_wheel}) already exists in output directory.' + if 'abi3' in repaired_wheel.name: + message += ('\n' + "It looks like you are building wheels against Python's limited API/stable ABI;\n" + 'please limit your Python selection to a single version when building limited API\n' + 'wheels, with CIBW_BUILD.') + log.error(message) + sys.exit(1) + # we're all done here; move it to output (overwrite existing) shutil.move(str(repaired_wheel), options.output_dir) log.build_end() diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 45f0141ab..afd759bad 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -325,6 +325,16 @@ def build(options: BuildOptions) -> None: # clean up shutil.rmtree(venv_dir) + if (options.output_dir / repaired_wheel.name).exists(): + message = f'Created wheel ({repaired_wheel}) already exists in output directory.' + if 'abi3' in repaired_wheel.name: + message += ('\n' + "It looks like you are building wheels against Python's limited API/stable ABI;\n" + 'please limit your Python selection to a single version when building limited API\n' + 'wheels, with CIBW_BUILD.') + log.error(message) + sys.exit(1) + # we're all done here; move it to output (remove if already exists) shutil.move(str(repaired_wheel), options.output_dir) log.build_end() From c74989daafbd9a1c55f2fd88de66210d923b1105 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Sun, 31 Jan 2021 00:26:31 +0100 Subject: [PATCH 2/8] Add successful test building with Py_LIMITED_API --- test/test_limited_api.py | 19 +++++++++++++++++++ test/utils.py | 8 ++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 test/test_limited_api.py diff --git a/test/test_limited_api.py b/test/test_limited_api.py new file mode 100644 index 000000000..c7846ea28 --- /dev/null +++ b/test/test_limited_api.py @@ -0,0 +1,19 @@ +from . import test_projects, utils + +basic_project = test_projects.new_c_project( + setup_py_setup_args_add='py_limited_api=True', +) + + +def test_setup_py(tmp_path): + project_dir = tmp_path / 'project' + basic_project.generate(project_dir) + + # build the wheels + actual_wheels = utils.cibuildwheel_run(project_dir, add_env={ + 'CIBW_BUILD': 'cp27-* cp35-*', + }) + + # check that the expected wheels are produced + expected_wheels = utils.expected_wheels('spam', '0.1.0', limited_api=True) + assert set(actual_wheels) == set(expected_wheels) diff --git a/test/utils.py b/test/utils.py index 8dcb2f604..cd721e0c0 100644 --- a/test/utils.py +++ b/test/utils.py @@ -88,7 +88,8 @@ def _get_arm64_macosx_deployment_target(macosx_deployment_target: str) -> str: def expected_wheels(package_name, package_version, manylinux_versions=None, macosx_deployment_target='10.9', machine_arch=None, *, - exclude_27=IS_WINDOWS_RUNNING_ON_TRAVIS): + exclude_27=IS_WINDOWS_RUNNING_ON_TRAVIS, + limited_api=False): ''' Returns a list of expected wheels from a run of cibuildwheel. ''' @@ -106,7 +107,10 @@ def expected_wheels(package_name, package_version, manylinux_versions=None, else: manylinux_versions = ['manylinux2014'] - python_abi_tags = ['cp35-cp35m', 'cp36-cp36m', 'cp37-cp37m', 'cp38-cp38', 'cp39-cp39'] + if limited_api: + python_abi_tags = ['cp35-abi3'] + else: + python_abi_tags = ['cp35-cp35m', 'cp36-cp36m', 'cp37-cp37m', 'cp38-cp38', 'cp39-cp39'] if machine_arch in ['x86_64', 'AMD64', 'x86']: python_abi_tags += ['cp27-cp27m', 'pp27-pypy_73', 'pp36-pypy36_pp73', 'pp37-pypy37_pp73'] From 419270eac52abc5e2f2603b560f78e17aacf8af5 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Sun, 31 Jan 2021 01:40:10 +0100 Subject: [PATCH 3/8] Add py_limited_api to setup.cfg in tests --- test/test_limited_api.py | 13 +++++++++---- test/utils.py | 4 ++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/test/test_limited_api.py b/test/test_limited_api.py index c7846ea28..b2c562175 100644 --- a/test/test_limited_api.py +++ b/test/test_limited_api.py @@ -1,19 +1,24 @@ +import textwrap + from . import test_projects, utils basic_project = test_projects.new_c_project( - setup_py_setup_args_add='py_limited_api=True', + setup_cfg_add=textwrap.dedent(r''' + [bdist_wheel] + py_limited_api=cp36 + ''') ) -def test_setup_py(tmp_path): +def test_setup_cfg(tmp_path): project_dir = tmp_path / 'project' basic_project.generate(project_dir) # build the wheels actual_wheels = utils.cibuildwheel_run(project_dir, add_env={ - 'CIBW_BUILD': 'cp27-* cp35-*', + 'CIBW_BUILD': 'cp27-* cp36-*', }) # check that the expected wheels are produced - expected_wheels = utils.expected_wheels('spam', '0.1.0', limited_api=True) + expected_wheels = utils.expected_wheels('spam', '0.1.0', limited_api='cp36') assert set(actual_wheels) == set(expected_wheels) diff --git a/test/utils.py b/test/utils.py index cd721e0c0..ef96d5069 100644 --- a/test/utils.py +++ b/test/utils.py @@ -89,7 +89,7 @@ def _get_arm64_macosx_deployment_target(macosx_deployment_target: str) -> str: def expected_wheels(package_name, package_version, manylinux_versions=None, macosx_deployment_target='10.9', machine_arch=None, *, exclude_27=IS_WINDOWS_RUNNING_ON_TRAVIS, - limited_api=False): + limited_api=None): ''' Returns a list of expected wheels from a run of cibuildwheel. ''' @@ -108,7 +108,7 @@ def expected_wheels(package_name, package_version, manylinux_versions=None, manylinux_versions = ['manylinux2014'] if limited_api: - python_abi_tags = ['cp35-abi3'] + python_abi_tags = [f'{limited_api}-abi3'] else: python_abi_tags = ['cp35-cp35m', 'cp36-cp36m', 'cp37-cp37m', 'cp38-cp38', 'cp39-cp39'] From 0e4580a6b0fe0cf5719eda9cc05f95c8f2a7b5dc Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Sun, 31 Jan 2021 02:02:42 +0100 Subject: [PATCH 4/8] How does PyPy react to limited API? --- test/test_limited_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_limited_api.py b/test/test_limited_api.py index b2c562175..d18aee1f2 100644 --- a/test/test_limited_api.py +++ b/test/test_limited_api.py @@ -16,7 +16,7 @@ def test_setup_cfg(tmp_path): # build the wheels actual_wheels = utils.cibuildwheel_run(project_dir, add_env={ - 'CIBW_BUILD': 'cp27-* cp36-*', + 'CIBW_BUILD': 'cp27-* cp36-* pp27-* pp36-*', }) # check that the expected wheels are produced From 8b0db9dd78e2bd2236bee5867792a116a754396c Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Sun, 31 Jan 2021 17:48:16 +0100 Subject: [PATCH 5/8] Don't test PyPy in test_limited_api, and add a test using PIP_BUILD_OPTION --- test/test_limited_api.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/test/test_limited_api.py b/test/test_limited_api.py index d18aee1f2..a6dff32b5 100644 --- a/test/test_limited_api.py +++ b/test/test_limited_api.py @@ -2,7 +2,7 @@ from . import test_projects, utils -basic_project = test_projects.new_c_project( +limited_api_project = test_projects.new_c_project( setup_cfg_add=textwrap.dedent(r''' [bdist_wheel] py_limited_api=cp36 @@ -12,13 +12,30 @@ def test_setup_cfg(tmp_path): project_dir = tmp_path / 'project' - basic_project.generate(project_dir) + limited_api_project.generate(project_dir) # build the wheels actual_wheels = utils.cibuildwheel_run(project_dir, add_env={ - 'CIBW_BUILD': 'cp27-* cp36-* pp27-* pp36-*', + 'CIBW_BUILD': 'cp27-* cp36-*', # PyPy does not have a Py_LIMITED_API equivalent }) # check that the expected wheels are produced - expected_wheels = utils.expected_wheels('spam', '0.1.0', limited_api='cp36') + expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0', limited_api='cp36') + if '-pp' not in w] + assert set(actual_wheels) == set(expected_wheels) + + +def test_build_option_env(tmp_path): + project_dir = tmp_path / 'project' + test_projects.new_c_project().generate(project_dir) + + # build the wheels + actual_wheels = utils.cibuildwheel_run(project_dir, add_env={ + 'CIBW_ENVIRONMENT': 'PIP_BUILD_OPTION="--py-limited-api=cp36"', + 'CIBW_BUILD': 'cp27-* cp36-*', # PyPy does not have a Py_LIMITED_API equivalent + }) + + # check that the expected wheels are produced + expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0', limited_api='cp36') + if '-pp' not in w] assert set(actual_wheels) == set(expected_wheels) From 2920e75af649b3b389fed977abfadd91c2082860 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Sun, 31 Jan 2021 22:37:33 +0100 Subject: [PATCH 6/8] Add check on duplicate wheel error --- test/test_limited_api.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/test/test_limited_api.py b/test/test_limited_api.py index a6dff32b5..4548314be 100644 --- a/test/test_limited_api.py +++ b/test/test_limited_api.py @@ -1,5 +1,8 @@ +import subprocess import textwrap +import pytest + from . import test_projects, utils limited_api_project = test_projects.new_c_project( @@ -25,7 +28,7 @@ def test_setup_cfg(tmp_path): assert set(actual_wheels) == set(expected_wheels) -def test_build_option_env(tmp_path): +def test_build_option_env(tmp_path, capfd): project_dir = tmp_path / 'project' test_projects.new_c_project().generate(project_dir) @@ -39,3 +42,17 @@ def test_build_option_env(tmp_path): expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0', limited_api='cp36') if '-pp' not in w] assert set(actual_wheels) == set(expected_wheels) + + +def test_duplicate_wheel_error(tmp_path, capfd): + project_dir = tmp_path / 'project' + limited_api_project.generate(project_dir) + + with pytest.raises(subprocess.CalledProcessError): + utils.cibuildwheel_run(project_dir, add_env={ + 'CIBW_BUILD': 'cp36-* cp37-*', + }) + + captured = capfd.readouterr() + assert "already exists in output directory" in captured.err + assert "It looks like you are building wheels against Python's limited API" in captured.err From 77d94cc2496d1b879673fffd628cd9d2f584e7b5 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Sun, 31 Jan 2021 23:18:39 +0100 Subject: [PATCH 7/8] Capturing the output of a command helps, if you want to do something based on that output --- cibuildwheel/linux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 60d5e87b7..2573f5af8 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -218,7 +218,7 @@ def build(options: BuildOptions) -> None: # clean up test environment docker.call(['rm', '-rf', venv_dir]) - existing_output_files = docker.call(['ls'], cwd=container_output_dir).split('\n') + existing_output_files = docker.call(['ls'], capture_output=True, cwd=container_output_dir).split('\n') for repaired_wheel in repaired_wheels: if repaired_wheel.name in existing_output_files: message = f'Created wheel ({repaired_wheel}) already exists in output directory.' From cc0991a76ec24afdf401a727d9744b4b5ff92b2e Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Tue, 2 Feb 2021 15:46:30 +0100 Subject: [PATCH 8/8] Debug pytest --- test/test_limited_api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/test_limited_api.py b/test/test_limited_api.py index 4548314be..0e9626d23 100644 --- a/test/test_limited_api.py +++ b/test/test_limited_api.py @@ -54,5 +54,7 @@ def test_duplicate_wheel_error(tmp_path, capfd): }) captured = capfd.readouterr() + print('out', captured.out) + print('err', captured.err) assert "already exists in output directory" in captured.err assert "It looks like you are building wheels against Python's limited API" in captured.err