From a16866b33b6cc08fe0b587bef1cf5bf14d2c814f Mon Sep 17 00:00:00 2001 From: Gereon Kremer Date: Thu, 3 Mar 2022 19:39:16 +0100 Subject: [PATCH] Build python wheels in our CI (#8087) This PR integrates building and publishing cvc5 with its base and pythonic python APIs as a package to PyPi into our CI. We build wheels for Linux and macOS for CPython 3.6 to 3.10 and PyPy 3.7 and 3.8. The job is run nightly and for a release, and only published to PyPi for a release (as long as there is no reasonable way to automatically prune nightly builds from either PyPi or TestPyPi). --- .../actions/install-dependencies/action.yml | 21 +++- .../package-python-wheel-macos/action.yml | 24 ++++ .../actions/package-python-wheel/action.yml | 108 ++++++++++++++++++ .github/workflows/package_pypi.yml | 52 +++++++++ contrib/packaging_python/Readme.md | 2 +- docs/conf.py.in | 2 +- 6 files changed, 203 insertions(+), 6 deletions(-) create mode 100644 .github/actions/package-python-wheel-macos/action.yml create mode 100644 .github/actions/package-python-wheel/action.yml create mode 100644 .github/workflows/package_pypi.yml diff --git a/.github/actions/install-dependencies/action.yml b/.github/actions/install-dependencies/action.yml index a204bfa21..a27758c05 100644 --- a/.github/actions/install-dependencies/action.yml +++ b/.github/actions/install-dependencies/action.yml @@ -5,14 +5,16 @@ inputs: default: false with-python-bindings: default: false + with-python-packaging: + default: false runs: using: composite steps: - name: Install Linux software + if: runner.os == 'Linux' shell: bash run: | echo "::group::Install Linux software" - if [[ $RUNNER_OS != "Linux" ]]; then exit 0; fi sudo apt-get update sudo apt-get install -y \ build-essential \ @@ -44,10 +46,10 @@ runs: # Note: macOS comes with a libedit; it does not need to brew-installed - name: Install macOS software + if: runner.os == 'macOS' shell: bash run: | echo "::group::Install macOS software" - if [[ $RUNNER_OS != "macOS" ]]; then exit 0; fi brew update --quiet brew install \ ccache \ @@ -69,21 +71,32 @@ runs: echo "::endgroup::" - name: Install software for Python bindings + if: inputs.with-python-bindings == 'true' shell: bash run: | echo "::group::Install software for Python bindings" - if [[ "${{ inputs.with-python-bindings }}" != "true" ]]; then exit 0; fi + python3 -m pip install -q --upgrade pip python3 -m pip install pytest scikit-build python3 -m pytest --version python3 -m pip install Cython==0.29.* echo "$(python3 -m site --user-base)/bin" >> $GITHUB_PATH echo "::endgroup::" + + - name: Install software for Python packaging + if: inputs.with-python-packaging == 'true' + shell: bash + run: | + echo "::group::Install software for Python packaging" + python3 -m pip install -q --upgrade pip + python3 -m pip install twine + python3 -m pip install -U urllib3 requests + echo "::endgroup::" - name: Install software for documentation + if: inputs.with-documentation == 'true' shell: bash run: | echo "::group::Install software for documentation" - if [[ "${{ inputs.with-documentation }}" != "true" ]]; then exit 0; fi sudo apt-get install -y doxygen python3-docutils python3-jinja2 python3 -m pip install \ sphinxcontrib-bibtex sphinx-tabs sphinx-rtd-theme breathe \ diff --git a/.github/actions/package-python-wheel-macos/action.yml b/.github/actions/package-python-wheel-macos/action.yml new file mode 100644 index 000000000..568591572 --- /dev/null +++ b/.github/actions/package-python-wheel-macos/action.yml @@ -0,0 +1,24 @@ +name: Package python wheel for macOS +description: Package cvc5 into a python wheel on macOS for one python version +inputs: + python-version: + default: "" +runs: + using: composite + steps: + - uses: actions/setup-python@v2 + if: runner.os == 'macOS' + with: + python-version: ${{ inputs.python-version }} + + - name: Build wheel + shell: bash + if: runner.os == 'macOS' + env: + MACOSX_DEPLOYMENT_TARGET: 10.13 + run: | + echo "::group::Build macOS wheel for ${{ inputs.python-version }}" + ./contrib/packaging_python/mk_clean_wheel.sh python "production --auto-download" + + ls *.whl + echo "::endgroup::" diff --git a/.github/actions/package-python-wheel/action.yml b/.github/actions/package-python-wheel/action.yml new file mode 100644 index 000000000..87c6fdbca --- /dev/null +++ b/.github/actions/package-python-wheel/action.yml @@ -0,0 +1,108 @@ +name: Package python wheels +description: Package cvc5 into python wheels for all supported python versions +inputs: + upload-to-pypi: + default: false + upload-to-test-pypi: + default: false + pypi-token: + default: "" + test-pypi-token: + default: "" +runs: + using: composite + steps: + - name: Build wheels for Linux + if: runner.os == 'Linux' + shell: bash + run: | + echo "::group::Create docker image" + if ! docker image inspect pycvc5-manylinux2014 > /dev/null 2>&1; then + echo "Need to build docker image" + docker build -q -t pycvc5-manylinux2014 contrib/packaging_python/manylinux2014 + fi + echo "::endgroup::" + + OPTS="production --auto-download" + for version in cp36 cp37 cp38 cp39 cp310 pp37 pp38 + do + echo "::group::Build extension for python $version" + docker run --rm \ + -v `pwd`:/home/pycvc5 \ + pycvc5-manylinux2014 \ + ./contrib/packaging_python/mk_clean_wheel.sh /opt/python/${version}*/bin/python "$OPTS" + echo "::endgroup::" + done + + ls *.whl + + - name: Build wheels for macOS + if: runner.os == 'macOS' + uses: ./.github/actions/package-python-wheel-macos + with: + python-version: '3.6' + + - name: Build wheels for macOS + if: runner.os == 'macOS' + uses: ./.github/actions/package-python-wheel-macos + with: + python-version: '3.7' + + - name: Build wheels for macOS + if: runner.os == 'macOS' + uses: ./.github/actions/package-python-wheel-macos + with: + python-version: '3.8' + + - name: Build wheels for macOS + if: runner.os == 'macOS' + uses: ./.github/actions/package-python-wheel-macos + with: + python-version: '3.9' + + - name: Build wheels for macOS + if: runner.os == 'macOS' + uses: ./.github/actions/package-python-wheel-macos + with: + python-version: '3.10' + + - name: Build wheels for macOS + if: runner.os == 'macOS' + uses: ./.github/actions/package-python-wheel-macos + with: + python-version: 'pypy-3.7' + + - name: Build wheels for macOS + if: runner.os == 'macOS' + uses: ./.github/actions/package-python-wheel-macos + with: + python-version: 'pypy-3.8' + + - name: Upload wheels to pypi.org + if: inputs.upload-to-pypi == 'true' + shell: bash + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ inputs.pypi-token }} + run: | + echo "::group::Upload to pypi.org" + for wheel in `ls *.whl` + do + twine upload $wheel + done + echo "::endgroup::" + + - name: Upload wheels to test.pypi.org + if: inputs.upload-to-test-pypi == 'true' + shell: bash + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ inputs.test-pypi-token }} + run: | + echo "::group::Upload to test.pypi.org" + for wheel in `ls *.whl` + do + twine upload --repository testpypi $wheel + done + echo "::endgroup::" + diff --git a/.github/workflows/package_pypi.yml b/.github/workflows/package_pypi.yml new file mode 100644 index 000000000..1c561a8d5 --- /dev/null +++ b/.github/workflows/package_pypi.yml @@ -0,0 +1,52 @@ +on: + push: + pull_request: + release: + types: [published] + schedule: + - cron: '0 1 * * *' + +name: PyPi packaging + +jobs: + build: + strategy: + matrix: + os: [ ubuntu-latest, macos-latest ] + + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Install dependencies + uses: ./.github/actions/install-dependencies + if: runner.os == 'Linux' + with: + with-documentation: false + with-python-bindings: false + with-python-packaging: true + + - name: Install dependencies + uses: ./.github/actions/install-dependencies + if: runner.os == 'macOS' + with: + with-documentation: false + with-python-bindings: true + with-python-packaging: true + + - name: Setup caches + uses: ./.github/actions/setup-cache + with: + cache-key: cvc5-pypi + + - name: Package PyPi wheel packages + uses: ./.github/actions/package-python-wheel + with: + upload-to-pypi: ${{ github.event_name == 'release' }} + upload-to-test-pypi: false + test-pypi-token: ${{ secrets.PYPI_TOKEN }} + pypi-token: ${{ secrets.PYPI_TOKEN }} + diff --git a/contrib/packaging_python/Readme.md b/contrib/packaging_python/Readme.md index d75a201fe..1a049524a 100644 --- a/contrib/packaging_python/Readme.md +++ b/contrib/packaging_python/Readme.md @@ -40,7 +40,7 @@ The `mk_clean_wheel.sh`: To upload a wheel to test PyPi, - twine upload --repository testpypi -u $USERNAME -p $PASSWORD + twine upload --repository testpypi -u $USERNAME -p $PASSWORD PATH_TO_WHEEL Note that you will need a TestPyPi login. Once it has been uploaded, you can test (from anywhere, not just the container) that the wheel works by installing diff --git a/docs/conf.py.in b/docs/conf.py.in index 025e710d3..c9a79909a 100644 --- a/docs/conf.py.in +++ b/docs/conf.py.in @@ -121,7 +121,7 @@ examples_file_patterns = { 'urlname': 'examples{}', }, '(.*)': { - 'local': '/' + os.path.relpath('${CMAKE_BINARY_DIR}/deps/src/z3pycompat-EP', '${CMAKE_CURRENT_SOURCE_DIR}') + '{}', + 'local': '/' + os.path.relpath('${CMAKE_BINARY_DIR}/deps/src/CVC5PythonicAPI', '${CMAKE_CURRENT_SOURCE_DIR}') + '{}', 'url': 'https://github.com/cvc5/cvc5_z3py_compat/tree/main{}', 'urlname': 'cvc5_z3py_compat:{}', } -- 2.30.2