Build python wheels in our CI (#8087)

This PR integrates building and publishing cvc5 with its base and pythonic python APIs as a package to PyPi into our CI.
We build wheels for Linux and macOS for CPython 3.6 to 3.10 and PyPy 3.7 and 3.8.
The job is run nightly and for a release, and only published to PyPi for a release (as long as there is no reasonable way to automatically prune nightly builds from either PyPi or TestPyPi).

diff --git a/.github/actions/install-dependencies/action.yml b/.github/actions/install-dependencies/action.yml
index a204bfa2143414858d2049a3acee3616bedb4a71..a27758c055a211518b9736b2f14ba7f8d61f9c80 100644 (file)
--- a/.github/actions/install-dependencies/action.yml
+++ b/.github/actions/install-dependencies/action.yml
@@ -5,14 +5,16 @@ inputs:
     default: false
   with-python-bindings:
     default: false
+  with-python-packaging:
+    default: false
 runs:
   using: composite
   steps:
     - name: Install Linux software
+      if: runner.os == 'Linux'
       shell: bash
       run: |
         echo "::group::Install Linux software"
-        if [[ $RUNNER_OS != "Linux" ]]; then exit 0; fi
         sudo apt-get update
         sudo apt-get install -y \
           build-essential \
@@ -44,10 +46,10 @@ runs:
 
     # Note: macOS comes with a libedit; it does not need to brew-installed
     - name: Install macOS software
+      if: runner.os == 'macOS'
       shell: bash
       run: |
         echo "::group::Install macOS software"
-        if [[ $RUNNER_OS != "macOS" ]]; then exit 0; fi
         brew update --quiet
         brew install \
           ccache \
@@ -69,21 +71,32 @@ runs:
         echo "::endgroup::"
 
     - name: Install software for Python bindings
+      if: inputs.with-python-bindings == 'true'
       shell: bash
       run: |
         echo "::group::Install software for Python bindings"
-        if [[ "${{ inputs.with-python-bindings }}" != "true" ]]; then exit 0; fi
+        python3 -m pip install -q --upgrade pip
         python3 -m pip install pytest scikit-build
         python3 -m pytest --version
         python3 -m pip install Cython==0.29.*
         echo "$(python3 -m site --user-base)/bin" >> $GITHUB_PATH
         echo "::endgroup::"
+
+    - name: Install software for Python packaging
+      if: inputs.with-python-packaging == 'true'
+      shell: bash
+      run: |
+        echo "::group::Install software for Python packaging"
+        python3 -m pip install -q --upgrade pip
+        python3 -m pip install twine
+        python3 -m pip install -U urllib3 requests
+        echo "::endgroup::"
     
     - name: Install software for documentation
+      if: inputs.with-documentation == 'true'
       shell: bash
       run: |
         echo "::group::Install software for documentation"
-        if [[ "${{ inputs.with-documentation }}" != "true" ]]; then exit 0; fi
         sudo apt-get install -y doxygen python3-docutils python3-jinja2
         python3 -m pip install \
           sphinxcontrib-bibtex sphinx-tabs sphinx-rtd-theme breathe \

diff --git a/.github/actions/package-python-wheel-macos/action.yml b/.github/actions/package-python-wheel-macos/action.yml
new file mode 100644 (file)
index 0000000..5685915
--- /dev/null
+++ b/.github/actions/package-python-wheel-macos/action.yml
@@ -0,0 +1,24 @@
+name: Package python wheel for macOS
+description: Package cvc5 into a python wheel on macOS for one python version
+inputs:
+  python-version:
+    default: ""
+runs:
+  using: composite
+  steps:
+    - uses: actions/setup-python@v2
+      if: runner.os == 'macOS'
+      with:
+        python-version: ${{ inputs.python-version }}
+    
+    - name: Build wheel
+      shell: bash
+      if: runner.os == 'macOS'
+      env:
+        MACOSX_DEPLOYMENT_TARGET: 10.13
+      run: |
+        echo "::group::Build macOS wheel for ${{ inputs.python-version }}"
+        ./contrib/packaging_python/mk_clean_wheel.sh python "production --auto-download"
+
+        ls *.whl
+        echo "::endgroup::"

diff --git a/.github/actions/package-python-wheel/action.yml b/.github/actions/package-python-wheel/action.yml
new file mode 100644 (file)
index 0000000..87c6fdb
--- /dev/null
+++ b/.github/actions/package-python-wheel/action.yml
@@ -0,0 +1,108 @@
+name: Package python wheels
+description: Package cvc5 into python wheels for all supported python versions
+inputs:
+  upload-to-pypi:
+    default: false
+  upload-to-test-pypi:
+    default: false
+  pypi-token:
+    default: ""
+  test-pypi-token:
+    default: ""
+runs:
+  using: composite
+  steps:
+    - name: Build wheels for Linux
+      if: runner.os == 'Linux'
+      shell: bash
+      run: |
+        echo "::group::Create docker image"
+        if ! docker image inspect pycvc5-manylinux2014 > /dev/null 2>&1; then
+          echo "Need to build docker image"
+          docker build -q -t pycvc5-manylinux2014 contrib/packaging_python/manylinux2014
+        fi
+        echo "::endgroup::"
+
+        OPTS="production --auto-download"
+        for version in cp36 cp37 cp38 cp39 cp310 pp37 pp38
+        do
+          echo "::group::Build extension for python $version"
+          docker run --rm \
+            -v `pwd`:/home/pycvc5 \
+            pycvc5-manylinux2014 \
+            ./contrib/packaging_python/mk_clean_wheel.sh /opt/python/${version}*/bin/python "$OPTS"
+          echo "::endgroup::"
+        done
+
+        ls *.whl
+      
+    - name: Build wheels for macOS
+      if: runner.os == 'macOS'
+      uses: ./.github/actions/package-python-wheel-macos
+      with:
+        python-version: '3.6'
+
+    - name: Build wheels for macOS
+      if: runner.os == 'macOS'
+      uses: ./.github/actions/package-python-wheel-macos
+      with:
+        python-version: '3.7'
+
+    - name: Build wheels for macOS
+      if: runner.os == 'macOS'
+      uses: ./.github/actions/package-python-wheel-macos
+      with:
+        python-version: '3.8'
+
+    - name: Build wheels for macOS
+      if: runner.os == 'macOS'
+      uses: ./.github/actions/package-python-wheel-macos
+      with:
+        python-version: '3.9'
+
+    - name: Build wheels for macOS
+      if: runner.os == 'macOS'
+      uses: ./.github/actions/package-python-wheel-macos
+      with:
+        python-version: '3.10'
+
+    - name: Build wheels for macOS
+      if: runner.os == 'macOS'
+      uses: ./.github/actions/package-python-wheel-macos
+      with:
+        python-version: 'pypy-3.7'
+        
+    - name: Build wheels for macOS
+      if: runner.os == 'macOS'
+      uses: ./.github/actions/package-python-wheel-macos
+      with:
+        python-version: 'pypy-3.8'
+
+    - name: Upload wheels to pypi.org
+      if: inputs.upload-to-pypi == 'true'
+      shell: bash
+      env:
+        TWINE_USERNAME: __token__
+        TWINE_PASSWORD: ${{ inputs.pypi-token }}
+      run: |
+        echo "::group::Upload to pypi.org"
+        for wheel in `ls *.whl`
+        do
+          twine upload $wheel
+        done
+        echo "::endgroup::"
+
+    - name: Upload wheels to test.pypi.org
+      if: inputs.upload-to-test-pypi == 'true'
+      shell: bash
+      env:
+        TWINE_USERNAME: __token__
+        TWINE_PASSWORD: ${{ inputs.test-pypi-token }}
+      run: |
+        echo "::group::Upload to test.pypi.org"
+        for wheel in `ls *.whl`
+        do
+          twine upload --repository testpypi $wheel
+        done
+        echo "::endgroup::"
+

diff --git a/.github/workflows/package_pypi.yml b/.github/workflows/package_pypi.yml
new file mode 100644 (file)
index 0000000..1c561a8
--- /dev/null
+++ b/.github/workflows/package_pypi.yml
@@ -0,0 +1,52 @@
+on:
+  push:
+  pull_request:
+  release:
+    types: [published]
+  schedule:
+    - cron: '0 1 * * *'
+
+name: PyPi packaging
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, macos-latest ]
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Install dependencies
+      uses: ./.github/actions/install-dependencies
+      if: runner.os == 'Linux'
+      with:
+        with-documentation: false
+        with-python-bindings: false
+        with-python-packaging: true
+
+    - name: Install dependencies
+      uses: ./.github/actions/install-dependencies
+      if: runner.os == 'macOS'
+      with:
+        with-documentation: false
+        with-python-bindings: true
+        with-python-packaging: true
+
+    - name: Setup caches
+      uses: ./.github/actions/setup-cache
+      with:
+        cache-key: cvc5-pypi
+
+    - name: Package PyPi wheel packages
+      uses: ./.github/actions/package-python-wheel
+      with:
+        upload-to-pypi: ${{ github.event_name == 'release' }}
+        upload-to-test-pypi: false
+        test-pypi-token: ${{ secrets.PYPI_TOKEN }}
+        pypi-token: ${{ secrets.PYPI_TOKEN }}
+

diff --git a/contrib/packaging_python/Readme.md b/contrib/packaging_python/Readme.md
index d75a201fea34ca69e71e5a40905cd225d456838d..1a049524ac6106a847d00863ddac7c8a4bf3e2d0 100644 (file)
--- a/contrib/packaging_python/Readme.md
+++ b/contrib/packaging_python/Readme.md
@@ -40,7 +40,7 @@ The `mk_clean_wheel.sh`:
 
 To upload a wheel to test PyPi,
 
-    twine upload --repository testpypi -u $USERNAME -p $PASSWORD <path to wheel>
+    twine upload --repository testpypi -u $USERNAME -p $PASSWORD PATH_TO_WHEEL
 
 Note that you will need a TestPyPi login. Once it has been uploaded, you can
 test (from anywhere, not just the container) that the wheel works by installing

diff --git a/docs/conf.py.in b/docs/conf.py.in
index 025e710d3a19e777cbf649bee099075fd8f51ca6..c9a79909aa93d7be115ee684b90670b8ccc42600 100644 (file)
--- a/docs/conf.py.in
+++ b/docs/conf.py.in
@@ -121,7 +121,7 @@ examples_file_patterns = {
                 'urlname': 'examples{}',
         },
         '<z3pycompat>(.*)': {
-                'local': '/' + os.path.relpath('${CMAKE_BINARY_DIR}/deps/src/z3pycompat-EP', '${CMAKE_CURRENT_SOURCE_DIR}') + '{}',
+                'local': '/' + os.path.relpath('${CMAKE_BINARY_DIR}/deps/src/CVC5PythonicAPI', '${CMAKE_CURRENT_SOURCE_DIR}') + '{}',
                 'url': 'https://github.com/cvc5/cvc5_z3py_compat/tree/main{}',
                 'urlname': 'cvc5_z3py_compat:{}',
         }