From 07885cbcdb0b19265379c3941600faadc8a22d71 Mon Sep 17 00:00:00 2001 From: Daniel Stone Date: Tue, 24 Mar 2020 11:11:36 +0000 Subject: [PATCH] CI: Add native Windows VS2019 build Adds a native build of Mesa using Meson with the Visual Studio 2019 toolchain on a Windows host. Though Docker is supported on Windows, Docker-in-Docker is not possible, nor are podman and skopeo available. We handle this by creating the container from a shell-executor Windows machine, which gives us a native PowerShell that we can execute Docker from. This attempts to do the same copy-from-upstream-or-create-if-not-exists optimisation as the ci-templates do for our Linux builds, albeit open-coded in PowerShell. The Mesa build itself is executed inside a container, using Meson and Ninja. Signed-off-by: Daniel Stone Reviewed-by: Eric Anholt Acked-by: Jose Fonseca Acked-by: Brian Paul Acked-by: Eric Engestrom Tested-by: Marge Bot Part-of: --- .gitlab-ci.yml | 49 ++++++++++++++++++++--- .gitlab-ci/windows/Dockerfile | 10 +++++ .gitlab-ci/windows/README.md | 32 +++++++++++++++ .gitlab-ci/windows/mesa_build.ps1 | 19 +++++++++ .gitlab-ci/windows/mesa_container.ps1 | 56 +++++++++++++++++++++++++++ .gitlab-ci/windows/mesa_deps.ps1 | 38 ++++++++++++++++++ 6 files changed, 198 insertions(+), 6 deletions(-) create mode 100644 .gitlab-ci/windows/Dockerfile create mode 100644 .gitlab-ci/windows/README.md create mode 100644 .gitlab-ci/windows/mesa_build.ps1 create mode 100644 .gitlab-ci/windows/mesa_container.ps1 create mode 100644 .gitlab-ci/windows/mesa_deps.ps1 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d1523875c29..75b2578b6c8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -185,6 +185,43 @@ arm_test: - meson-arm64 - arm_test +# Native Windows docker builds +# +# Unlike the above Linux-based builds - including MinGW/SCons builds which +# cross-compile for Windows - which use the freedesktop ci-templates, we +# cannot use the same scheme here. As Windows lacks support for +# Docker-in-Docker, and Podman does not run natively on Windows, we have +# to open-code much of the same ourselves. +# +# This is achieved by first running in a native Windows shell instance +# (host PowerShell) in the container stage to build and push the image, +# then in the build stage by executing inside Docker. + +.windows-docker-vs2019: + variables: + WINDOWS_TAG: "2020-03-24" + WINDOWS_IMAGE: "$CI_REGISTRY_IMAGE/windows/x64_build:$WINDOWS_TAG" + WINDOWS_UPSTREAM_IMAGE: "$CI_REGISTRY/$FDO_UPSTREAM_REPO/windows/x64_build:$WINDOWS_TAG" + +windows_build_vs2019: + extends: + - .container + - .windows-docker-vs2019 + stage: container + variables: + GIT_STRATEGY: fetch # we do actually need the full repository though + tags: + - windows + - shell + - "1809" + script: + - .\.gitlab-ci\windows\mesa_container.ps1 $CI_REGISTRY $CI_REGISTRY_USER $CI_REGISTRY_PASSWORD $WINDOWS_IMAGE $WINDOWS_UPSTREAM_IMAGE + +.use-windows_build_vs2019: + extends: .windows-docker-vs2019 + image: "$WINDOWS_IMAGE" + needs: + - windows_build_vs2019 # BUILD @@ -218,7 +255,9 @@ arm_test: .build-windows: extends: .build-common tags: - - mesa-windows + - windows + - docker + - "1809" cache: key: ${CI_JOB_NAME} paths: @@ -386,15 +425,13 @@ meson-clang: CC: "ccache clang-9" CXX: "ccache clang++-9" -.meson-windows: +meson-windows-vs2019: extends: - .build-windows + - .use-windows_build_vs2019 stage: meson-misc - before_script: - - $ENV:ARCH = "x86" - - $ENV:VERSION = "2019\Community" script: - - cmd /C .gitlab-ci\meson-build.bat + - . .\.gitlab-ci\windows\mesa_build.ps1 scons-win64: extends: .scons-build diff --git a/.gitlab-ci/windows/Dockerfile b/.gitlab-ci/windows/Dockerfile new file mode 100644 index 00000000000..02d9584e4a6 --- /dev/null +++ b/.gitlab-ci/windows/Dockerfile @@ -0,0 +1,10 @@ +# escape=` + +FROM mcr.microsoft.com/windows:1809 + +# Make sure any failure in PowerShell scripts is fatal +SHELL ["powershell", "-ExecutionPolicy", "RemoteSigned", "-Command", "$ErrorActionPreference = 'Stop';"] +ENV ErrorActionPreference='Stop' + +COPY mesa_deps.ps1 C:\ +RUN C:\mesa_deps.ps1 diff --git a/.gitlab-ci/windows/README.md b/.gitlab-ci/windows/README.md new file mode 100644 index 00000000000..d934a222d5d --- /dev/null +++ b/.gitlab-ci/windows/README.md @@ -0,0 +1,32 @@ +# Native Windows GitLab CI builds + +Unlike Linux, Windows cannot reuse the freedesktop ci-templates as they exist +as we do not have Podman, Skopeo, or even Docker-in-Docker builds available +under Windows. + +We still reuse the same model: build a base container with the core operating +system and infrequently-changed build dependencies, then execute Mesa builds +only inside that base container. This is open-coded in PowerShell scripts. + +## Base container build + +The base container build job executes the `mesa_container.ps1` script which +reproduces the ci-templates behaviour. It looks for the registry image in +the user's namespace, and exits if found. If not found, it tries to copy +the same image tag from the upstream Mesa repository. If that is not found, +the image is rebuilt inside the user's namespace. + +The rebuild executes `docker build` which calls `mesa_deps.ps1` inside the +container to fetch and install all build dependencies. This includes Visual +Studio Community Edition (downloaded from Microsoft, under the license which +allows use by open-source projects), other build tools from Chocolatey, and +finally Meson and Python dependencies from PyPI. + +This job is executed inside a Windows shell environment directly inside the +host, without Docker. + +## Mesa build + +The Mesa build runs inside the base container, executing `mesa_build.ps1`. +This simply compiles Mesa using Meson and Ninja, executing the build and +unit tests. Currently, no build artifacts are captured. diff --git a/.gitlab-ci/windows/mesa_build.ps1 b/.gitlab-ci/windows/mesa_build.ps1 new file mode 100644 index 00000000000..8d7b79c6497 --- /dev/null +++ b/.gitlab-ci/windows/mesa_build.ps1 @@ -0,0 +1,19 @@ +# force the CA cert cache to be rebuilt, in case Meson tries to access anything +Write-Host "Refreshing Windows TLS CA cache" +(New-Object System.Net.WebClient).DownloadString("https://github.com") >$null + +Get-Date +Write-Host "Compiling Mesa" +$builddir = New-Item -ItemType Directory -Name "build" +Push-Location $builddir.FullName +cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=amd64 && meson -Dgallium-drivers=swrast -Dbuild-tests=true .. && ninja test" +$buildstatus = $? +Pop-Location +Remove-Item -Recurse -Path $builddir + +Get-Date + +if (!$buildstatus) { + Write-Host "Mesa build or test failed" + Exit 1 +} diff --git a/.gitlab-ci/windows/mesa_container.ps1 b/.gitlab-ci/windows/mesa_container.ps1 new file mode 100644 index 00000000000..5f960036508 --- /dev/null +++ b/.gitlab-ci/windows/mesa_container.ps1 @@ -0,0 +1,56 @@ +# Implements the equivalent of ci-templates container-ifnot-exists, using +# Docker directly as we don't have buildah/podman/skopeo available under +# Windows, nor can we execute Docker-in-Docker +$registry_uri = $args[0] +$registry_username = $args[1] +$registry_password = $args[2] +$registry_user_image = $args[3] +$registry_central_image = $args[4] + +Set-Location -Path ".\.gitlab-ci\windows" + +docker login -u "$registry_username" -p "$registry_password" "$registry_uri" +if (!$?) { + Write-Host "docker login failed to $registry_uri" + Exit 1 +} + +# if the image already exists, don't rebuild it +docker pull "$registry_user_image" +if ($?) { + Write-Host "User image $registry_user_image already exists; not rebuilding" + docker logout "$registry_uri" + Exit 0 +} + +# if the image already exists upstream, copy it +docker pull "$registry_central_image" +if ($?) { + Write-Host "Copying central image $registry_central_image to user image $registry_user_image" + docker tag "$registry_user_image" "$registry_central_image" + docker push "$registry_user_image" + $pushstatus = $? + docker logout "$registry_uri" + if (!$pushstatus) { + Write-Host "Pushing image to $registry_user_image failed" + Exit 1 + } + Exit 0 +} + +Write-Host "No image found at $registry_user_image or $registry_central_image; rebuilding" +docker build --no-cache -t "$registry_user_image" . +if (!$?) { + Write-Host "Container build failed" + docker logout "$registry_uri" + Exit 1 +} +Get-Date + +docker push "$registry_user_image" +$pushstatus = $? +docker logout "$registry_uri" +if (!$pushstatus) { + Write-Host "Pushing image to $registry_user_image failed" + Exit 1 +} diff --git a/.gitlab-ci/windows/mesa_deps.ps1 b/.gitlab-ci/windows/mesa_deps.ps1 new file mode 100644 index 00000000000..d700d3e5d55 --- /dev/null +++ b/.gitlab-ci/windows/mesa_deps.ps1 @@ -0,0 +1,38 @@ +Get-Date +Write-Host "Installing Chocolatey" +Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) +Import-Module "$env:ProgramData\chocolatey\helpers\chocolateyProfile.psm1" +Update-SessionEnvironment +Write-Host "Installing Chocolatey packages" +choco install --allow-empty-checksums -y cmake --installargs "ADD_CMAKE_TO_PATH=System" +choco install --allow-empty-checksums -y python3 git git-lfs ninja pkgconfiglite winflexbison +Update-SessionEnvironment + +Start-Process -NoNewWindow -Wait git -ArgumentList 'config --global core.autocrlf false' + +# we want more secure TLS 1.2 for most things, but it breaks SourceForge +# downloads so must be done after Chocolatey use +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; + +# VS16.x is 2019 +$msvc_2019_url = 'https://aka.ms/vs/16/release/vs_buildtools.exe' + +Get-Date +Write-Host "Downloading Visual Studio 2019 build tools" +Invoke-WebRequest -Uri $msvc_2019_url -OutFile C:\vs_buildtools.exe + +Get-Date +Write-Host "Installing Visual Studio 2019" +Start-Process -NoNewWindow -Wait C:\vs_buildtools.exe -ArgumentList '--wait --quiet --norestart --nocache --installPath C:\BuildTools --add Microsoft.VisualStudio.Workload.VCTools --add Microsoft.VisualStudio.Workload.NativeDesktop --add Microsoft.VisualStudio.Component.VC.ATL --add Microsoft.VisualStudio.Component.VC.ATLMFC --add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 --add Microsoft.VisualStudio.Component.Graphics.Tools --add Microsoft.VisualStudio.Component.Windows10SDK.18362 --includeRecommended' +Remove-Item C:\vs_buildtools.exe -Force +Get-Item C:\BuildTools | Out-Host + +Get-Date +Write-Host "Installing Meson" +Start-Process -NoNewWindow -Wait pip3 -ArgumentList 'install meson' + +Write-Host "Installing Mako" +Start-Process -NoNewWindow -Wait pip3 -ArgumentList 'install mako' + +Get-Date +Write-Host "Complete" -- 2.30.2