From 184d47a7adb8f58a036b1dcaa72d09219a65feb2 Mon Sep 17 00:00:00 2001 From: Thomas Petazzoni Date: Fri, 14 Aug 2020 21:52:31 +0200 Subject: [PATCH] support/scripts/gen-bootlin-toolchains: add new script to support Bootlin toolchains https://toolchains.bootlin.com/ has been providing for a few years a number of ready-to-use pre-built toolchains, for a wide range of architectures (which it turns out, are all built using Buildroot). While toolchains.bootlin.com provides Buildroot config fragments to easily use those toolchains with Buildroot (see [0] for example), this is not visible anywhere. So instead, we would like to add support for these toolchains in Buildroot just like we have existing support for Linaro, ARM, Synopsys, etc. toolchains. [0] https://toolchains.bootlin.com/downloads/releases/toolchains/aarch64/fragments/aarch64--glibc--bleeding-edge-2020.02-2.frag However, the number of toolchains provided by toolchains.bootlin.com is really large, and they are regularly updated. Maintaining that manually would be time consuming and error-prone. So instead, this commit introduces a script that automatically generates: - toolchain/toolchain-external/toolchain-external-bootlin/Config.in.options - toolchain/toolchain-external/toolchain-external-bootlin/toolchain-external-bootlin.mk - toolchain/toolchain-external/toolchain-external-bootlin/toolchain-external-bootlin.hash - support/testing/tests/toolchain/test_external_bootlin.py We create a single external toolchain package, with a Kconfig "choice" as a sub-option to select the toolchain variant to be used. The script contains a Python dict that provides the mapping between the toolchains provided by toolchains.bootlin.com, and the architecture options/variants they are applicable to. The test cases allow to verify that the toolchain configuration is correct, and that it is able to build a Busybox based system. It doesn't do any runtime testing as such testing is already done by toolchains.bootlin.com: the test cases here are only meant to verify that the toolchain-external-bootlin package works as expected. Signed-off-by: Thomas Petazzoni Reviewed-by: Titouan Christophe Tested-by: Titouan Christophe Signed-off-by: Yann E. MORIN --- support/scripts/gen-bootlin-toolchains | 473 +++++++++++++++++++++++++ 1 file changed, 473 insertions(+) create mode 100755 support/scripts/gen-bootlin-toolchains diff --git a/support/scripts/gen-bootlin-toolchains b/support/scripts/gen-bootlin-toolchains new file mode 100755 index 0000000000..a5e4b1a99b --- /dev/null +++ b/support/scripts/gen-bootlin-toolchains @@ -0,0 +1,473 @@ +#!/usr/bin/env python3 + +import os.path +import re +import requests +import textwrap + +BASE_URL = "https://toolchains.bootlin.com/downloads/releases/toolchains" + +AUTOGENERATED_COMMENT = """# This file was auto-generated by support/scripts/gen-bootlin-toolchains +# Do not edit +""" + +# In the below dict: + +# - 'conditions' indicate the cumulative conditions under which the +# toolchain will be made available. In several situations, a given +# toolchain is usable on several architectures variants (for +# example, an ARMv6 toolchain can be used on ARMv7) +# - 'test_options' indicate one specific configuration where the +# toolchain can be used. It is used to create the runtime test +# cases. If 'test_options' does not exist, the code assumes it can +# be made equal to 'conditions' +# - 'prefix' is the prefix of the cross-compilation toolchain tools + +arches = { + 'aarch64': { + 'conditions': ['BR2_aarch64'], + 'prefix': 'aarch64', + }, + 'aarch64be': { + 'conditions': ['BR2_aarch64_be'], + 'prefix': 'aarch64_be', + }, + 'arcle-750d': { + 'conditions': ['BR2_arcle', 'BR2_arc750d'], + 'prefix': 'arc', + }, + 'arcle-hs38': { + 'conditions': ['BR2_arcle', 'BR2_archs38'], + 'prefix': 'arc', + }, + 'armv5-eabi': { + 'conditions': ['BR2_ARM_CPU_ARMV5', 'BR2_ARM_EABI'], + 'test_options': ['BR2_arm', 'BR2_arm926t', 'BR2_ARM_EABI'], + 'prefix': 'arm', + }, + 'armv6-eabihf': { + 'conditions': ['BR2_ARM_CPU_ARMV6', 'BR2_ARM_EABIHF'], + 'test_options': ['BR2_arm', 'BR2_arm1176jzf_s', 'BR2_ARM_EABIHF'], + 'prefix': 'arm', + }, + 'armv7-eabihf': { + 'conditions': ['BR2_ARM_CPU_ARMV7A', 'BR2_ARM_EABIHF'], + 'test_options': ['BR2_arm', 'BR2_cortex_a8', 'BR2_ARM_EABIHF'], + 'prefix': 'arm', + }, + 'armv7m': { + 'conditions': ['BR2_ARM_CPU_ARMV7M'], + 'test_options': ['BR2_arm', 'BR2_cortex_m4'], + 'prefix': 'arm', + }, + 'm68k-68xxx': { + 'conditions': ['BR2_m68k_m68k'], + 'test_options': ['BR2_m68k', 'BR2_m68k_68040'], + 'prefix': 'm68k', + }, + 'm68k-coldfire': { + 'conditions': ['BR2_m68k_cf'], + 'test_options': ['BR2_m68k', 'BR2_m68k_cf5208'], + 'prefix': 'm68k', + }, + 'microblazebe': { + 'conditions': ['BR2_microblazebe'], + 'prefix': 'microblaze', + }, + 'microblazeel': { + 'conditions': ['BR2_microblazeel'], + 'prefix': 'microblazeel', + }, + 'mips32': { + # Not sure it could be used by other mips32 variants? + 'conditions': ['BR2_mips', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'], + 'prefix': 'mips', + }, + 'mips32el': { + # Not sure it could be used by other mips32el variants? + 'conditions': ['BR2_mipsel', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'], + 'prefix': 'mipsel', + }, + 'mips32r5el': { + 'conditions': ['BR2_mipsel', 'BR2_mips_32r5', '!BR2_MIPS_SOFT_FLOAT'], + 'prefix': 'mipsel', + }, + 'mips32r6el': { + 'conditions': ['BR2_mipsel', 'BR2_mips_32r6', '!BR2_MIPS_SOFT_FLOAT'], + 'prefix': 'mipsel', + }, + 'mips64': { + # Not sure it could be used by other mips64 variants? + 'conditions': ['BR2_mips64', 'BR2_mips_64', '!BR2_MIPS_SOFT_FLOAT'], + 'prefix': 'mips64', + }, + 'mips64-n32': { + # Not sure it could be used by other mips64 variants? + 'conditions': ['BR2_mips64', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'], + 'prefix': 'mips64', + }, + 'mips64el-n32': { + # Not sure it could be used by other mips64el variants? + 'conditions': ['BR2_mips64el', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'], + 'prefix': 'mips64el', + }, + 'mips64r6el-n32': { + 'conditions': ['BR2_mips64el', 'BR2_mips_64r6', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'], + 'prefix': 'mips64el', + }, + 'nios2': { + 'conditions': ['BR2_nios2'], + 'prefix': 'nios2', + }, + 'openrisc': { + 'conditions': ['BR2_or1k'], + 'prefix': 'or1k', + }, + 'powerpc-e500mc': { + # Not sure it could be used by other powerpc variants? + 'conditions': ['BR2_powerpc', 'BR2_powerpc_e500mc'], + 'prefix': 'powerpc', + }, + 'powerpc64-e5500': { + 'conditions': ['BR2_powerpc64', 'BR2_powerpc_e5500'], + 'prefix': 'powerpc64', + }, + 'powerpc64-power8': { + 'conditions': ['BR2_powerpc64', 'BR2_powerpc_power8'], + 'prefix': 'powerpc64', + }, + 'powerpc64le-power8': { + 'conditions': ['BR2_powerpc64le', 'BR2_powerpc_power8'], + 'prefix': 'powerpc64le', + }, + 'riscv32-ilp32d': { + 'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_32', 'BR2_RISCV_ABI_ILP32D'], + 'prefix': 'riscv32', + }, + 'riscv64': { + 'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_64', 'BR2_RISCV_ABI_LP64'], + 'prefix': 'riscv64', + }, + 'sh-sh4': { + 'conditions': ['BR2_sh', 'BR2_sh4'], + 'prefix': 'sh4', + }, + 'sh-sh4aeb': { + 'conditions': ['BR2_sh', 'BR2_sh4aeb'], + 'prefix': 'sh4aeb', + }, + 'sparc64': { + 'conditions': ['BR2_sparc64', 'BR2_sparc_v9'], + 'prefix': 'sparc64', + }, + 'sparcv8': { + 'conditions': ['BR2_sparc', 'BR2_sparc_v8'], + 'prefix': 'sparc', + }, + 'x86-64-core-i7': { + 'conditions': ['BR2_x86_64', + 'BR2_X86_CPU_HAS_MMX', + 'BR2_X86_CPU_HAS_SSE', + 'BR2_X86_CPU_HAS_SSE2', + 'BR2_X86_CPU_HAS_SSE3', + 'BR2_X86_CPU_HAS_SSSE3', + 'BR2_X86_CPU_HAS_SSE4', + 'BR2_X86_CPU_HAS_SSE42'], + 'test_options': ['BR2_x86_64', 'BR2_x86_corei7'], + 'prefix': 'x86_64', + }, + 'x86-core2': { + 'conditions': ['BR2_i386', + 'BR2_X86_CPU_HAS_MMX', + 'BR2_X86_CPU_HAS_SSE', + 'BR2_X86_CPU_HAS_SSE2', + 'BR2_X86_CPU_HAS_SSE3', + 'BR2_X86_CPU_HAS_SSSE3'], + 'test_options': ['BR2_i386', 'BR2_x86_core2'], + 'prefix': 'i686', + }, + 'x86-i686': { + 'conditions': ['BR2_i386', + '!BR2_x86_i486', + '!BR2_x86_i586', + '!BR2_x86_x1000'], + 'test_options': ['BR2_i386', + 'BR2_x86_i686'], + 'prefix': 'i686', + }, + 'xtensa-lx60': { + 'conditions': ['BR2_xtensa', 'BR2_xtensa_fsf'], + 'prefix': 'xtensa', + }, +} + + +class Toolchain: + def __init__(self, arch, libc, variant, version): + self.arch = arch + self.libc = libc + self.variant = variant + self.version = version + self.fname_prefix = "%s--%s--%s-%s" % (self.arch, self.libc, self.variant, self.version) + self.option_name = "BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_%s_%s_%s" % \ + (self.arch.replace("-", "_").upper(), self.libc.upper(), self.variant.replace("-", "_").upper()) + self.fragment = requests.get(self.fragment_url).text.split("\n") + self.sha256 = requests.get(self.hash_url).text.split(" ")[0] + + @property + def tarball_url(self): + return os.path.join(BASE_URL, self.arch, "tarballs", + self.fname_prefix + ".tar.bz2") + + @property + def hash_url(self): + return os.path.join(BASE_URL, self.arch, "tarballs", + self.fname_prefix + ".sha256") + + @property + def fragment_url(self): + return os.path.join(BASE_URL, self.arch, "fragments", + self.fname_prefix + ".frag") + + def gen_config_in_options(self, f): + f.write("config %s\n" % self.option_name) + f.write("\tbool \"%s %s %s %s\"\n" % + (self.arch, self.libc, self.variant, self.version)) + for c in arches[self.arch]['conditions']: + f.write("\tdepends on %s\n" % c) + selects = [] + for frag in self.fragment: + # libc type + if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_UCLIBC"): + selects.append("BR2_TOOLCHAIN_EXTERNAL_UCLIBC") + elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_GLIBC"): + selects.append("BR2_TOOLCHAIN_EXTERNAL_GLIBC") + # all glibc toolchains have RPC support + selects.append("BR2_TOOLCHAIN_HAS_NATIVE_RPC") + elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_MUSL"): + selects.append("BR2_TOOLCHAIN_EXTERNAL_MUSL") + + # gcc version + if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_GCC_"): + m = re.match("^BR2_TOOLCHAIN_EXTERNAL_GCC_([0-9_]*)=y$", frag) + assert m, "Cannot get gcc version for toolchain %s" % self.fname_prefix + selects.append("BR2_TOOLCHAIN_GCC_AT_LEAST_%s" % m[1]) + + # kernel headers version + if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HEADERS_"): + m = re.match("^BR2_TOOLCHAIN_EXTERNAL_HEADERS_([0-9_]*)=y$", frag) + assert m, "Cannot get kernel headers version for toolchain %s" % self.fname_prefix + selects.append("BR2_TOOLCHAIN_HEADERS_AT_LEAST_%s" % m[1]) + + # C++ + if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CXX"): + selects.append("BR2_INSTALL_LIBSTDCPP") + + # SSP + if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_SSP"): + selects.append("BR2_TOOLCHAIN_HAS_SSP") + + # wchar + if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_WCHAR"): + selects.append("BR2_USE_WCHAR") + + # locale + if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_LOCALE"): + # locale implies the availability of wchar + selects.append("BR2_USE_WCHAR") + selects.append("BR2_ENABLE_LOCALE") + + # thread support + if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS"): + selects.append("BR2_TOOLCHAIN_HAS_THREADS") + + if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_DEBUG"): + selects.append("BR2_TOOLCHAIN_HAS_THREADS_DEBUG") + + if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_NPTL"): + selects.append("BR2_TOOLCHAIN_HAS_THREADS_NPTL") + + # RPC + if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_INET_RPC"): + selects.append("BR2_TOOLCHAIN_HAS_NATIVE_RPC") + + # D language + if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_DLANG"): + selects.append("BR2_TOOLCHAIN_HAS_DLANG") + + # fortran + if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_FORTRAN"): + selects.append("BR2_TOOLCHAIN_HAS_FORTRAN") + + # OpenMP + if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_OPENMP"): + selects.append("BR2_TOOLCHAIN_HAS_OPENMP") + + for select in selects: + f.write("\tselect %s\n" % select) + + f.write("\thelp\n") + + desc = "Bootlin toolchain for the %s architecture, using the %s C library. " % \ + (self.arch, self.libc) + + if self.variant == "stable": + desc += "This is a stable version, which means it is using stable and proven versions of gcc, gdb and binutils." + else: + desc += "This is a bleeding-edge version, which means it is using the latest versions of gcc, gdb and binutils." + + f.write(textwrap.fill(desc, width=62, initial_indent="\t ", subsequent_indent="\t ") + "\n") + f.write("\n") + f.write("\t https://toolchains.bootlin.com/\n") + + f.write("\n") + + def gen_mk(self, f): + f.write("ifeq ($(%s),y)\n" % self.option_name) + f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION = %s\n" % self.version) + f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SOURCE = %s--%s--%s-$(TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION).tar.bz2\n" % + (self.arch, self.libc, self.variant)) + f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SITE = %s\n" % + os.path.join(BASE_URL, self.arch, "tarballs")) + f.write("endif\n\n") + pass + + def gen_hash(self, f): + f.write("# From %s\n" % self.hash_url) + f.write("sha256 %s %s\n" % (self.sha256, os.path.basename(self.tarball_url))) + + def gen_test(self, f): + if self.variant == "stable": + variant = "Stable" + else: + variant = "BleedingEdge" + testname = "TestExternalToolchainBootlin" + \ + self.arch.replace("-", "").capitalize() + \ + self.libc.capitalize() + variant + f.write("\n\n") + f.write("class %s(TestExternalToolchain):\n" % testname) + f.write(" config = \"\"\"\n") + if 'test_options' in arches[self.arch]: + test_options = arches[self.arch]['test_options'] + else: + test_options = arches[self.arch]['conditions'] + for opt in test_options: + if opt.startswith("!"): + f.write(" # %s is not set\n" % opt[1:]) + else: + f.write(" %s=y\n" % opt) + f.write(" BR2_TOOLCHAIN_EXTERNAL=y\n") + f.write(" BR2_TOOLCHAIN_EXTERNAL_BOOTLIN=y\n") + f.write(" %s=y\n" % self.option_name) + f.write(" # BR2_TARGET_ROOTFS_TAR is not set\n") + f.write(" \"\"\"\n") + f.write(" toolchain_prefix = \"%s-linux\"\n" % arches[self.arch]['prefix']) + f.write("\n") + f.write(" def test_run(self):\n") + f.write(" TestExternalToolchain.common_check(self)\n") + + def __repr__(self): + return "Toolchain(arch=%s libc=%s variant=%s version=%s, option=%s)" % \ + (self.arch, self.libc, self.variant, self.version, self.option_name) + + +def get_toolchains(): + toolchains = list() + for arch, details in arches.items(): + print(arch) + url = os.path.join(BASE_URL, arch, "available_toolchains") + page = requests.get(url).text + fnames = sorted(re.findall(r'