NetCube-Systems-Austria/kumquat-buildroot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
27ac6619aa
kumquat-buildroot/support/scripts/gen-bootlin-toolchains
Romain Naour 249ce1c584 support/scripts/gen-bootlin-toolchains: add s390x toolchain 
s390x support has been added since toolchain-builder release
2021.05 [1].

[1] 4bbb52a6a3

Signed-off-by: Romain Naour <romain.naour@gmail.com>
Signed-off-by: Arnout Vandecappelle (Essensium/Mind) <arnout@mind.be>
2022-03-10 22:09:22 +01:00

550 lines
20 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python3
import os.path
import re
import requests
import textwrap
BASE_URL = "https://toolchains.bootlin.com/downloads/releases/toolchains"
AUTOGENERATED_COMMENT = """# This file was auto-generated by support/scripts/gen-bootlin-toolchains
# Do not edit
"""
# In the below dict:
# - 'conditions' indicate the cumulative conditions under which the
# toolchain will be made available. In several situations, a given
# toolchain is usable on several architectures variants (for
# example, an ARMv6 toolchain can be used on ARMv7)
# - 'test_options' indicate one specific configuration where the
# toolchain can be used. It is used to create the runtime test
# cases. If 'test_options' does not exist, the code assumes it can
# be made equal to 'conditions'
# - 'prefix' is the prefix of the cross-compilation toolchain tools
arches = {
'aarch64': {
'conditions': ['BR2_aarch64'],
'prefix': 'aarch64',
},
'aarch64be': {
'conditions': ['BR2_aarch64_be'],
'prefix': 'aarch64_be',
},
'arcle-750d': {
'conditions': ['BR2_arcle', 'BR2_arc750d'],
'prefix': 'arc',
},
'arcle-hs38': {
'conditions': ['BR2_arcle', 'BR2_archs38'],
'prefix': 'arc',
},
'armv5-eabi': {
'conditions': ['BR2_ARM_CPU_ARMV5', 'BR2_ARM_EABI'],
'test_options': ['BR2_arm', 'BR2_arm926t', 'BR2_ARM_EABI'],
'prefix': 'arm',
},
'armv6-eabihf': {
'conditions': ['BR2_ARM_CPU_ARMV6', 'BR2_ARM_EABIHF'],
'test_options': ['BR2_arm', 'BR2_arm1176jzf_s', 'BR2_ARM_EABIHF'],
'prefix': 'arm',
},
'armv7-eabihf': {
'conditions': ['BR2_ARM_CPU_ARMV7A', 'BR2_ARM_EABIHF'],
'test_options': ['BR2_arm', 'BR2_cortex_a8', 'BR2_ARM_EABIHF'],
'prefix': 'arm',
},
'armv7m': {
'conditions': ['BR2_ARM_CPU_ARMV7M'],
'test_options': ['BR2_arm', 'BR2_cortex_m4'],
'prefix': 'arm',
},
'm68k-68xxx': {
'conditions': ['BR2_m68k_m68k'],
'test_options': ['BR2_m68k', 'BR2_m68k_68040'],
'prefix': 'm68k',
},
'm68k-coldfire': {
'conditions': ['BR2_m68k_cf'],
'test_options': ['BR2_m68k', 'BR2_m68k_cf5208'],
'prefix': 'm68k',
},
'microblazebe': {
'conditions': ['BR2_microblazebe'],
'prefix': 'microblaze',
},
'microblazeel': {
'conditions': ['BR2_microblazeel'],
'prefix': 'microblazeel',
},
'mips32': {
# Not sure it could be used by other mips32 variants?
'conditions': ['BR2_mips', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'],
'prefix': 'mips',
},
'mips32el': {
# Not sure it could be used by other mips32el variants?
'conditions': ['BR2_mipsel', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'],
'prefix': 'mipsel',
},
'mips32r5el': {
'conditions': ['BR2_mipsel', 'BR2_mips_32r5', '!BR2_MIPS_SOFT_FLOAT'],
'prefix': 'mipsel',
},
'mips32r6el': {
'conditions': ['BR2_mipsel', 'BR2_mips_32r6', '!BR2_MIPS_SOFT_FLOAT'],
'prefix': 'mipsel',
},
'mips64': {
# Not sure it could be used by other mips64 variants?
'conditions': ['BR2_mips64', 'BR2_mips_64', '!BR2_MIPS_SOFT_FLOAT'],
'prefix': 'mips64',
},
'mips64-n32': {
# Not sure it could be used by other mips64 variants?
'conditions': ['BR2_mips64', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
'prefix': 'mips64',
},
'mips64el-n32': {
# Not sure it could be used by other mips64el variants?
'conditions': ['BR2_mips64el', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
'prefix': 'mips64el',
},
'mips64r6el-n32': {
'conditions': ['BR2_mips64el', 'BR2_mips_64r6', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
'prefix': 'mips64el',
},
'nios2': {
'conditions': ['BR2_nios2'],
'prefix': 'nios2',
},
'openrisc': {
'conditions': ['BR2_or1k'],
'prefix': 'or1k',
},
'powerpc-440fp': {
# Not sure it could be used by other powerpc variants?
'conditions': ['BR2_powerpc', 'BR2_powerpc_440fp'],
'prefix': 'powerpc',
},
'powerpc-e300c3': {
# Not sure it could be used by other powerpc variants?
'conditions': ['BR2_powerpc', 'BR2_powerpc_e300c3'],
'prefix': 'powerpc',
},
'powerpc-e500mc': {
# Not sure it could be used by other powerpc variants?
'conditions': ['BR2_powerpc', 'BR2_powerpc_e500mc'],
'prefix': 'powerpc',
},
'powerpc64-e5500': {
'conditions': ['BR2_powerpc64', 'BR2_powerpc_e5500'],
'prefix': 'powerpc64',
},
'powerpc64-e6500': {
'conditions': ['BR2_powerpc64', 'BR2_powerpc_e6500'],
'prefix': 'powerpc64',
},
'powerpc64-power8': {
'conditions': ['BR2_powerpc64', 'BR2_powerpc_power8'],
'prefix': 'powerpc64',
},
'powerpc64le-power8': {
'conditions': ['BR2_powerpc64le', 'BR2_powerpc_power8'],
'prefix': 'powerpc64le',
},
'riscv32-ilp32d': {
'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_32', 'BR2_RISCV_ABI_ILP32D'],
'prefix': 'riscv32',
},
'riscv64-lp64d': {
'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_64', 'BR2_RISCV_ABI_LP64D'],
'prefix': 'riscv64',
},
's390x-z13': {
'conditions': ['BR2_s390x', 'BR2_s390x_z13'],
'prefix': 's390x',
},
'sh-sh4': {
'conditions': ['BR2_sh', 'BR2_sh4'],
'prefix': 'sh4',
},
'sh-sh4aeb': {
'conditions': ['BR2_sh', 'BR2_sh4aeb'],
'prefix': 'sh4aeb',
},
'sparc64': {
'conditions': ['BR2_sparc64', 'BR2_sparc_v9'],
'prefix': 'sparc64',
},
'sparcv8': {
'conditions': ['BR2_sparc', 'BR2_sparc_v8'],
'prefix': 'sparc',
},
'x86-64': {
'conditions': ['BR2_x86_64',
'BR2_X86_CPU_HAS_MMX',
'BR2_X86_CPU_HAS_SSE',
'BR2_X86_CPU_HAS_SSE2'],
'test_options': ['BR2_x86_64', 'BR2_x86_x86_64'],
'prefix': 'x86_64',
},
'x86-64-v2': {
'conditions': ['BR2_x86_64',
'BR2_X86_CPU_HAS_MMX',
'BR2_X86_CPU_HAS_SSE',
'BR2_X86_CPU_HAS_SSE2',
'BR2_X86_CPU_HAS_SSE3',
'BR2_X86_CPU_HAS_SSSE3',
'BR2_X86_CPU_HAS_SSE4',
'BR2_X86_CPU_HAS_SSE42'],
'test_options': ['BR2_x86_64', 'BR2_x86_x86_64_v2'],
'prefix': 'x86_64',
},
'x86-64-v3': {
'conditions': ['BR2_x86_64',
'BR2_X86_CPU_HAS_MMX',
'BR2_X86_CPU_HAS_SSE',
'BR2_X86_CPU_HAS_SSE2',
'BR2_X86_CPU_HAS_SSE3',
'BR2_X86_CPU_HAS_SSSE3',
'BR2_X86_CPU_HAS_SSE4',
'BR2_X86_CPU_HAS_SSE42',
'BR2_X86_CPU_HAS_AVX',
'BR2_X86_CPU_HAS_AVX2'],
'test_options': ['BR2_x86_64', 'BR2_x86_x86_64_v3'],
'prefix': 'x86_64',
},
'x86-64-v4': {
'conditions': ['BR2_x86_64',
'BR2_X86_CPU_HAS_MMX',
'BR2_X86_CPU_HAS_SSE',
'BR2_X86_CPU_HAS_SSE2',
'BR2_X86_CPU_HAS_SSE3',
'BR2_X86_CPU_HAS_SSSE3',
'BR2_X86_CPU_HAS_SSE4',
'BR2_X86_CPU_HAS_SSE42',
'BR2_X86_CPU_HAS_AVX',
'BR2_X86_CPU_HAS_AVX2',
'BR2_X86_CPU_HAS_AVX512'],
'test_options': ['BR2_x86_64', 'BR2_x86_x86_64_v4'],
'prefix': 'x86_64',
},
'x86-64-core-i7': {
'conditions': ['BR2_x86_64',
'BR2_X86_CPU_HAS_MMX',
'BR2_X86_CPU_HAS_SSE',
'BR2_X86_CPU_HAS_SSE2',
'BR2_X86_CPU_HAS_SSE3',
'BR2_X86_CPU_HAS_SSSE3',
'BR2_X86_CPU_HAS_SSE4',
'BR2_X86_CPU_HAS_SSE42'],
'test_options': ['BR2_x86_64', 'BR2_x86_corei7'],
'prefix': 'x86_64',
},
'x86-core2': {
'conditions': ['BR2_i386',
'BR2_X86_CPU_HAS_MMX',
'BR2_X86_CPU_HAS_SSE',
'BR2_X86_CPU_HAS_SSE2',
'BR2_X86_CPU_HAS_SSE3',
'BR2_X86_CPU_HAS_SSSE3'],
'test_options': ['BR2_i386', 'BR2_x86_core2'],
'prefix': 'i686',
},
'x86-i686': {
'conditions': ['BR2_i386',
'!BR2_x86_i486',
'!BR2_x86_i586',
'!BR2_x86_x1000'],
'test_options': ['BR2_i386',
'BR2_x86_i686'],
'prefix': 'i686',
},
'xtensa-lx60': {
'conditions': ['BR2_xtensa', 'BR2_XTENSA_CUSTOM', 'BR2_XTENSA_LITTLE_ENDIAN'],
'prefix': 'xtensa',
},
}
class Toolchain:
def __init__(self, arch, libc, variant, version):
self.arch = arch
self.libc = libc
self.variant = variant
self.version = version
self.fname_prefix = "%s--%s--%s-%s" % (self.arch, self.libc, self.variant, self.version)
self.option_name = "BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_%s_%s_%s" % \
(self.arch.replace("-", "_").upper(), self.libc.upper(), self.variant.replace("-", "_").upper())
self.fragment = requests.get(self.fragment_url).text.split("\n")
self.sha256 = requests.get(self.hash_url).text.split(" ")[0]
@property
def tarball_url(self):
return os.path.join(BASE_URL, self.arch, "tarballs",
self.fname_prefix + ".tar.bz2")
@property
def hash_url(self):
return os.path.join(BASE_URL, self.arch, "tarballs",
self.fname_prefix + ".sha256")
@property
def fragment_url(self):
return os.path.join(BASE_URL, self.arch, "fragments",
self.fname_prefix + ".frag")
def gen_config_in_options(self, f):
f.write("config %s\n" % self.option_name)
f.write("\tbool \"%s %s %s %s\"\n" %
(self.arch, self.libc, self.variant, self.version))
depends = []
selects = []
for c in arches[self.arch]['conditions']:
depends.append(c)
for frag in self.fragment:
# libc type
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_UCLIBC"):
selects.append("BR2_TOOLCHAIN_EXTERNAL_UCLIBC")
elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_GLIBC"):
# glibc needs mmu support
depends.append("BR2_USE_MMU")
# glibc doesn't support static only configuration
depends.append("!BR2_STATIC_LIBS")
selects.append("BR2_TOOLCHAIN_EXTERNAL_GLIBC")
elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_MUSL"):
# musl needs mmu support
depends.append("BR2_USE_MMU")
selects.append("BR2_TOOLCHAIN_EXTERNAL_MUSL")
# gcc version
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_GCC_"):
m = re.match("^BR2_TOOLCHAIN_EXTERNAL_GCC_([0-9_]*)=y$", frag)
assert m, "Cannot get gcc version for toolchain %s" % self.fname_prefix
selects.append("BR2_TOOLCHAIN_GCC_AT_LEAST_%s" % m[1])
# kernel headers version
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HEADERS_"):
m = re.match("^BR2_TOOLCHAIN_EXTERNAL_HEADERS_([0-9_]*)=y$", frag)
assert m, "Cannot get kernel headers version for toolchain %s" % self.fname_prefix
selects.append("BR2_TOOLCHAIN_HEADERS_AT_LEAST_%s" % m[1])
# C++
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CXX"):
selects.append("BR2_INSTALL_LIBSTDCPP")
# SSP
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_SSP"):
selects.append("BR2_TOOLCHAIN_HAS_SSP")
# wchar
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_WCHAR"):
selects.append("BR2_USE_WCHAR")
# locale
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_LOCALE"):
# locale implies the availability of wchar
selects.append("BR2_USE_WCHAR")
selects.append("BR2_ENABLE_LOCALE")
# thread support
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS"):
selects.append("BR2_TOOLCHAIN_HAS_THREADS")
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_DEBUG"):
selects.append("BR2_TOOLCHAIN_HAS_THREADS_DEBUG")
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_NPTL"):
selects.append("BR2_TOOLCHAIN_HAS_THREADS_NPTL")
# RPC
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_INET_RPC"):
selects.append("BR2_TOOLCHAIN_HAS_NATIVE_RPC")
# D language
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_DLANG"):
selects.append("BR2_TOOLCHAIN_HAS_DLANG")
# fortran
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_FORTRAN"):
selects.append("BR2_TOOLCHAIN_HAS_FORTRAN")
# OpenMP
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_OPENMP"):
selects.append("BR2_TOOLCHAIN_HAS_OPENMP")
for depend in depends:
f.write("\tdepends on %s\n" % depend)
for select in selects:
f.write("\tselect %s\n" % select)
f.write("\thelp\n")
desc = "Bootlin toolchain for the %s architecture, using the %s C library. " % \
(self.arch, self.libc)
if self.variant == "stable":
desc += "This is a stable version, which means it is using stable and proven versions of gcc, gdb and binutils."
else:
desc += "This is a bleeding-edge version, which means it is using the latest versions of gcc, gdb and binutils."
f.write(textwrap.fill(desc, width=62, initial_indent="\t ", subsequent_indent="\t ") + "\n")
f.write("\n")
f.write("\t https://toolchains.bootlin.com/\n")
f.write("\n")
def gen_mk(self, f):
f.write("ifeq ($(%s),y)\n" % self.option_name)
f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION = %s\n" % self.version)
f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SOURCE = %s--%s--%s-$(TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION).tar.bz2\n" %
(self.arch, self.libc, self.variant))
f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SITE = %s\n" %
os.path.join(BASE_URL, self.arch, "tarballs"))
f.write("endif\n\n")
pass
def gen_hash(self, f):
f.write("# From %s\n" % self.hash_url)
f.write("sha256 %s %s\n" % (self.sha256, os.path.basename(self.tarball_url)))
def gen_test(self, f):
if self.variant == "stable":
variant = "Stable"
else:
variant = "BleedingEdge"
testname = "TestExternalToolchainBootlin" + \
self.arch.replace("-", "").capitalize() + \
self.libc.capitalize() + variant
f.write("\n\n")
f.write("class %s(TestExternalToolchain):\n" % testname)
f.write(" config = \"\"\"\n")
if 'test_options' in arches[self.arch]:
test_options = arches[self.arch]['test_options']
else:
test_options = arches[self.arch]['conditions']
for opt in test_options:
if opt.startswith("!"):
f.write(" # %s is not set\n" % opt[1:])
else:
f.write(" %s=y\n" % opt)
f.write(" BR2_TOOLCHAIN_EXTERNAL=y\n")
f.write(" BR2_TOOLCHAIN_EXTERNAL_BOOTLIN=y\n")
f.write(" %s=y\n" % self.option_name)
f.write(" # BR2_TARGET_ROOTFS_TAR is not set\n")
f.write(" \"\"\"\n")
f.write(" toolchain_prefix = \"%s-linux\"\n" % arches[self.arch]['prefix'])
f.write("\n")
f.write(" def test_run(self):\n")
f.write(" TestExternalToolchain.common_check(self)\n")
def __repr__(self):
return "Toolchain(arch=%s libc=%s variant=%s version=%s, option=%s)" % \
(self.arch, self.libc, self.variant, self.version, self.option_name)
def get_toolchains():
toolchains = list()
for arch, details in arches.items():
print(arch)
url = os.path.join(BASE_URL, arch, "available_toolchains")
page = requests.get(url).text
fnames = sorted(re.findall(r'<td><a href="(\w[^"]+)"', page))
# This dict will allow us to keep only the latest version for
# each toolchain.
tmp = dict()
for fname in fnames:
parts = fname.split('--')
assert parts[0] == arch, "Arch does not match: %s vs. %s" % (parts[0], arch)
libc = parts[1]
if parts[2].startswith("stable-"):
variant = "stable"
version = parts[2][len("stable-"):]
elif parts[2].startswith("bleeding-edge-"):
variant = "bleeding-edge"
version = parts[2][len("bleeding-edge-"):]
tmp[(arch, libc, variant)] = version
toolchains += [Toolchain(k[0], k[1], k[2], v) for k, v in tmp.items()]
return toolchains
def gen_config_in_options(toolchains, fpath):
with open(fpath, "w") as f:
f.write(AUTOGENERATED_COMMENT)
f.write("config BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_ARCH_SUPPORTS\n")
f.write("\tbool\n")
for arch, details in arches.items():
f.write("\tdefault y if %s\n" % " && ".join(details['conditions']))
f.write("\n")
f.write("if BR2_TOOLCHAIN_EXTERNAL_BOOTLIN\n\n")
f.write("config BR2_TOOLCHAIN_EXTERNAL_PREFIX\n")
f.write("\tdefault \"$(ARCH)-linux\"\n")
f.write("\n")
f.write("config BR2_PACKAGE_PROVIDES_TOOLCHAIN_EXTERNAL\n")
f.write("\tdefault \"toolchain-external-bootlin\"\n")
f.write("\n")
f.write("choice\n")
f.write("\tprompt \"Bootlin toolchain variant\"\n")
for toolchain in toolchains:
toolchain.gen_config_in_options(f)
f.write("endchoice\n")
f.write("endif\n")
def gen_mk(toolchains, fpath):
with open(fpath, "w") as f:
f.write("#" * 80 + "\n")
f.write("#\n")
f.write("# toolchain-external-bootlin\n")
f.write("#\n")
f.write("#" * 80 + "\n")
f.write("\n")
f.write(AUTOGENERATED_COMMENT)
for toolchain in toolchains:
toolchain.gen_mk(f)
f.write("$(eval $(toolchain-external-package))\n")
def gen_hash(toolchains, fpath):
with open(fpath, "w") as f:
f.write(AUTOGENERATED_COMMENT)
for toolchain in toolchains:
toolchain.gen_hash(f)
def gen_runtime_test(toolchains, fpath):
with open(fpath, "w") as f:
f.write(AUTOGENERATED_COMMENT)
f.write("from tests.toolchain.test_external import TestExternalToolchain\n")
for toolchain in toolchains:
toolchain.gen_test(f)
def gen_toolchains(toolchains):
maindir = "toolchain/toolchain-external/toolchain-external-bootlin"
gen_config_in_options(toolchains, os.path.join(maindir, "Config.in.options"))
gen_mk(toolchains, os.path.join(maindir, "toolchain-external-bootlin.mk"))
gen_hash(toolchains, os.path.join(maindir, "toolchain-external-bootlin.hash"))
gen_runtime_test(toolchains,
os.path.join("support", "testing", "tests", "toolchain", "test_external_bootlin.py"))
toolchains = get_toolchains()
gen_toolchains(toolchains)
Reference in New Issue View Git Blame Copy Permalink