util/gen_arm_fs_files.py

   1 #!/usr/bin/env python3
   2
   3 # Copyright (c) 2020 ARM Limited
   4 # All rights reserved.
   5 #
   6 # The license below extends only to copyright in the software and shall
   7 # not be construed as granting a license to any other intellectual
   8 # property including but not limited to intellectual property relating
   9 # to a hardware implementation of the functionality of the software
  10 # licensed hereunder.  You may use the software subject to the license
  11 # terms below provided that you ensure that this notice is replicated
  12 # unmodified and in its entirety in all distributions of the software,
  13 # modified or unmodified, in source code or in binary form.
  14 #
  15 # Copyright (c) 2017-2018 Metempsy Technology Consulting
  16 # All rights reserved.
  17 #
  18 # Redistribution and use in source and binary forms, with or without
  19 # modification, are permitted provided that the following conditions are
  20 # met: redistributions of source code must retain the above copyright
  21 # notice, this list of conditions and the following disclaimer;
  22 # redistributions in binary form must reproduce the above copyright
  23 # notice, this list of conditions and the following disclaimer in the
  24 # documentation and/or other materials provided with the distribution;
  25 # neither the name of the copyright holders nor the names of its
  26 # contributors may be used to endorse or promote products derived from
  27 # this software without specific prior written permission.
  28 #
  29 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  30 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  31 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  32 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  33 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  34 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  35 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  36 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  37 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  38 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  39 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  40
  41 from optparse import OptionParser
  42 from subprocess import call
  43 from platform import machine
  44 from distutils import spawn
  45 from glob import glob
  46
  47 import sys
  48 import os
  49
  50 def run_cmd(explanation, working_dir, cmd, stdout = None):
  51     print("Running phase '%s'" % explanation)
  52     sys.stdout.flush()
  53
  54     # some of the commands need $PWD to be properly set
  55     env = os.environ.copy()
  56     env['PWD'] = working_dir
  57
  58     return_code = call(cmd, cwd = working_dir, stdout = stdout,
  59                        env = env)
  60
  61     if return_code == 0:
  62         return
  63
  64     print("Error running phase %s. Returncode: %d" % (explanation, return_code))
  65     sys.exit(1)
  66
  67 def linux_clone():
  68     kernel_vexpress_gem5_dir = os.path.join(
  69         options.dest_dir, "linux-kernel-vexpress_gem5")
  70
  71     run_cmd("clone linux kernel for VExpress_GEM5_V1 platform",
  72         options.dest_dir,
  73         ["git", "clone", "https://gem5.googlesource.com/arm/linux",
  74          kernel_vexpress_gem5_dir])
  75
  76 def linux64():
  77     kernel_vexpress_gem5_dir = os.path.join(
  78         options.dest_dir, "linux-kernel-vexpress_gem5")
  79
  80     linux_bin = os.path.join(
  81         binaries_dir, "vmlinux.vexpress_gem5_v1_64")
  82
  83     with open(revisions_dir + "/linux", "w+") as rev_file:
  84         run_cmd("write revision of linux-kernel-vexpress_gem5 repo",
  85             kernel_vexpress_gem5_dir,
  86             ["git", "rev-parse", "--short", "HEAD"],
  87             rev_file)
  88
  89     run_cmd("configure kernel for arm64",
  90         kernel_vexpress_gem5_dir,
  91         ["make", "ARCH=arm64", "CROSS_COMPILE=aarch64-linux-gnu-",
  92          "gem5_defconfig", make_jobs_str])
  93     run_cmd("compile kernel for arm64",
  94         kernel_vexpress_gem5_dir,
  95         ["make", "ARCH=arm64", "CROSS_COMPILE=aarch64-linux-gnu-",
  96         make_jobs_str])
  97     run_cmd("copy arm64 vmlinux",
  98         kernel_vexpress_gem5_dir,
  99         ["cp", "vmlinux", linux_bin])
 100     run_cmd("cleanup arm64 kernel compilation",
 101         kernel_vexpress_gem5_dir,
 102         ["make", "distclean"])
 103
 104 def linux32():
 105     kernel_vexpress_gem5_dir = os.path.join(
 106         options.dest_dir, "linux-kernel-vexpress_gem5")
 107
 108     linux_bin = os.path.join(
 109         binaries_dir, "vmlinux.vexpress_gem5_v1")
 110
 111     run_cmd("configure kernel for arm",
 112         kernel_vexpress_gem5_dir,
 113         ["make", "ARCH=arm", "CROSS_COMPILE=arm-linux-gnueabihf-",
 114          "gem5_defconfig"])
 115     run_cmd("compile kernel for arm",
 116         kernel_vexpress_gem5_dir,
 117         ["make", "ARCH=arm", "CROSS_COMPILE=arm-linux-gnueabihf-",
 118         make_jobs_str])
 119     run_cmd("copy arm vmlinux",
 120         kernel_vexpress_gem5_dir,
 121         ["cp", "vmlinux", linux_bin])
 122
 123 def linux():
 124     """
 125     Checkout and build linux kernel for VExpress_GEM5_V1 (arm and arm64)
 126     """
 127     linux_clone()
 128     linux64()
 129     linux32()
 130
 131 def linux_legacy():
 132     """
 133     Checkout and build linux kernel and DTB for VExpress_EMM64/EMM
 134     """
 135     kernel_vexpress_emm64_dir = os.path.join(options.dest_dir,
 136         "linux-kernel-vexpress_emm64")
 137     run_cmd("clone linux kernel for VExpress_EMM64 platform",
 138         options.dest_dir,
 139         ["git", "clone", "https://gem5.googlesource.com/arm/linux-arm64-legacy",
 140          kernel_vexpress_emm64_dir])
 141     with open(revisions_dir + "/linux-arm64-legacy", "w+") as rev_file:
 142         run_cmd("write revision of linux-kernel-vexpress_emm64 repo",
 143             kernel_vexpress_emm64_dir,
 144             ["git", "rev-parse", "--short", "HEAD"],
 145             rev_file)
 146     run_cmd("configure kernel",
 147         kernel_vexpress_emm64_dir,
 148         ["make", "ARCH=arm64", "CROSS_COMPILE=aarch64-linux-gnu-",
 149          "CC=aarch64-linux-gnu-gcc-4.8", "gem5_defconfig"])
 150     run_cmd("compile kernel",
 151         kernel_vexpress_emm64_dir,
 152         ["make", "ARCH=arm64", "CROSS_COMPILE=aarch64-linux-gnu-",
 153          "CC=aarch64-linux-gnu-gcc-4.8", make_jobs_str])
 154     run_cmd("copy vmlinux",
 155         kernel_vexpress_emm64_dir,
 156         ["cp", "vmlinux", binaries_dir + "/vmlinux.vexpress_emm64"])
 157     run_cmd("copy DTB",
 158         kernel_vexpress_emm64_dir,
 159         ["cp", "arch/arm64/boot/dts/aarch64_gem5_server.dtb", binaries_dir])
 160
 161     kernel_vexpress_emm_dir = options.dest_dir + "/linux-kernel-vexpress_emm"
 162     run_cmd("clone linux kernel for VExpress_EMM platform",
 163         options.dest_dir,
 164         ["git", "clone", "https://gem5.googlesource.com/arm/linux-arm-legacy",
 165          kernel_vexpress_emm_dir])
 166     with open(revisions_dir + "/linux-arm-legacy", "w+") as rev_file:
 167         run_cmd("write revision of linux-kernel-vexpress_emm64 repo",
 168             kernel_vexpress_emm_dir,
 169             ["git", "rev-parse", "--short", "HEAD"],
 170             rev_file)
 171     run_cmd("configure kernel",
 172         kernel_vexpress_emm_dir,
 173         ["make", "ARCH=arm", "CROSS_COMPILE=arm-linux-gnueabihf-",
 174          "CC=arm-linux-gnueabihf-gcc-4.8", "vexpress_gem5_server_defconfig"])
 175     run_cmd("compile kernel",
 176         kernel_vexpress_emm_dir,
 177         ["make", "ARCH=arm", "CROSS_COMPILE=arm-linux-gnueabihf-",
 178          "CC=arm-linux-gnueabihf-gcc-4.8", make_jobs_str])
 179     run_cmd("copy vmlinux",
 180         kernel_vexpress_emm_dir,
 181         ["cp", "vmlinux", binaries_dir + "/vmlinux.vexpress_emm"])
 182     run_cmd("rename DTB for 1 CPU",
 183         kernel_vexpress_emm_dir,
 184         ["cp", "arch/arm/boot/dts/vexpress-v2p-ca15-tc1-gem5.dtb",
 185          binaries_dir + "/vexpress-v2p-ca15-tc1-gem5_1cpus.dtb"])
 186     run_cmd("copy DTBs",
 187         kernel_vexpress_emm_dir,
 188         ["cp"] + glob(kernel_vexpress_emm_dir + "/arch/arm/boot/dts/*gem5_*dtb") +
 189         [binaries_dir])
 190
 191 def dtbs():
 192     """
 193     Build DTBs for VExpress_GEM5_V1
 194     """
 195     dt_dir = gem5_dir + "/system/arm/dt"
 196     run_cmd("compile DTBs for VExpress_GEM5_V1 platform",
 197         dt_dir,
 198         ["make", make_jobs_str])
 199     run_cmd("copy DTBs",
 200         dt_dir,
 201         ["cp"] + glob(dt_dir + "/*dtb") + [binaries_dir])
 202
 203 def bootloaders():
 204     """
 205     Build bootloaders arm64/arm
 206     """
 207
 208     bootloader_arm64_dir = gem5_dir + "/system/arm/bootloader/arm64"
 209     run_cmd("compile arm64 bootloader",
 210         bootloader_arm64_dir,
 211         ["make"])
 212     run_cmd("copy arm64 bootloader",
 213         bootloader_arm64_dir,
 214         ["cp", "boot.arm64", "boot_emm.arm64", "boot_v2.arm64", binaries_dir])
 215
 216     bootloader_arm_dir = gem5_dir + "/system/arm/bootloader/arm"
 217     run_cmd("compile arm bootloader",
 218         bootloader_arm_dir,
 219         ["make"])
 220     run_cmd("copy arm bootloaders",
 221         bootloader_arm_dir,
 222         ["cp", "boot.arm", "boot_emm.arm", binaries_dir])
 223
 224 def m5():
 225     """
 226     Build m5 binaries
 227     """
 228     m5_dir = gem5_dir + "/util/m5"
 229     run_cmd("compile arm64 m5",
 230         m5_dir,
 231         ["make", "-f", "Makefile.aarch64"])
 232     run_cmd("copy arm64 m5",
 233         m5_dir,
 234         ["cp", "m5", binaries_dir + "/m5.aarch64"])
 235     run_cmd("clean arm64 m5",
 236         m5_dir,
 237         ["make", "clean", "-f", "Makefile.aarch64"])
 238     run_cmd("compile arm m5",
 239         m5_dir,
 240         ["make", "-f", "Makefile.arm"])
 241     run_cmd("copy arm m5",
 242         m5_dir,
 243         ["cp", "m5", binaries_dir + "/m5.aarch32"])
 244
 245 def xen():
 246     """
 247     Build Xen for aarch64
 248     """
 249     xen_dir = os.path.join(options.dest_dir, "xen")
 250     bootwrapper_dir = os.path.join(options.dest_dir, "bootwrapper")
 251     linux_cmdline = "console=hvc0 root=/dev/vda rw mem=1G"
 252     xen_cmdline = "dtuart=/uart@1c090000 console=dtuart no-bootscrub " + \
 253                   "dom0_mem=1G loglvl=all guest_loglvl=all"
 254
 255     run_cmd("clone Xen",
 256         options.dest_dir,
 257         ["git", "clone", "git://xenbits.xen.org/xen.git",
 258          xen_dir])
 259
 260     run_cmd("clone boot-wrapper-aarch64",
 261         options.dest_dir,
 262         ["git", "clone", "git://git.kernel.org/pub/" +
 263             "scm/linux/kernel/git/mark/boot-wrapper-aarch64.git",
 264          bootwrapper_dir])
 265
 266     # Need to compile arm64 Linux
 267     linux_dir = os.path.join(options.dest_dir, "linux-kernel-vexpress_gem5")
 268     linux_bin = os.path.join(linux_dir,
 269         "arch", "arm64", "boot", "Image")
 270     if not os.path.exists(linux_bin):
 271         linux_clone()
 272         linux64()
 273
 274     # Need to compile DTBs
 275     dtb_bin = os.path.join(binaries_dir, "armv8_gem5_v2_1cpu.dtb")
 276     if not os.path.exists(dtb_bin):
 277         dtbs()
 278
 279     # Building Xen
 280     run_cmd("building xen for aarch64",
 281         xen_dir,
 282         ["make", "dist-xen", "XEN_TARGET_ARCH=arm64",
 283          "CROSS_COMPILE=aarch64-linux-gnu-",
 284          "CONFIG_EARLY_PRINTK=vexpress", make_jobs_str])
 285
 286     # Building boot-wrapper-aarch64
 287     run_cmd("autoreconf boot-wrapper-aarch64",
 288         bootwrapper_dir, ["autoreconf", "-i"])
 289     run_cmd("configure boot-wrapper-aarch64",
 290         bootwrapper_dir, ["./configure",
 291             "--host=aarch64-linux-gnu",
 292             "--with-kernel-dir={}".format(linux_dir),
 293             "--with-dtb={}".format(dtb_bin),
 294             "--with-cmdline='{}'".format(linux_cmdline),
 295             "--with-xen-cmdline='{}'".format(xen_cmdline),
 296             "--with-xen={}".format(os.path.join(xen_dir, "xen", "xen")),
 297             "--enable-psci",
 298             "--enable-gicv3"])
 299     run_cmd("build boot-wrapper-aarch64",
 300         bootwrapper_dir, ["make"])
 301
 302     # Copying the final binary
 303     run_cmd("copy xen binary",
 304         bootwrapper_dir, ["cp", "xen-system.axf", binaries_dir])
 305
 306     with open(os.path.join(revisions_dir, "xen"), "w+") as rev_file:
 307         run_cmd("write revision of xen repo",
 308             xen_dir,
 309             ["git", "rev-parse", "--short", "HEAD"],
 310             rev_file)
 311
 312 script_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
 313 gem5_dir = os.path.dirname(script_dir)
 314
 315 all_binaries = {
 316     "linux" : linux,
 317     "linux-legacy" : linux_legacy,
 318     "dtbs" : dtbs,
 319     "bootloaders" : bootloaders,
 320     "m5" : m5,
 321     "xen" : xen,
 322 }
 323
 324 parser = OptionParser()
 325
 326 parser.add_option("--gem5-dir", default = gem5_dir,
 327     metavar = "GEM5_DIR",
 328     help = "gem5 root directory to be used for bootloader and "
 329            "VExpress_GEM5_V1 DTB sources. The default value is the gem5 root "
 330            "directory of the executed script (%default)")
 331 parser.add_option("--dest-dir", default = "/tmp",
 332     metavar = "DEST_DIR",
 333     help = "Directory to use for checking out the different kernel "
 334            "repositories. Generated files will be copied to "
 335            "DEST_DIR/binaries (which must not exist). The default "
 336            "value is %default")
 337 parser.add_option("-j", "--make-jobs", type = "int", default = 1,
 338     metavar = "MAKE_JOBS",
 339     help = "Number of jobs to use with the 'make' commands. Default value: "
 340            "%default")
 341 parser.add_option("-b", "--fs-binaries", action="append",
 342     choices=list(all_binaries.keys()), default=[],
 343     help = "List of FS files to be generated. Defaulting to all")
 344
 345 (options, args) = parser.parse_args()
 346
 347 if args:
 348     print("Unrecognized argument(s) %s." % args)
 349     sys.exit(1)
 350
 351 if not os.path.isdir(options.dest_dir):
 352     print("Error: %s is not a directory." % options.dest_dir)
 353     sys.exit(1)
 354
 355 if not os.path.isdir(options.gem5_dir):
 356     print("Error: %s is not a directory." % options.gem5_dir)
 357     sys.exit(1)
 358
 359 if machine() != "x86_64":
 360     print("Error: This script should run in a x86_64 machine")
 361     sys.exit(1)
 362
 363 binaries_dir = options.dest_dir + "/binaries"
 364
 365 if os.path.exists(binaries_dir):
 366     print("Error: %s already exists." % binaries_dir)
 367     sys.exit(1)
 368
 369 revisions_dir = options.dest_dir + "/revisions"
 370
 371 if os.path.exists(revisions_dir):
 372     print("Error: %s already exists." %revisions_dir)
 373     sys.exit(1)
 374
 375 os.mkdir(binaries_dir);
 376 os.mkdir(revisions_dir);
 377
 378 make_jobs_str = "-j" + str(options.make_jobs)
 379
 380 rev_file = open(revisions_dir + "/gem5", "w+")
 381 run_cmd("write revision of gem5 repo",
 382     gem5_dir,
 383     ["git", "rev-parse", "--short", "HEAD"],
 384     rev_file)
 385 rev_file.close()
 386
 387 binaries = options.fs_binaries if options.fs_binaries else list(all_binaries.keys())
 388 for fs_binary in binaries:
 389     all_binaries[fs_binary]()
 390
 391 print("Done! All the generated files can be found in %s" % binaries_dir)
 392
 393 sys.exit(0)
 394