util/cpt_upgrader.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2012-2013 ARM Limited
   4 # All rights reserved
   5 #
   6 # The license below extends only to copyright in the software and shall
   7 # not be construed as granting a license to any other intellectual
   8 # property including but not limited to intellectual property relating
   9 # to a hardware implementation of the functionality of the software
  10 # licensed hereunder.  You may use the software subject to the license
  11 # terms below provided that you ensure that this notice is replicated
  12 # unmodified and in its entirety in all distributions of the software,
  13 # modified or unmodified, in source code or in binary form.
  14 #
  15 # Redistribution and use in source and binary forms, with or without
  16 # modification, are permitted provided that the following conditions are
  17 # met: redistributions of source code must retain the above copyright
  18 # notice, this list of conditions and the following disclaimer;
  19 # redistributions in binary form must reproduce the above copyright
  20 # notice, this list of conditions and the following disclaimer in the
  21 # documentation and/or other materials provided with the distribution;
  22 # neither the name of the copyright holders nor the names of its
  23 # contributors may be used to endorse or promote products derived from
  24 # this software without specific prior written permission.
  25 #
  26 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  27 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  28 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  29 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  30 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  31 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  32 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  33 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  34 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  35 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  36 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  37 #
  38 # Authors: Ali Saidi
  39 #
  40
  41 # This python code is used to migrate checkpoints that were created in one
  42 # version of the simulator to newer version. As features are added or bugs are
  43 # fixed some of the state that needs to be checkpointed can change. If you have
  44 # many historic checkpoints that you use, manually editing them to fix them is
  45 # both time consuming and error-prone.
  46
  47 # This script provides a way to migrate checkpoints to the newer repository in
  48 # a programatic way. It can be imported into another script or used on the
  49 # command line. From the command line the script will either migrate every
  50 # checkpoint it finds recursively (-r option) or a single checkpoint. When a
  51 # change is made to the gem5 repository that breaks previous checkpoints a
  52 # from_N() method should be implemented here and the gem5CheckpointVersion
  53 # variable in src/sim/serialize.hh should be incremented. For each version
  54 # between the checkpoints current version and the new version the from_N()
  55 # method will be run, passing in a ConfigParser object which contains the open
  56 # file. As these operations can be isa specific the method can verify the isa
  57 # and use regexes to find the correct sections that need to be updated.
  58
  59
  60 import ConfigParser
  61 import sys, os
  62 import os.path as osp
  63
  64 # An example of a translator
  65 def from_0(cpt):
  66     if cpt.get('root','isa') == 'arm':
  67         for sec in cpt.sections():
  68             import re
  69             # Search for all the execution contexts
  70             if re.search('.*sys.*\.cpu.*\.x.\..*', sec):
  71                 # Update each one
  72                 mr = cpt.get(sec, 'miscRegs').split()
  73                 #mr.insert(21,0)
  74                 #mr.insert(26,0)
  75                 cpt.set(sec, 'miscRegs', ' '.join(str(x) for x in mr))
  76
  77 # The backing store supporting the memories in the system has changed
  78 # in that it is now stored globally per address range. As a result the
  79 # actual storage is separate from the memory controllers themselves.
  80 def from_1(cpt):
  81     for sec in cpt.sections():
  82         import re
  83         # Search for a physical memory
  84         if re.search('.*sys.*\.physmem$', sec):
  85             # Add the number of stores attribute to the global physmem
  86             cpt.set(sec, 'nbr_of_stores', '1')
  87
  88             # Get the filename and size as this is moving to the
  89             # specific backing store
  90             mem_filename = cpt.get(sec, 'filename')
  91             mem_size = cpt.get(sec, '_size')
  92             cpt.remove_option(sec, 'filename')
  93             cpt.remove_option(sec, '_size')
  94
  95             # Get the name so that we can create the new section
  96             system_name = str(sec).split('.')[0]
  97             section_name = system_name + '.physmem.store0'
  98             cpt.add_section(section_name)
  99             cpt.set(section_name, 'store_id', '0')
 100             cpt.set(section_name, 'range_size', mem_size)
 101             cpt.set(section_name, 'filename', mem_filename)
 102         elif re.search('.*sys.*\.\w*mem$', sec):
 103             # Due to the lack of information about a start address,
 104             # this migration only works if there is a single memory in
 105             # the system, thus starting at 0
 106             raise ValueError("more than one memory detected (" + sec + ")")
 107
 108 def from_2(cpt):
 109     for sec in cpt.sections():
 110         import re
 111         # Search for a CPUs
 112         if re.search('.*sys.*cpu', sec):
 113             try:
 114                 junk = cpt.get(sec, 'instCnt')
 115                 cpt.set(sec, '_pid', '0')
 116             except ConfigParser.NoOptionError:
 117                 pass
 118
 119 # The ISA is now a separate SimObject, which means that we serialize
 120 # it in a separate section instead of as a part of the ThreadContext.
 121 def from_3(cpt):
 122     isa = cpt.get('root','isa')
 123     isa_fields = {
 124         "alpha" : ( "fpcr", "uniq", "lock_flag", "lock_addr", "ipr" ),
 125         "arm" : ( "miscRegs" ),
 126         "sparc" : ( "asi", "tick", "fprs", "gsr", "softint", "tick_cmpr",
 127                     "stick", "stick_cmpr", "tpc", "tnpc", "tstate", "tt",
 128                     "tba", "pstate", "tl", "pil", "cwp", "gl", "hpstate",
 129                     "htstate", "hintp", "htba", "hstick_cmpr",
 130                     "strandStatusReg", "fsr", "priContext", "secContext",
 131                     "partId", "lsuCtrlReg", "scratchPad",
 132                     "cpu_mondo_head", "cpu_mondo_tail",
 133                     "dev_mondo_head", "dev_mondo_tail",
 134                     "res_error_head", "res_error_tail",
 135                     "nres_error_head", "nres_error_tail",
 136                     "tick_intr_sched",
 137                     "cpu", "tc_num", "tick_cmp", "stick_cmp", "hstick_cmp"),
 138         "x86" : ( "regVal" ),
 139         }
 140
 141     isa_fields = isa_fields.get(isa, [])
 142     isa_sections = []
 143     for sec in cpt.sections():
 144         import re
 145
 146         re_cpu_match = re.match('^(.*sys.*\.cpu[^.]*)\.xc\.(.+)$', sec)
 147         # Search for all the execution contexts
 148         if not re_cpu_match:
 149             continue
 150
 151         if re_cpu_match.group(2) != "0":
 152             # This shouldn't happen as we didn't support checkpointing
 153             # of in-order and O3 CPUs.
 154             raise ValueError("Don't know how to migrate multi-threaded CPUs "
 155                              "from version 1")
 156
 157         isa_section = []
 158         for fspec in isa_fields:
 159             for (key, value) in cpt.items(sec, raw=True):
 160                 if key in isa_fields:
 161                     isa_section.append((key, value))
 162
 163         name = "%s.isa" % re_cpu_match.group(1)
 164         isa_sections.append((name, isa_section))
 165
 166         for (key, value) in isa_section:
 167             cpt.remove_option(sec, key)
 168
 169     for (sec, options) in isa_sections:
 170         # Some intermediate versions of gem5 have empty ISA sections
 171         # (after we made the ISA a SimObject, but before we started to
 172         # serialize into a separate ISA section).
 173         if not cpt.has_section(sec):
 174             cpt.add_section(sec)
 175         else:
 176             if cpt.items(sec):
 177                 raise ValueError("Unexpected populated ISA section in old "
 178                                  "checkpoint")
 179
 180         for (key, value) in options:
 181             cpt.set(sec, key, value)
 182
 183 # Version 5 of the checkpoint format removes the MISCREG_CPSR_MODE
 184 # register from the ARM register file.
 185 def from_4(cpt):
 186     if cpt.get('root','isa') == 'arm':
 187         for sec in cpt.sections():
 188             import re
 189             # Search for all ISA sections
 190             if re.search('.*sys.*\.cpu.*\.isa', sec):
 191                 mr = cpt.get(sec, 'miscRegs').split()
 192                 # Remove MISCREG_CPSR_MODE
 193                 del mr[137]
 194                 cpt.set(sec, 'miscRegs', ' '.join(str(x) for x in mr))
 195
 196 # Version 6 of the checkpoint format adds tlb to x86 checkpoints
 197 def from_5(cpt):
 198     if cpt.get('root','isa') == 'x86':
 199         for sec in cpt.sections():
 200             import re
 201             # Search for all ISA sections
 202             if re.search('.*sys.*\.cpu.*\.dtb$', sec):
 203                 cpt.set(sec, '_size', '0')
 204                 cpt.set(sec, 'lruSeq', '0')
 205
 206             if re.search('.*sys.*\.cpu.*\.itb$', sec):
 207                 cpt.set(sec, '_size', '0')
 208                 cpt.set(sec, 'lruSeq', '0')
 209     else:
 210         print "ISA is not x86"
 211
 212 # Version 7 of the checkpoint adds support for the IDE dmaAbort flag
 213 def from_6(cpt):
 214     # Update IDE disk devices with dmaAborted
 215     for sec in cpt.sections():
 216         # curSector only exists in IDE devices, so key on that attribute
 217         if cpt.has_option(sec, "curSector"):
 218             cpt.set(sec, "dmaAborted", "false")
 219
 220
 221 migrations = []
 222 migrations.append(from_0)
 223 migrations.append(from_1)
 224 migrations.append(from_2)
 225 migrations.append(from_3)
 226 migrations.append(from_4)
 227 migrations.append(from_5)
 228 migrations.append(from_6)
 229
 230 verbose_print = False
 231
 232 def verboseprint(*args):
 233     if not verbose_print:
 234         return
 235     for arg in args:
 236         print arg,
 237     print
 238
 239 def process_file(path, **kwargs):
 240     if not osp.isfile(path):
 241         import errno
 242         raise IOError(ennro.ENOENT, "No such file", path)
 243
 244     verboseprint("Processing file %s...." % path)
 245
 246     if kwargs.get('backup', True):
 247         import shutil
 248         shutil.copyfile(path, path + '.bak')
 249
 250     cpt = ConfigParser.SafeConfigParser()
 251
 252     # gem5 is case sensitive with paramaters
 253     cpt.optionxform = str
 254
 255     # Read the current data
 256     cpt_file = file(path, 'r')
 257     cpt.readfp(cpt_file)
 258     cpt_file.close()
 259
 260     # Make sure we know what we're starting from
 261     if not cpt.has_option('root','cpt_ver'):
 262         raise LookupError("cannot determine version of checkpoint")
 263
 264     cpt_ver = cpt.getint('root','cpt_ver')
 265
 266     # If the current checkpoint is longer than the migrations list, we have a problem
 267     # and someone didn't update this file
 268     if cpt_ver > len(migrations):
 269         raise ValueError("upgrade script is too old and needs updating")
 270
 271     verboseprint("\t...file is at version %#x" % cpt_ver)
 272
 273     if cpt_ver == len(migrations):
 274         verboseprint("\t...nothing to do")
 275         return
 276
 277     # Walk through every function from now until the end fixing the checkpoint
 278     for v in xrange(cpt_ver,len(migrations)):
 279         verboseprint("\t...migrating to version %#x" %  (v + 1))
 280         migrations[v](cpt)
 281         cpt.set('root','cpt_ver', str(v + 1))
 282
 283     # Write the old data back
 284     verboseprint("\t...completed")
 285     cpt.write(file(path, 'w'))
 286
 287 if __name__ == '__main__':
 288     from optparse import OptionParser
 289     parser = OptionParser("usage: %prog [options] <filename or directory>")
 290     parser.add_option("-r", "--recurse", action="store_true",
 291                       help="Recurse through all subdirectories modifying "\
 292                            "each checkpoint that is found")
 293     parser.add_option("-N", "--no-backup", action="store_false",
 294                       dest="backup", default=True,
 295                       help="Do no backup each checkpoint before modifying it")
 296     parser.add_option("-v", "--verbose", action="store_true",
 297                       help="Print out debugging information as")
 298
 299     (options, args) = parser.parse_args()
 300     if len(args) != 1:
 301         parser.error("You must specify a checkpoint file to modify or a "\
 302                      "directory of checkpoints to recursively update")
 303
 304     verbose_print = options.verbose
 305
 306     # Deal with shell variables and ~
 307     path = osp.expandvars(osp.expanduser(args[0]))
 308
 309     # Process a single file if we have it
 310     if osp.isfile(path):
 311         process_file(path, **vars(options))
 312     # Process an entire directory
 313     elif osp.isdir(path):
 314         cpt_file = osp.join(path, 'm5.cpt')
 315         if options.recurse:
 316             # Visit very file and see if it matches
 317             for root,dirs,files in os.walk(path):
 318                 for name in files:
 319                     if name == 'm5.cpt':
 320                         process_file(osp.join(root,name), **vars(options))
 321                 for dir in dirs:
 322                     pass
 323         # Maybe someone passed a cpt.XXXXXXX directory and not m5.cpt
 324         elif osp.isfile(cpt_file):
 325             process_file(cpt_file, **vars(options))
 326         else:
 327             print "Error: checkpoint file not found at in %s " % path,
 328             print "and recurse not specified"
 329             sys.exit(1)
 330     sys.exit(0)
 331