util/cpt_upgrader.py

   1 #!/usr/bin/env python3
   2
   3 # Copyright (c) 2012-2013,2015-2016, 2020 ARM Limited
   4 # All rights reserved
   5 #
   6 # The license below extends only to copyright in the software and shall
   7 # not be construed as granting a license to any other intellectual
   8 # property including but not limited to intellectual property relating
   9 # to a hardware implementation of the functionality of the software
  10 # licensed hereunder.  You may use the software subject to the license
  11 # terms below provided that you ensure that this notice is replicated
  12 # unmodified and in its entirety in all distributions of the software,
  13 # modified or unmodified, in source code or in binary form.
  14 #
  15 # Redistribution and use in source and binary forms, with or without
  16 # modification, are permitted provided that the following conditions are
  17 # met: redistributions of source code must retain the above copyright
  18 # notice, this list of conditions and the following disclaimer;
  19 # redistributions in binary form must reproduce the above copyright
  20 # notice, this list of conditions and the following disclaimer in the
  21 # documentation and/or other materials provided with the distribution;
  22 # neither the name of the copyright holders nor the names of its
  23 # contributors may be used to endorse or promote products derived from
  24 # this software without specific prior written permission.
  25 #
  26 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  27 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  28 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  29 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  30 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  31 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  32 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  33 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  34 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  35 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  36 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  37
  38 # This python code is used to migrate checkpoints that were created in one
  39 # version of the simulator to newer version. As features are added or bugs are
  40 # fixed some of the state that needs to be checkpointed can change. If you have
  41 # many historic checkpoints that you use, manually editing them to fix them is
  42 # both time consuming and error-prone.
  43
  44 # This script provides a way to migrate checkpoints to the newer repository in
  45 # a programmatic way. It can be imported into another script or used on the
  46 # command line. From the command line the script will either migrate every
  47 # checkpoint it finds recursively (-r option) or a single checkpoint. When a
  48 # change is made to the gem5 repository that breaks previous checkpoints an
  49 # upgrade() method should be implemented in its own .py file and placed in
  50 # src/util/cpt_upgraders/.  For each upgrader whose tag is not present in
  51 # the checkpoint tag list, the upgrade() method will be run, passing in a
  52 # ConfigParser object which contains the open file. As these operations can
  53 # be isa specific the method can verify the isa and use regexes to find the
  54 # correct sections that need to be updated.
  55
  56 # It is also possible to use this mechanism to revert prior tags.  In this
  57 # case, implement a downgrade() method instead.  Dependencies should still
  58 # work naturally - a tag depending on a tag with a downgrader means that it
  59 # insists on the other tag being removed and its downgrader executed before
  60 # its upgrader (or downgrader) can run.  It is still the case that a tag
  61 # can only be used once.
  62
  63 # Dependencies between tags are expressed by two variables at the top-level
  64 # of the upgrader script: "depends" can be either a string naming another
  65 # tag that it depends upon or a list of such strings; and "fwd_depends"
  66 # accepts the same datatypes but it reverses the sense of the dependency
  67 # arrow(s) -- it expresses that that tag depends upon the tag of the current
  68 # upgrader. This can be especially valuable when maintaining private
  69 # upgraders in private branches.
  70
  71
  72
  73 from six.moves import configparser
  74 import glob, types, sys, os
  75 import os.path as osp
  76
  77 verbose_print = False
  78
  79 def verboseprint(*args):
  80     if not verbose_print:
  81         return
  82     for arg in args:
  83         print(arg, end=' ')
  84     print("\n")
  85
  86 class Upgrader:
  87     tag_set = set()
  88     untag_set = set() # tags to remove by downgrading
  89     by_tag = {}
  90     legacy = {}
  91     def __init__(self, filename):
  92         self.filename = filename
  93         exec(open(filename).read(), {}, self.__dict__)
  94
  95         if not hasattr(self, 'tag'):
  96             self.tag = osp.basename(filename)[:-3]
  97         if not hasattr(self, 'depends'):
  98             self.depends = []
  99         elif isinstance(self.depends, str):
 100             self.depends = [self.depends]
 101
 102         if not isinstance(self.depends, list):
 103             print("Error: 'depends' for {} is the wrong type".format(self.tag))
 104             sys.exit(1)
 105
 106         if hasattr(self, 'fwd_depends'):
 107             if isinstance(self.fwd_depends, str):
 108                 self.fwd_depends = [self.fwd_depends]
 109         else:
 110             self.fwd_depends = []
 111
 112         if not isinstance(self.fwd_depends, list):
 113             print("Error: 'fwd_depends' for {} is the wrong type".format(
 114                 self.tag))
 115             sys.exit(1)
 116
 117         if hasattr(self, 'upgrader'):
 118             if not isinstance(self.upgrader, types.FunctionType):
 119                 print("Error: 'upgrader' for {} is {}, not function".format(
 120                     self.tag, type(self)))
 121                 sys.exit(1)
 122             Upgrader.tag_set.add(self.tag)
 123         elif hasattr(self, 'downgrader'):
 124             if not isinstance(self.downgrader, types.FunctionType):
 125                 print("Error: 'downgrader' for {} is {}, not function".format(
 126                     self.tag, type(self)))
 127                 sys.exit(1)
 128             Upgrader.untag_set.add(self.tag)
 129         else:
 130             print("Error: no upgrader or downgrader method for".format(
 131                 self.tag))
 132             sys.exit(1)
 133
 134         if hasattr(self, 'legacy_version'):
 135             Upgrader.legacy[self.legacy_version] = self
 136
 137         Upgrader.by_tag[self.tag] = self
 138
 139     def ready(self, tags):
 140         for dep in self.depends:
 141             if dep not in tags:
 142                 return False
 143         return True
 144
 145     def update(self, cpt, tags):
 146         if hasattr(self, 'upgrader'):
 147             self.upgrader(cpt)
 148             tags.add(self.tag)
 149             verboseprint("applied upgrade for", self.tag)
 150         else:
 151             self.downgrader(cpt)
 152             tags.remove(self.tag)
 153             verboseprint("applied downgrade for", self.tag)
 154
 155     @staticmethod
 156     def get(tag):
 157         return Upgrader.by_tag[tag]
 158
 159     @staticmethod
 160     def load_all():
 161         util_dir = osp.dirname(osp.abspath(__file__))
 162
 163         for py in glob.glob(util_dir + '/cpt_upgraders/*.py'):
 164             Upgrader(py)
 165
 166         # make linear dependences for legacy versions
 167         i = 3
 168         while i in Upgrader.legacy:
 169             Upgrader.legacy[i].depends = [Upgrader.legacy[i-1].tag]
 170             i = i + 1
 171
 172         # resolve forward dependencies and audit normal dependencies
 173         for tag, upg in list(Upgrader.by_tag.items()):
 174             for fd in upg.fwd_depends:
 175                 if fd not in Upgrader.by_tag:
 176                     print("Error: '{}' cannot (forward) depend on "
 177                           "nonexistent tag '{}'".format(fd, tag))
 178                     sys.exit(1)
 179                 Upgrader.by_tag[fd].depends.append(tag)
 180             for dep in upg.depends:
 181                 if dep not in Upgrader.by_tag:
 182                     print("Error: '{}' cannot depend on "
 183                           "nonexistent tag '{}'".format(tag, dep))
 184                     sys.exit(1)
 185
 186 def process_file(path, **kwargs):
 187     if not osp.isfile(path):
 188         import errno
 189         raise IOError(ennro.ENOENT, "No such file", path)
 190
 191     verboseprint("Processing file %s...." % path)
 192
 193     if kwargs.get('backup', True):
 194         import shutil
 195         shutil.copyfile(path, path + '.bak')
 196
 197     cpt = configparser.SafeConfigParser()
 198
 199     # gem5 is case sensitive with paramaters
 200     cpt.optionxform = str
 201
 202     # Read the current data
 203     cpt_file = file(path, 'r')
 204     cpt.readfp(cpt_file)
 205     cpt_file.close()
 206
 207     change = False
 208
 209     # Make sure we know what we're starting from
 210     if cpt.has_option('root','cpt_ver'):
 211         cpt_ver = cpt.getint('root','cpt_ver')
 212
 213         # Legacy linear checkpoint version
 214         # convert to list of tags before proceeding
 215         tags = set([])
 216         for i in range(2, cpt_ver+1):
 217             tags.add(Upgrader.legacy[i].tag)
 218         verboseprint("performed legacy version -> tags conversion")
 219         change = True
 220
 221         cpt.remove_option('root', 'cpt_ver')
 222     elif cpt.has_option('Globals','version_tags'):
 223         tags = set((''.join(cpt.get('Globals','version_tags'))).split())
 224     else:
 225         print("fatal: no version information in checkpoint")
 226         exit(1)
 227
 228     verboseprint("has tags", ' '.join(tags))
 229     # If the current checkpoint has a tag we don't know about, we have
 230     # a divergence that (in general) must be addressed by (e.g.) merging
 231     # simulator support for its changes.
 232     unknown_tags = tags - (Upgrader.tag_set | Upgrader.untag_set)
 233     if unknown_tags:
 234         print("warning: upgrade script does not recognize the following "
 235               "tags in this checkpoint:", ' '.join(unknown_tags))
 236
 237     # Apply migrations for tags not in checkpoint and tags present for which
 238     # downgraders are present, respecting dependences
 239     to_apply = (Upgrader.tag_set - tags) | (Upgrader.untag_set & tags)
 240     while to_apply:
 241         ready = set([ t for t in to_apply if Upgrader.get(t).ready(tags) ])
 242         if not ready:
 243             print("could not apply these upgrades:", ' '.join(to_apply))
 244             print("update dependences impossible to resolve; aborting")
 245             exit(1)
 246
 247         for tag in ready:
 248             Upgrader.get(tag).update(cpt, tags)
 249             change = True
 250
 251         to_apply -= ready
 252
 253     if not change:
 254         verboseprint("...nothing to do")
 255         return
 256
 257     cpt.set('Globals', 'version_tags', ' '.join(tags))
 258
 259     # Write the old data back
 260     verboseprint("...completed")
 261     cpt.write(file(path, 'w'))
 262
 263 if __name__ == '__main__':
 264     from optparse import OptionParser, SUPPRESS_HELP
 265     parser = OptionParser("usage: %prog [options] <filename or directory>")
 266     parser.add_option("-r", "--recurse", action="store_true",
 267                       help="Recurse through all subdirectories modifying "\
 268                            "each checkpoint that is found")
 269     parser.add_option("-N", "--no-backup", action="store_false",
 270                       dest="backup", default=True,
 271                       help="Do no backup each checkpoint before modifying it")
 272     parser.add_option("-v", "--verbose", action="store_true",
 273                       help="Print out debugging information as")
 274     parser.add_option("--get-cc-file", action="store_true",
 275                       # used during build; generate src/sim/tags.cc and exit
 276                       help=SUPPRESS_HELP)
 277
 278     (options, args) = parser.parse_args()
 279     verbose_print = options.verbose
 280
 281     Upgrader.load_all()
 282
 283     if options.get_cc_file:
 284         print("// this file is auto-generated by util/cpt_upgrader.py")
 285         print("#include <string>")
 286         print("#include <set>")
 287         print()
 288         print("std::set<std::string> version_tags = {")
 289         for tag in Upgrader.tag_set:
 290             print("  \"{}\",".format(tag))
 291         print("};")
 292         exit(0)
 293     elif len(args) != 1:
 294         parser.error("You must specify a checkpoint file to modify or a "\
 295                      "directory of checkpoints to recursively update")
 296
 297     # Deal with shell variables and ~
 298     path = osp.expandvars(osp.expanduser(args[0]))
 299
 300     # Process a single file if we have it
 301     if osp.isfile(path):
 302         process_file(path, **vars(options))
 303     # Process an entire directory
 304     elif osp.isdir(path):
 305         cpt_file = osp.join(path, 'm5.cpt')
 306         if options.recurse:
 307             # Visit very file and see if it matches
 308             for root,dirs,files in os.walk(path):
 309                 for name in files:
 310                     if name == 'm5.cpt':
 311                         process_file(osp.join(root,name), **vars(options))
 312                 for dir in dirs:
 313                     pass
 314         # Maybe someone passed a cpt.XXXXXXX directory and not m5.cpt
 315         elif osp.isfile(cpt_file):
 316             process_file(cpt_file, **vars(options))
 317         else:
 318             print("Error: checkpoint file not found in {} ".format(path))
 319             print("and recurse not specified")
 320             sys.exit(1)
 321     sys.exit(0)
 322