3 # Copyright (c) 2012-2013,2015-2016, 2020 ARM Limited
6 # The license below extends only to copyright in the software and shall
7 # not be construed as granting a license to any other intellectual
8 # property including but not limited to intellectual property relating
9 # to a hardware implementation of the functionality of the software
10 # licensed hereunder. You may use the software subject to the license
11 # terms below provided that you ensure that this notice is replicated
12 # unmodified and in its entirety in all distributions of the software,
13 # modified or unmodified, in source code or in binary form.
15 # Redistribution and use in source and binary forms, with or without
16 # modification, are permitted provided that the following conditions are
17 # met: redistributions of source code must retain the above copyright
18 # notice, this list of conditions and the following disclaimer;
19 # redistributions in binary form must reproduce the above copyright
20 # notice, this list of conditions and the following disclaimer in the
21 # documentation and/or other materials provided with the distribution;
22 # neither the name of the copyright holders nor the names of its
23 # contributors may be used to endorse or promote products derived from
24 # this software without specific prior written permission.
26 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 # This python code is used to migrate checkpoints that were created in one
39 # version of the simulator to newer version. As features are added or bugs are
40 # fixed some of the state that needs to be checkpointed can change. If you have
41 # many historic checkpoints that you use, manually editing them to fix them is
42 # both time consuming and error-prone.
44 # This script provides a way to migrate checkpoints to the newer repository in
45 # a programmatic way. It can be imported into another script or used on the
46 # command line. From the command line the script will either migrate every
47 # checkpoint it finds recursively (-r option) or a single checkpoint. When a
48 # change is made to the gem5 repository that breaks previous checkpoints an
49 # upgrade() method should be implemented in its own .py file and placed in
50 # src/util/cpt_upgraders/. For each upgrader whose tag is not present in
51 # the checkpoint tag list, the upgrade() method will be run, passing in a
52 # ConfigParser object which contains the open file. As these operations can
53 # be isa specific the method can verify the isa and use regexes to find the
54 # correct sections that need to be updated.
56 # It is also possible to use this mechanism to revert prior tags. In this
57 # case, implement a downgrade() method instead. Dependencies should still
58 # work naturally - a tag depending on a tag with a downgrader means that it
59 # insists on the other tag being removed and its downgrader executed before
60 # its upgrader (or downgrader) can run. It is still the case that a tag
61 # can only be used once.
63 # Dependencies between tags are expressed by two variables at the top-level
64 # of the upgrader script: "depends" can be either a string naming another
65 # tag that it depends upon or a list of such strings; and "fwd_depends"
66 # accepts the same datatypes but it reverses the sense of the dependency
67 # arrow(s) -- it expresses that that tag depends upon the tag of the current
68 # upgrader. This can be especially valuable when maintaining private
69 # upgraders in private branches.
71 from __future__
import print_function
73 from six
.moves
import configparser
74 import glob
, types
, sys
, os
79 def verboseprint(*args
):
88 untag_set
= set() # tags to remove by downgrading
91 def __init__(self
, filename
):
92 self
.filename
= filename
93 exec(open(filename
).read(), {}, self
.__dict
__)
95 if not hasattr(self
, 'tag'):
96 self
.tag
= osp
.basename(filename
)[:-3]
97 if not hasattr(self
, 'depends'):
99 elif isinstance(self
.depends
, str):
100 self
.depends
= [self
.depends
]
102 if not isinstance(self
.depends
, list):
103 print("Error: 'depends' for {} is the wrong type".format(self
.tag
))
106 if hasattr(self
, 'fwd_depends'):
107 if isinstance(self
.fwd_depends
, str):
108 self
.fwd_depends
= [self
.fwd_depends
]
110 self
.fwd_depends
= []
112 if not isinstance(self
.fwd_depends
, list):
113 print("Error: 'fwd_depends' for {} is the wrong type".format(
117 if hasattr(self
, 'upgrader'):
118 if not isinstance(self
.upgrader
, types
.FunctionType
):
119 print("Error: 'upgrader' for {} is {}, not function".format(
120 self
.tag
, type(self
)))
122 Upgrader
.tag_set
.add(self
.tag
)
123 elif hasattr(self
, 'downgrader'):
124 if not isinstance(self
.downgrader
, types
.FunctionType
):
125 print("Error: 'downgrader' for {} is {}, not function".format(
126 self
.tag
, type(self
)))
128 Upgrader
.untag_set
.add(self
.tag
)
130 print("Error: no upgrader or downgrader method for".format(
134 if hasattr(self
, 'legacy_version'):
135 Upgrader
.legacy
[self
.legacy_version
] = self
137 Upgrader
.by_tag
[self
.tag
] = self
139 def ready(self
, tags
):
140 for dep
in self
.depends
:
145 def update(self
, cpt
, tags
):
146 if hasattr(self
, 'upgrader'):
149 verboseprint("applied upgrade for", self
.tag
)
152 tags
.remove(self
.tag
)
153 verboseprint("applied downgrade for", self
.tag
)
157 return Upgrader
.by_tag
[tag
]
161 util_dir
= osp
.dirname(osp
.abspath(__file__
))
163 for py
in glob
.glob(util_dir
+ '/cpt_upgraders/*.py'):
166 # make linear dependences for legacy versions
168 while i
in Upgrader
.legacy
:
169 Upgrader
.legacy
[i
].depends
= [Upgrader
.legacy
[i
-1].tag
]
172 # resolve forward dependencies and audit normal dependencies
173 for tag
, upg
in Upgrader
.by_tag
.items():
174 for fd
in upg
.fwd_depends
:
175 if fd
not in Upgrader
.by_tag
:
176 print("Error: '{}' cannot (forward) depend on "
177 "nonexistent tag '{}'".format(fd
, tag
))
179 Upgrader
.by_tag
[fd
].depends
.append(tag
)
180 for dep
in upg
.depends
:
181 if dep
not in Upgrader
.by_tag
:
182 print("Error: '{}' cannot depend on "
183 "nonexistent tag '{}'".format(tag
, dep
))
186 def process_file(path
, **kwargs
):
187 if not osp
.isfile(path
):
189 raise IOError(ennro
.ENOENT
, "No such file", path
)
191 verboseprint("Processing file %s...." % path
)
193 if kwargs
.get('backup', True):
195 shutil
.copyfile(path
, path
+ '.bak')
197 cpt
= configparser
.SafeConfigParser()
199 # gem5 is case sensitive with paramaters
200 cpt
.optionxform
= str
202 # Read the current data
203 cpt_file
= file(path
, 'r')
209 # Make sure we know what we're starting from
210 if cpt
.has_option('root','cpt_ver'):
211 cpt_ver
= cpt
.getint('root','cpt_ver')
213 # Legacy linear checkpoint version
214 # convert to list of tags before proceeding
216 for i
in xrange(2, cpt_ver
+1):
217 tags
.add(Upgrader
.legacy
[i
].tag
)
218 verboseprint("performed legacy version -> tags conversion")
221 cpt
.remove_option('root', 'cpt_ver')
222 elif cpt
.has_option('Globals','version_tags'):
223 tags
= set((''.join(cpt
.get('Globals','version_tags'))).split())
225 print("fatal: no version information in checkpoint")
228 verboseprint("has tags", ' '.join(tags
))
229 # If the current checkpoint has a tag we don't know about, we have
230 # a divergence that (in general) must be addressed by (e.g.) merging
231 # simulator support for its changes.
232 unknown_tags
= tags
- (Upgrader
.tag_set | Upgrader
.untag_set
)
234 print("warning: upgrade script does not recognize the following "
235 "tags in this checkpoint:", ' '.join(unknown_tags
))
237 # Apply migrations for tags not in checkpoint and tags present for which
238 # downgraders are present, respecting dependences
239 to_apply
= (Upgrader
.tag_set
- tags
) |
(Upgrader
.untag_set
& tags
)
241 ready
= set([ t
for t
in to_apply
if Upgrader
.get(t
).ready(tags
) ])
243 print("could not apply these upgrades:", ' '.join(to_apply
))
244 print("update dependences impossible to resolve; aborting")
248 Upgrader
.get(tag
).update(cpt
, tags
)
254 verboseprint("...nothing to do")
257 cpt
.set('Globals', 'version_tags', ' '.join(tags
))
259 # Write the old data back
260 verboseprint("...completed")
261 cpt
.write(file(path
, 'w'))
263 if __name__
== '__main__':
264 from optparse
import OptionParser
, SUPPRESS_HELP
265 parser
= OptionParser("usage: %prog [options] <filename or directory>")
266 parser
.add_option("-r", "--recurse", action
="store_true",
267 help="Recurse through all subdirectories modifying "\
268 "each checkpoint that is found")
269 parser
.add_option("-N", "--no-backup", action
="store_false",
270 dest
="backup", default
=True,
271 help="Do no backup each checkpoint before modifying it")
272 parser
.add_option("-v", "--verbose", action
="store_true",
273 help="Print out debugging information as")
274 parser
.add_option("--get-cc-file", action
="store_true",
275 # used during build; generate src/sim/tags.cc and exit
278 (options
, args
) = parser
.parse_args()
279 verbose_print
= options
.verbose
283 if options
.get_cc_file
:
284 print("// this file is auto-generated by util/cpt_upgrader.py")
285 print("#include <string>")
286 print("#include <set>")
288 print("std::set<std::string> version_tags = {")
289 for tag
in Upgrader
.tag_set
:
290 print(" \"{}\",".format(tag
))
294 parser
.error("You must specify a checkpoint file to modify or a "\
295 "directory of checkpoints to recursively update")
297 # Deal with shell variables and ~
298 path
= osp
.expandvars(osp
.expanduser(args
[0]))
300 # Process a single file if we have it
302 process_file(path
, **vars(options
))
303 # Process an entire directory
304 elif osp
.isdir(path
):
305 cpt_file
= osp
.join(path
, 'm5.cpt')
307 # Visit very file and see if it matches
308 for root
,dirs
,files
in os
.walk(path
):
311 process_file(osp
.join(root
,name
), **vars(options
))
314 # Maybe someone passed a cpt.XXXXXXX directory and not m5.cpt
315 elif osp
.isfile(cpt_file
):
316 process_file(cpt_file
, **vars(options
))
318 print("Error: checkpoint file not found in {} ".format(path
))
319 print("and recurse not specified")