From: Lisa Hsu Date: Thu, 3 Jun 2010 17:34:40 +0000 (-0700) Subject: utils: checkpoint aggregator: some physmem files are too big to read at once, X-Git-Tag: stable_2012_02_02~1050 X-Git-Url: https://git.libre-soc.org/?a=commitdiff_plain;h=aeb6e2e3ec920f8590f36bdb33c03c2968ab5dde;p=gem5.git utils: checkpoint aggregator: some physmem files are too big to read at once, break it up into reading one page at a time. Also, avoid redoing a aggregating a checkpoint that's already done. --HG-- rename : util/checkpoint-aggregator.py => util/checkpoint_aggregator.py --- diff --git a/util/checkpoint-aggregator.py b/util/checkpoint-aggregator.py deleted file mode 100755 index 6e40db01e..000000000 --- a/util/checkpoint-aggregator.py +++ /dev/null @@ -1,167 +0,0 @@ -# Copyright (c) 2009 The Regents of The University of Michigan -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer; -# redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution; -# neither the name of the copyright holders nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Authors: Lisa Hsu - -from ConfigParser import ConfigParser -import gzip - -import sys, re, optparse, os - -class myCP(ConfigParser): - def __init__(self): - ConfigParser.__init__(self) - - def optionxform(self, optionstr): - return optionstr - -def aggregate(options, args): - merged = myCP() - page_ptr = 0 - - allfiles = os.listdir(os.getcwd()) - cpts = [] - for arg in args: - found = False - for f in allfiles: - if re.compile("cpt." + arg + ".\d+").search(f): - found = True - cpts.append(f) - break - if not found: - print "missing checkpoint: ", arg - sys.exit(1) - - dirname = "-".join([options.prefix, "cpt"]) - print dirname - agg_name = "-".join(args) - print agg_name - fullpath = os.path.join("..", dirname, "cpt." + agg_name + ".10000") - if not os.path.isdir(fullpath): - os.system("mkdir -p " + fullpath) - - myfile = open(fullpath + "/system.physmem.physmem", "wb+") - merged_mem = gzip.GzipFile(fileobj=myfile, mode="wb") - - max_curtick = 0 - when = 0 - for (i, arg) in enumerate(args): - config = myCP() - config.readfp(open(cpts[i] + "/m5.cpt")) - - for sec in config.sections(): - if re.compile("cpu").search(sec): - newsec = re.sub("cpu", "cpu" + str(i), sec) - merged.add_section(newsec) - if re.compile("workload$").search(sec): - merged.set(newsec, "M5_pid", i) - - items = config.items(sec) - for item in items: - if item[0] == "ppn": - if config.getint(sec, "tag") != 0: - merged.set(newsec, item[0], int(item[1]) + page_ptr) - continue - elif item[0] == "asn": - tmp = re.compile("(.*).Entry(\d+)").search(sec).groups() - if config.has_option(tmp[0], "nlu"): - size = config.getint(tmp[0], "nlu") - if int(tmp[1]) < size: - merged.set(newsec, item[0], i) - continue - else: - merged.set(newsec, item[0], i) - continue - merged.set(newsec, item[0], item[1]) - elif sec == "system": - pass - elif sec == "Globals": - tick = config.getint(sec, "curTick") - if tick > max_curtick: - max_curtick = tick - when = config.getint("system.cpu.tickEvent", "_when") - else: - if i == 0: - print sec - merged.add_section(sec) - for item in config.items(sec): - merged.set(sec, item[0], item[1]) - if item[0] == "curtick": - merged.optionxform(str("curTick")) - elif item[0] == "numevents": - merged.optionxform(str("numEvents")) - - page_ptr = page_ptr + int(config.get("system", "page_ptr")) - - ### memory stuff - f = open(cpts[i] + "/system.physmem.physmem", "rb") - gf = gzip.GzipFile(fileobj=f, mode="rb") - bytes = int(config.get("system", "page_ptr")) << 13 - print "bytes to be read: ", bytes - - bytesRead = gf.read(int(config.get("system", "page_ptr")) << 13) - merged_mem.write(bytesRead) - - gf.close() - f.close() - - merged.add_section("system") - merged.set("system", "page_ptr", page_ptr) - print "WARNING: " - print "Make sure the simulation using this checkpoint has at least " - if page_ptr > (1<<20): - print "8G ", - elif page_ptr > (1<<19): - print "4G ", - elif page_ptr > (1<<18): - print "2G ", - elif page_ptr > (1<<17): - print "1G ", - elif page_ptr > (1<<16): - print "512KB ", - else: - print "this is a small sim, you're probably fine", - print "of memory." - - merged.add_section("Globals") - merged.set("Globals", "curTick", max_curtick) - - for i in xrange(len(args)): - merged.set("system.cpu" + str(i) + ".tickEvent", "_when", when) - - merged.write(file(fullpath + "/m5.cpt", "wb")) - merged_mem.close() - myfile.close() - -if __name__ == "__main__": - - parser = optparse.OptionParser() - parser.add_option("--prefix", type="string", default="agg") - - (options, args) = parser.parse_args() - - aggregate(options, args) - diff --git a/util/checkpoint_aggregator.py b/util/checkpoint_aggregator.py new file mode 100755 index 000000000..acfbca0d0 --- /dev/null +++ b/util/checkpoint_aggregator.py @@ -0,0 +1,173 @@ +# Copyright (c) 2009 The Regents of The University of Michigan +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Lisa Hsu + +from ConfigParser import ConfigParser +import gzip + +import sys, re, optparse, os + +class myCP(ConfigParser): + def __init__(self): + ConfigParser.__init__(self) + + def optionxform(self, optionstr): + return optionstr + +def aggregate(options, args): + merged = myCP() + page_ptr = 0 + + allfiles = os.listdir(os.getcwd()) + cpts = [] + for arg in args: + found = False + for f in allfiles: + if re.compile("cpt." + arg + ".\d+").search(f): + found = True + cpts.append(f) + break + if not found: + print "missing checkpoint: ", arg + sys.exit(1) + + dirname = "-".join([options.prefix, "cpt"]) + agg_name = "-".join(args) + print agg_name + fullpath = os.path.join("..", dirname, "cpt." + agg_name + ".10000") + if not os.path.isdir(fullpath): + os.system("mkdir -p " + fullpath) + elif os.path.isfile(fullpath + "/system.physmem.physmem"): + if os.path.isfile(fullpath + "/m5.cpt"): + print fullpath, " already done" + return + + myfile = open(fullpath + "/system.physmem.physmem", "wb+") + merged_mem = gzip.GzipFile(fileobj=myfile, mode="wb") + + max_curtick = 0 + when = 0 + for (i, arg) in enumerate(args): + print arg + config = myCP() + config.readfp(open(cpts[i] + "/m5.cpt")) + + for sec in config.sections(): + if re.compile("cpu").search(sec): + newsec = re.sub("cpu", "cpu" + str(i), sec) + merged.add_section(newsec) + if re.compile("workload$").search(sec): + merged.set(newsec, "M5_pid", i) + + items = config.items(sec) + for item in items: + if item[0] == "ppn": + if config.getint(sec, "tag") != 0: + merged.set(newsec, item[0], int(item[1]) + page_ptr) + continue + elif item[0] == "asn": + tmp = re.compile("(.*).Entry(\d+)").search(sec).groups() + if config.has_option(tmp[0], "nlu"): + size = config.getint(tmp[0], "nlu") + if int(tmp[1]) < size: + merged.set(newsec, item[0], i) + continue + else: + merged.set(newsec, item[0], i) + continue + merged.set(newsec, item[0], item[1]) + elif sec == "system": + pass + elif sec == "Globals": + tick = config.getint(sec, "curTick") + if tick > max_curtick: + max_curtick = tick + when = config.getint("system.cpu.tickEvent", "_when") + else: + if i == 0: + merged.add_section(sec) + for item in config.items(sec): + merged.set(sec, item[0], item[1]) + if item[0] == "curtick": + merged.optionxform(str("curTick")) + elif item[0] == "numevents": + merged.optionxform(str("numEvents")) + + page_ptr = page_ptr + int(config.get("system", "page_ptr")) + + ### memory stuff + f = open(cpts[i] + "/system.physmem.physmem", "rb") + gf = gzip.GzipFile(fileobj=f, mode="rb") + pages = int(config.get("system", "page_ptr")) + print "pages to be read: ", pages + + x = 0 + while x < pages: + bytesRead = gf.read(1 << 13) + merged_mem.write(bytesRead) + x += 1 + + gf.close() + f.close() + + merged.add_section("system") + merged.set("system", "page_ptr", page_ptr) + print "WARNING: " + print "Make sure the simulation using this checkpoint has at least " + if page_ptr > (1<<20): + print "8G ", + elif page_ptr > (1<<19): + print "4G ", + elif page_ptr > (1<<18): + print "2G ", + elif page_ptr > (1<<17): + print "1G ", + elif page_ptr > (1<<16): + print "512KB ", + else: + print "this is a small sim, you're probably fine", + print "of memory." + + merged.add_section("Globals") + merged.set("Globals", "curTick", max_curtick) + + for i in xrange(len(args)): + merged.set("system.cpu" + str(i) + ".tickEvent", "_when", when) + + merged.write(file(fullpath + "/m5.cpt", "wb")) + merged_mem.close() + myfile.close() + +if __name__ == "__main__": + + parser = optparse.OptionParser() + parser.add_option("--prefix", type="string", default="agg") + + (options, args) = parser.parse_args() + + aggregate(options, args) +