util/checkpoint_aggregator.py

   1 # Copyright (c) 2009 The Regents of The University of Michigan
   2 # All rights reserved.
   3 #
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met: redistributions of source code must retain the above copyright
   7 # notice, this list of conditions and the following disclaimer;
   8 # redistributions in binary form must reproduce the above copyright
   9 # notice, this list of conditions and the following disclaimer in the
  10 # documentation and/or other materials provided with the distribution;
  11 # neither the name of the copyright holders nor the names of its
  12 # contributors may be used to endorse or promote products derived from
  13 # this software without specific prior written permission.
  14 #
  15 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  16 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  17 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  18 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  19 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  20 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  21 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  22 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  23 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  24 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  25 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26 #
  27 # Authors: Lisa Hsu
  28
  29 from ConfigParser import ConfigParser
  30 import gzip
  31
  32 import sys, re, optparse, os
  33
  34 class myCP(ConfigParser):
  35     def __init__(self):
  36         ConfigParser.__init__(self)
  37
  38     def optionxform(self, optionstr):
  39         return optionstr
  40
  41 def aggregate(options, args):
  42     merged = myCP()
  43     page_ptr = 0
  44
  45     allfiles = os.listdir(os.getcwd())
  46     cpts = []
  47     for arg in args:
  48         found = False
  49         for f in allfiles:
  50             if re.compile("cpt." + arg + ".\d+").search(f):
  51                 found = True
  52                 cpts.append(f)
  53                 break
  54         if not found:
  55             print "missing checkpoint: ", arg
  56             sys.exit(1)
  57
  58     dirname = "-".join([options.prefix, "cpt"])
  59     agg_name = "-".join(args)
  60     print agg_name
  61     fullpath = os.path.join("..", dirname, "cpt." + agg_name + ".10000")
  62     if not os.path.isdir(fullpath):
  63         os.system("mkdir -p " + fullpath)
  64     elif os.path.isfile(fullpath + "/system.physmem.physmem"):
  65         if os.path.isfile(fullpath + "/m5.cpt"):
  66             print fullpath, " already done"
  67             return
  68
  69     myfile = open(fullpath + "/system.physmem.physmem", "wb+")
  70     merged_mem = gzip.GzipFile(fileobj=myfile, mode="wb")
  71
  72     max_curtick = 0
  73     when = 0
  74     for (i, arg) in enumerate(args):
  75         print arg
  76         config = myCP()
  77         config.readfp(open(cpts[i] + "/m5.cpt"))
  78
  79         for sec in config.sections():
  80             if re.compile("cpu").search(sec):
  81                 newsec = re.sub("cpu", "cpu" + str(i), sec)
  82                 merged.add_section(newsec)
  83                 if re.compile("workload$").search(sec):
  84                     merged.set(newsec, "M5_pid", i)
  85
  86                 items = config.items(sec)
  87                 for item in items:
  88                     if item[0] == "ppn":
  89                         if config.getint(sec, "tag") != 0:
  90                             merged.set(newsec, item[0], int(item[1]) + page_ptr)
  91                             continue
  92                     elif item[0] == "asn":
  93                         tmp = re.compile("(.*).Entry(\d+)").search(sec).groups()
  94                         if config.has_option(tmp[0], "nlu"):
  95                             size = config.getint(tmp[0], "nlu")
  96                             if int(tmp[1]) < size:
  97                                 merged.set(newsec, item[0], i)
  98                                 continue
  99                         else:
 100                             merged.set(newsec, item[0], i)
 101                             continue
 102                     merged.set(newsec, item[0], item[1])
 103             elif sec == "system":
 104                 pass
 105             elif sec == "Globals":
 106                 tick = config.getint(sec, "curTick")
 107                 if tick > max_curtick:
 108                     max_curtick = tick
 109                     when = config.getint("system.cpu.tickEvent", "_when")
 110             else:
 111                 if i == 0:
 112                     merged.add_section(sec)
 113                     for item in config.items(sec):
 114                         merged.set(sec, item[0], item[1])
 115                         if item[0] == "curtick":
 116                             merged.optionxform(str("curTick"))
 117                         elif item[0] == "numevents":
 118                             merged.optionxform(str("numEvents"))
 119
 120         page_ptr = page_ptr + int(config.get("system", "page_ptr"))
 121
 122         ### memory stuff
 123         f = open(cpts[i] + "/system.physmem.physmem", "rb")
 124         gf = gzip.GzipFile(fileobj=f, mode="rb")
 125         pages = int(config.get("system", "page_ptr"))
 126         print "pages to be read: ", pages
 127
 128         x = 0
 129         while x < pages:
 130             bytesRead = gf.read(1 << 13)
 131             merged_mem.write(bytesRead)
 132             x += 1
 133
 134         gf.close()
 135         f.close()
 136
 137     merged.add_section("system")
 138     merged.set("system", "page_ptr", page_ptr)
 139     print "WARNING: "
 140     print "Make sure the simulation using this checkpoint has at least "
 141     if page_ptr > (1<<20):
 142         print "8G ",
 143     elif page_ptr > (1<<19):
 144         print "4G ",
 145     elif page_ptr > (1<<18):
 146         print "2G ",
 147     elif page_ptr > (1<<17):
 148         print "1G ",
 149     elif page_ptr > (1<<16):
 150         print "512KB ",
 151     else:
 152         print "this is a small sim, you're probably fine",
 153     print "of memory."
 154
 155     merged.add_section("Globals")
 156     merged.set("Globals", "curTick", max_curtick)
 157
 158     for i in xrange(len(args)):
 159         merged.set("system.cpu" + str(i) + ".tickEvent", "_when", when)
 160
 161     merged.write(file(fullpath + "/m5.cpt", "wb"))
 162     merged_mem.close()
 163     myfile.close()
 164
 165 if __name__ == "__main__":
 166
 167     parser = optparse.OptionParser()
 168     parser.add_option("--prefix", type="string", default="agg")
 169
 170     (options, args) = parser.parse_args()
 171
 172     aggregate(options, args)
 173