util/checkpoint_aggregator.py

   1 # Copyright (c) 2009 The Regents of The University of Michigan
   2 # Copyright (c) 2011 Advanced Micro Devices, Inc.
   3 # Copyright (c) 2013 Mark D. Hill and David A. Wood
   4 # All rights reserved.
   5 #
   6 # Redistribution and use in source and binary forms, with or without
   7 # modification, are permitted provided that the following conditions are
   8 # met: redistributions of source code must retain the above copyright
   9 # notice, this list of conditions and the following disclaimer;
  10 # redistributions in binary form must reproduce the above copyright
  11 # notice, this list of conditions and the following disclaimer in the
  12 # documentation and/or other materials provided with the distribution;
  13 # neither the name of the copyright holders nor the names of its
  14 # contributors may be used to endorse or promote products derived from
  15 # this software without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28 #
  29 # Authors: Lisa Hsu
  30 #          Nilay Vaish
  31
  32 from ConfigParser import ConfigParser
  33 import gzip
  34
  35 import sys, re, os
  36
  37 class myCP(ConfigParser):
  38     def __init__(self):
  39         ConfigParser.__init__(self)
  40
  41     def optionxform(self, optionstr):
  42         return optionstr
  43
  44 def aggregate(output_dir, cpts, no_compress, memory_size):
  45     merged_config = None
  46     page_ptr = 0
  47
  48     output_path = output_dir
  49     if not os.path.isdir(output_path):
  50         os.system("mkdir -p " + output_path)
  51
  52     agg_mem_file = open(output_path + "/system.physmem.store0.pmem", "wb+")
  53     agg_config_file = open(output_path + "/m5.cpt", "wb+")
  54
  55     if not no_compress:
  56         merged_mem = gzip.GzipFile(fileobj= agg_mem_file, mode="wb")
  57
  58     max_curtick = 0
  59     num_digits = len(str(len(cpts)-1))
  60
  61     for (i, arg) in enumerate(cpts):
  62         print arg
  63         merged_config = myCP()
  64         config = myCP()
  65         config.readfp(open(cpts[i] + "/m5.cpt"))
  66
  67         for sec in config.sections():
  68             if re.compile("cpu").search(sec):
  69                 newsec = re.sub("cpu", "cpu" + str(i).zfill(num_digits), sec)
  70                 merged_config.add_section(newsec)
  71
  72                 items = config.items(sec)
  73                 for item in items:
  74                     if item[0] == "paddr":
  75                         merged_config.set(newsec, item[0], int(item[1]) + (page_ptr << 12))
  76                         continue
  77                     merged_config.set(newsec, item[0], item[1])
  78
  79                 if re.compile("workload.FdMap256$").search(sec):
  80                     merged_config.set(newsec, "M5_pid", i)
  81
  82             elif sec == "system":
  83                 pass
  84             elif sec == "Globals":
  85                 tick = config.getint(sec, "curTick")
  86                 if tick > max_curtick:
  87                     max_curtick = tick
  88             else:
  89                 if i == len(cpts)-1:
  90                     merged_config.add_section(sec)
  91                     for item in config.items(sec):
  92                         merged_config.set(sec, item[0], item[1])
  93
  94         if i != len(cpts)-1:
  95             merged_config.write(agg_config_file)
  96
  97         ### memory stuff
  98         pages = int(config.get("system", "pagePtr"))
  99         page_ptr = page_ptr + pages
 100         print "pages to be read: ", pages
 101
 102         f = open(cpts[i] + "/system.physmem.store0.pmem", "rb")
 103         gf = gzip.GzipFile(fileobj=f, mode="rb")
 104
 105         x = 0
 106         while x < pages:
 107             bytesRead = gf.read(1 << 12)
 108             if not no_compress:
 109                 merged_mem.write(bytesRead)
 110             else:
 111                 agg_mem_file.write(bytesRead)
 112             x += 1
 113
 114         gf.close()
 115         f.close()
 116
 117     merged_config.add_section("system")
 118     merged_config.set("system", "pagePtr", page_ptr)
 119     merged_config.set("system", "nextPID", len(cpts))
 120
 121     file_size = page_ptr * 4 * 1024
 122     dummy_data = "".zfill(4096)
 123     while file_size < memory_size:
 124         if not no_compress:
 125             merged_mem.write(dummy_data)
 126         else:
 127             agg_mem_file.write(dummy_data)
 128         file_size += 4 * 1024
 129         page_ptr += 1
 130
 131     print "WARNING: "
 132     print "Make sure the simulation using this checkpoint has at least ",
 133     print page_ptr, "x 4K of memory"
 134     merged_config.set("system.physmem.store0", "range_size", page_ptr * 4 * 1024)
 135
 136     merged_config.add_section("Globals")
 137     merged_config.set("Globals", "curTick", max_curtick)
 138
 139     merged_config.write(agg_config_file)
 140
 141     if not no_compress:
 142         merged_mem.close()
 143         agg_mem_file.close()
 144     else:
 145         agg_mem_file.close()
 146
 147 if __name__ == "__main__":
 148     from argparse import ArgumentParser
 149     parser = ArgumentParser("usage: %prog [options] <directory names which "\
 150                             "hold the checkpoints to be combined>")
 151     parser.add_argument("-o", "--output-dir", action="store",
 152                         help="Output directory")
 153     parser.add_argument("-c", "--no-compress", action="store_true")
 154     parser.add_argument("--cpts", nargs='+')
 155     parser.add_argument("--memory-size", action="store", type=int)
 156
 157     # Assume x86 ISA.  Any other ISAs would need extra stuff in this script
 158     # to appropriately parse their page tables and understand page sizes.
 159     options = parser.parse_args()
 160     print options.cpts, len(options.cpts)
 161     if len(options.cpts) <= 1:
 162         parser.error("You must specify atleast two checkpoint files that "\
 163                      "need to be combined.")
 164
 165     aggregate(options.output_dir, options.cpts, options.no_compress,
 166               options.memory_size)