util/checkpoint_aggregator.py

   1 # Copyright (c) 2009 The Regents of The University of Michigan
   2 # Copyright (c) 2011 Advanced Micro Devices, Inc.
   3 # Copyright (c) 2013 Mark D. Hill and David A. Wood
   4 # All rights reserved.
   5 #
   6 # Redistribution and use in source and binary forms, with or without
   7 # modification, are permitted provided that the following conditions are
   8 # met: redistributions of source code must retain the above copyright
   9 # notice, this list of conditions and the following disclaimer;
  10 # redistributions in binary form must reproduce the above copyright
  11 # notice, this list of conditions and the following disclaimer in the
  12 # documentation and/or other materials provided with the distribution;
  13 # neither the name of the copyright holders nor the names of its
  14 # contributors may be used to endorse or promote products derived from
  15 # this software without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 from configparser import ConfigParser
  30 import gzip
  31
  32 import sys, re, os
  33
  34 class myCP(ConfigParser):
  35     def __init__(self):
  36         ConfigParser.__init__(self)
  37
  38     def optionxform(self, optionstr):
  39         return optionstr
  40
  41 def aggregate(output_dir, cpts, no_compress, memory_size):
  42     merged_config = None
  43     page_ptr = 0
  44
  45     output_path = output_dir
  46     if not os.path.isdir(output_path):
  47         os.system("mkdir -p " + output_path)
  48
  49     agg_mem_file = open(output_path + "/system.physmem.store0.pmem", "wb+")
  50     agg_config_file = open(output_path + "/m5.cpt", "wb+")
  51
  52     if not no_compress:
  53         merged_mem = gzip.GzipFile(fileobj= agg_mem_file, mode="wb")
  54
  55     max_curtick = 0
  56     num_digits = len(str(len(cpts)-1))
  57
  58     for (i, arg) in enumerate(cpts):
  59         print(arg)
  60         merged_config = myCP()
  61         config = myCP()
  62         config.readfp(open(cpts[i] + "/m5.cpt"))
  63
  64         for sec in config.sections():
  65             if re.compile("cpu").search(sec):
  66                 newsec = re.sub("cpu", "cpu" + str(i).zfill(num_digits), sec)
  67                 merged_config.add_section(newsec)
  68
  69                 items = config.items(sec)
  70                 for item in items:
  71                     if item[0] == "paddr":
  72                         merged_config.set(newsec, item[0], int(item[1]) + (page_ptr << 12))
  73                         continue
  74                     merged_config.set(newsec, item[0], item[1])
  75
  76                 if re.compile("workload.FdMap256$").search(sec):
  77                     merged_config.set(newsec, "M5_pid", i)
  78
  79             elif sec == "system":
  80                 pass
  81             elif sec == "Globals":
  82                 tick = config.getint(sec, "curTick")
  83                 if tick > max_curtick:
  84                     max_curtick = tick
  85             else:
  86                 if i == len(cpts)-1:
  87                     merged_config.add_section(sec)
  88                     for item in config.items(sec):
  89                         merged_config.set(sec, item[0], item[1])
  90
  91         if i != len(cpts)-1:
  92             merged_config.write(agg_config_file)
  93
  94         ### memory stuff
  95         pages = int(config.get("system", "pagePtr"))
  96         page_ptr = page_ptr + pages
  97         print("pages to be read: ", pages)
  98
  99         f = open(cpts[i] + "/system.physmem.store0.pmem", "rb")
 100         gf = gzip.GzipFile(fileobj=f, mode="rb")
 101
 102         x = 0
 103         while x < pages:
 104             bytesRead = gf.read(1 << 12)
 105             if not no_compress:
 106                 merged_mem.write(bytesRead)
 107             else:
 108                 agg_mem_file.write(bytesRead)
 109             x += 1
 110
 111         gf.close()
 112         f.close()
 113
 114     merged_config.add_section("system")
 115     merged_config.set("system", "pagePtr", page_ptr)
 116     merged_config.set("system", "nextPID", len(cpts))
 117
 118     file_size = page_ptr * 4 * 1024
 119     dummy_data = "".zfill(4096)
 120     while file_size < memory_size:
 121         if not no_compress:
 122             merged_mem.write(dummy_data)
 123         else:
 124             agg_mem_file.write(dummy_data)
 125         file_size += 4 * 1024
 126         page_ptr += 1
 127
 128     print("WARNING: ")
 129     print("Make sure the simulation using this checkpoint has at least ", end=' ')
 130     print(page_ptr, "x 4K of memory")
 131     merged_config.set("system.physmem.store0", "range_size", page_ptr * 4 * 1024)
 132
 133     merged_config.add_section("Globals")
 134     merged_config.set("Globals", "curTick", max_curtick)
 135
 136     merged_config.write(agg_config_file)
 137
 138     if not no_compress:
 139         merged_mem.close()
 140         agg_mem_file.close()
 141     else:
 142         agg_mem_file.close()
 143
 144 if __name__ == "__main__":
 145     from argparse import ArgumentParser
 146     parser = ArgumentParser("usage: %prog [options] <directory names which "\
 147                             "hold the checkpoints to be combined>")
 148     parser.add_argument("-o", "--output-dir", action="store",
 149                         help="Output directory")
 150     parser.add_argument("-c", "--no-compress", action="store_true")
 151     parser.add_argument("--cpts", nargs='+')
 152     parser.add_argument("--memory-size", action="store", type=int)
 153
 154     # Assume x86 ISA.  Any other ISAs would need extra stuff in this script
 155     # to appropriately parse their page tables and understand page sizes.
 156     options = parser.parse_args()
 157     print(options.cpts, len(options.cpts))
 158     if len(options.cpts) <= 1:
 159         parser.error("You must specify atleast two checkpoint files that "\
 160                      "need to be combined.")
 161
 162     aggregate(options.output_dir, options.cpts, options.no_compress,
 163               options.memory_size)