From 8f1ca0a212ede0ecac1199027681f3c2ca9308db Mon Sep 17 00:00:00 2001 From: Radhika Jagtap Date: Mon, 7 Dec 2015 16:42:15 -0600 Subject: [PATCH] util: Add decode and encode scripts for elastic traces This patch adds python scripts to parse a protobuf encoded O3CPU elastic trace and convert it to a text file output and vice versa. --- util/decode_inst_dep_trace.py | 200 ++++++++++++++++++++++++++++++++++ util/encode_inst_dep_trace.py | 189 ++++++++++++++++++++++++++++++++ 2 files changed, 389 insertions(+) create mode 100755 util/decode_inst_dep_trace.py create mode 100755 util/encode_inst_dep_trace.py diff --git a/util/decode_inst_dep_trace.py b/util/decode_inst_dep_trace.py new file mode 100755 index 000000000..63c958d6d --- /dev/null +++ b/util/decode_inst_dep_trace.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python + +# Copyright (c) 2013 - 2015 ARM Limited +# All rights reserved +# +# The license below extends only to copyright in the software and shall +# not be construed as granting a license to any other intellectual +# property including but not limited to intellectual property relating +# to a hardware implementation of the functionality of the software +# licensed hereunder. You may use the software subject to the license +# terms below provided that you ensure that this notice is replicated +# unmodified and in its entirety in all distributions of the software, +# modified or unmodified, in source code or in binary form. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Copyright 2008 Google Inc. All rights reserved. +# http://code.google.com/p/protobuf/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Radhika Jagtap +# + +# This script is used to dump protobuf traces of the instruction dependency +# graph to ASCII format. +# +# The ASCII trace format uses one line per instruction with the format +# instruction sequence number, (optional) pc, (optional) weight, load, store, +# (optional) flags, (optional) addr, (optional) size, comp delay, +# (repeated) order dependencies comma-separated, and (repeated) register +# dependencies comma-separated. +# +# examples: +# seq_num,[pc],[weight,]load,store,[address,size,flags,]comp_delay:[rob_dep]: +# [reg_dep] +# 1,1,False,False,8500:: +# 2,1,False,False,1000:,1: +# 3,1,True,False,831248,4,74,500:,2: +# 4,1,False,False,0:,2: +# 5,1,False,False,500::,4 +# 6,1,False,True,831248,4,74,1000:,3:,4,5 + +import protolib +import sys + +# Import the packet proto definitions. If they are not found, attempt +# to generate them automatically. This assumes that the script is +# executed from the gem5 root. +try: + import inst_dep_record_pb2 +except: + print "Did not find proto definition, attempting to generate" + from subprocess import call + error = call(['protoc', '--python_out=util', '--proto_path=src/proto', + 'src/proto/inst_dep_record.proto']) + if not error: + import inst_dep_record_pb2 + print "Generated proto definitions for instruction dependency record" + else: + print "Failed to import proto definitions" + exit(-1) + +def main(): + if len(sys.argv) != 3: + print "Usage: ", sys.argv[0], " " + exit(-1) + + # Open the file on read mode + proto_in = protolib.openFileRd(sys.argv[1]) + + try: + ascii_out = open(sys.argv[2], 'w') + except IOError: + print "Failed to open ", sys.argv[2], " for writing" + exit(-1) + + # Read the magic number in 4-byte Little Endian + magic_number = proto_in.read(4) + + if magic_number != "gem5": + print "Unrecognized file" + exit(-1) + + print "Parsing packet header" + + # Add the packet header + header = inst_dep_record_pb2.InstDepRecordHeader() + protolib.decodeMessage(proto_in, header) + + print "Object id:", header.obj_id + print "Tick frequency:", header.tick_freq + + print "Parsing packets" + + num_packets = 0 + num_regdeps = 0 + num_robdeps = 0 + packet = inst_dep_record_pb2.InstDepRecord() + + # Decode the packet messages until we hit the end of the file + while protolib.decodeMessage(proto_in, packet): + num_packets += 1 + + # Write to file the seq num + ascii_out.write('%s' % (packet.seq_num)) + # Write to file the pc of the instruction, default is 0 + if packet.HasField('pc'): + ascii_out.write(',%s' % (packet.pc)) + else: + ascii_out.write(',0') + # Write to file the weight, default is 1 + if packet.HasField('weight'): + ascii_out.write(',%s' % (packet.weight)) + else: + ascii_out.write(',1') + # Write to file if it is a load and if it is a store + ascii_out.write(',%s,%s' % (packet.load, packet.store)) + + # Write to file if it has the optional fields addr, size, flags + if packet.HasField('addr'): + ascii_out.write(',%s' % (packet.addr)) + if packet.HasField('size'): + ascii_out.write(',%s' % (packet.size)) + if packet.HasField('flags'): + ascii_out.write(',%s' % (packet.flags)) + + # Write to file the comp delay + ascii_out.write(',%s' % (packet.comp_delay)) + + # Write to file the repeated field order dependency + ascii_out.write(':') + if packet.rob_dep: + num_robdeps += 1 + for dep in packet.rob_dep: + ascii_out.write(',%s' % dep) + # Write to file the repeated field register dependency + ascii_out.write(':') + if packet.reg_dep: + num_regdeps += 1 # No. of packets with atleast 1 register dependency + for dep in packet.reg_dep: + ascii_out.write(',%s' % dep) + # New line + ascii_out.write('\n') + + print "Parsed packets:", num_packets + print "Packets with at least 1 reg dep:", num_regdeps + print "Packets with at least 1 rob dep:", num_robdeps + + # We're done + ascii_out.close() + proto_in.close() + +if __name__ == "__main__": + main() diff --git a/util/encode_inst_dep_trace.py b/util/encode_inst_dep_trace.py new file mode 100755 index 000000000..9e2b1613f --- /dev/null +++ b/util/encode_inst_dep_trace.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python + +# Copyright (c) 2015 ARM Limited +# All rights reserved +# +# The license below extends only to copyright in the software and shall +# not be construed as granting a license to any other intellectual +# property including but not limited to intellectual property relating +# to a hardware implementation of the functionality of the software +# licensed hereunder. You may use the software subject to the license +# terms below provided that you ensure that this notice is replicated +# unmodified and in its entirety in all distributions of the software, +# modified or unmodified, in source code or in binary form. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Copyright 2008 Google Inc. All rights reserved. +# http://code.google.com/p/protobuf/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Radhika Jagtap +# + +# This script is used to dump ASCII traces of the instruction dependency +# graph to protobuf format. +# +# The ASCII trace format uses one line per instruction with the format +# instruction sequence number, (optional) pc, (optional) weight, load, store, +# (optional) flags, (optional) addr, (optional) size, comp delay, +# (repeated) order dependencies comma-separated, and (repeated) register +# dependencies comma-separated. +# +# examples: +# seq_num,[pc],[weight,]load,store,[address,size,flags,]comp_delay:[rob_dep]: +# [reg_dep] +# 1,1,False,False,8500:: +# 2,1,False,False,1000:,1: +# 3,1,True,False,831248,4,74,500:,2: +# 4,1,False,False,0:,2: +# 5,1,False,False,500::,4 +# 6,1,False,True,831248,4,74,1000:,3:,4,5 + +import protolib +import sys + +# Import the packet proto definitions. If they are not found, attempt +# to generate them automatically. This assumes that the script is +# executed from the gem5 root. +try: + import inst_dep_record_pb2 +except: + print "Did not find proto definition, attempting to generate" + from subprocess import call + error = call(['protoc', '--python_out=util', '--proto_path=src/proto', + 'src/proto/inst_dep_record.proto']) + if not error: + import inst_dep_record_pb2 + print "Generated proto definitions for instruction dependency record" + else: + print "Failed to import proto definitions" + exit(-1) + +def main(): + if len(sys.argv) != 3: + print "Usage: ", sys.argv[0], " " + exit(-1) + + # Open the file in write mode + proto_out = open(sys.argv[2], 'wb') + + # Open the file in read mode + try: + ascii_in = open(sys.argv[1], 'r') + except IOError: + print "Failed to open ", sys.argv[1], " for reading" + exit(-1) + + # Write the magic number in 4-byte Little Endian, similar to what + # is done in src/proto/protoio.cc + proto_out.write("gem5") + + # Add the packet header + header = inst_dep_record_pb2.InstDepRecordHeader() + header.obj_id = "Converted ASCII trace " + sys.argv[1] + # Assume the default tick rate + header.tick_freq = 1000000000 + header.window_size = 120 + protolib.encodeMessage(proto_out, header) + + num_records = 0 + # For each line in the ASCII trace, create a packet message and + # write it to the encoded output + for line in ascii_in: + inst_info_str, rob_dep_str, reg_dep_str = (line.strip()).split(':') + inst_info_list = inst_info_str.split(',') + dep_record = inst_dep_record_pb2.InstDepRecord() + + dep_record.seq_num = long(inst_info_list[0]) + dep_record.pc = long(inst_info_list[1]) + dep_record.weight = long(inst_info_list[2]) + dep_record.load = True if inst_info_list[3] == 'True' else False + dep_record.store = True if inst_info_list[4] == 'True' else False + + # If the instruction is a load or store record the addr, size flags + # in addition to recording the computation delay + if dep_record.load or dep_record.store: + addr, size, flags, comp_delay = inst_info_list[5:9] + dep_record.addr = long(addr) + dep_record.size = int(size) + dep_record.flags = int(flags) + dep_record.comp_delay = long(comp_delay) + elif not dep_record.load and not dep_record.store: + comp_delay = inst_info_list[4] + dep_record.comp_delay = long(comp_delay) + else: + print "Fatal:", seq_num, "is both load and store" + exit(1) + + # Parse the register and order dependencies both of which are + # repeated fields. An empty list is valid. + rob_deps = rob_dep_str.strip().split(',') + for a_dep in rob_deps: + # if the string is empty, split(',') returns 1 item: '' + # if the string is ",4", split(',') returns 2 items: '', '4' + # long('') gives error, so check if the item is non-empty + if a_dep: + dep_record.rob_dep.append(long(a_dep)) + + reg_deps = reg_dep_str.split(',') + for a_dep in reg_deps: + if a_dep: + dep_record.reg_dep.append(long(a_dep)) + + protolib.encodeMessage(proto_out, dep_record) + num_records += 1 + + print "Converted", num_records, "records." + # We're done + ascii_in.close() + proto_out.close() + +if __name__ == "__main__": + main() -- 2.30.2