--- /dev/null
+#!/usr/bin/env python
+
+# Copyright (c) 2012 ARM Limited
+# All rights reserved
+#
+# The license below extends only to copyright in the software and shall
+# not be construed as granting a license to any other intellectual
+# property including but not limited to intellectual property relating
+# to a hardware implementation of the functionality of the software
+# licensed hereunder. You may use the software subject to the license
+# terms below provided that you ensure that this notice is replicated
+# unmodified and in its entirety in all distributions of the software,
+# modified or unmodified, in source code or in binary form.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Author: Dam Sunwoo
+#
+
+# This script converts gem5 output to ARM DS-5 Streamline .apc project file
+# (Requires the gem5 runs to be run with ContextSwitchStatsDump enabled and
+# some patches applied to target Linux kernel.)
+# Visit http://www.gem5.org/Streamline for more details.
+#
+# Usage:
+# m5stats2streamline.py <stat_config.ini> <gem5 run folder> <dest .apc folder>
+#
+# <stat_config.ini>: .ini file that describes which stats to be included
+# in conversion. Sample .ini files can be found in
+# util/streamline.
+# NOTE: this is NOT the gem5 config.ini file.
+#
+# <gem5 run folder>: Path to gem5 run folder (must contain config.ini,
+# stats.txt[.gz], and system.tasks.txt.)
+#
+# <dest .apc folder>: Destination .apc folder path
+#
+# APC project generation based on Gator v12 (DS-5 v5.13)
+# Subsequent versions should be backward compatible
+
+import re, sys, os
+from ConfigParser import ConfigParser
+import gzip
+import xml.etree.ElementTree as ET
+import xml.dom.minidom as minidom
+import shutil
+import zlib
+
+import argparse
+
+parser = argparse.ArgumentParser(
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ description="""
+ Converts gem5 runs to ARM DS-5 Streamline .apc project file.
+ (NOTE: Requires gem5 runs to be run with ContextSwitchStatsDump
+ enabled and some patches applied to the target Linux kernel.)
+
+ Visit http://www.gem5.org/Streamline for more details.
+
+ APC project generation based on Gator v12 (DS-5 v5.13)
+ Subsequent versions should be backward compatible
+ """)
+
+parser.add_argument("stat_config_file", metavar="<stat_config.ini>",
+ help=".ini file that describes which stats to be included \
+ in conversion. Sample .ini files can be found in \
+ util/streamline. NOTE: this is NOT the gem5 config.ini \
+ file.")
+
+parser.add_argument("input_path", metavar="<gem5 run folder>",
+ help="Path to gem5 run folder (must contain config.ini, \
+ stats.txt[.gz], and system.tasks.txt.)")
+
+parser.add_argument("output_path", metavar="<dest .apc folder>",
+ help="Destination .apc folder path")
+
+parser.add_argument("--num-events", action="store", type=int,
+ default=1000000,
+ help="Maximum number of scheduling (context switch) \
+ events to be processed. Set to truncate early. \
+ Default=1000000")
+
+parser.add_argument("--gzipped-bmp-not-supported", action="store_true",
+ help="Do not use gzipped .bmp files for visual annotations. \
+ This option is only required when using Streamline versions \
+ older than 5.14")
+
+parser.add_argument("--verbose", action="store_true",
+ help="Enable verbose output")
+
+args = parser.parse_args()
+
+if not re.match("(.*)\.apc", args.output_path):
+ print "ERROR: <dest .apc folder> should end with '.apc'!"
+ sys.exit(1)
+
+# gzipped BMP files for visual annotation is supported in Streamline 5.14.
+# Setting this to True will significantly compress the .apc binary file that
+# includes frame buffer snapshots.
+gzipped_bmp_supported = not args.gzipped_bmp_not_supported
+
+ticks_in_ns = -1
+
+# Default max # of events. Increase this for longer runs.
+num_events = args.num_events
+
+start_tick = -1
+end_tick = -1
+
+# Parse gem5 config.ini file to determine some system configurations.
+# Number of CPUs, L2s, etc.
+def parseConfig(config_file):
+ global num_cpus, num_l2
+
+ print "\n==============================="
+ print "Parsing gem5 config.ini file..."
+ print config_file
+ print "===============================\n"
+ config = ConfigParser()
+ if not config.read(config_file):
+ print "ERROR: config file '", config_file, "' not found"
+ sys.exit(1)
+
+ if config.has_section("system.cpu"):
+ num_cpus = 1
+ else:
+ num_cpus = 0
+ while config.has_section("system.cpu" + str(num_cpus)):
+ num_cpus += 1
+
+ if config.has_section("system.l2"):
+ num_l2 = 1
+ else:
+ num_l2 = 0
+ while config.has_section("system.l2" + str(num_l2)):
+ num_l2 += 1
+
+ print "Num CPUs:", num_cpus
+ print "Num L2s:", num_l2
+ print ""
+
+ return (num_cpus, num_l2)
+
+
+process_dict = {}
+thread_dict = {}
+
+process_list = []
+
+idle_uid = -1
+kernel_uid = -1
+
+class Task(object):
+ def __init__(self, uid, pid, tgid, task_name, is_process, tick):
+ if pid == 0: # Idle
+ self.uid = 0
+ elif pid == -1: # Kernel
+ self.uid = 0
+ else:
+ self.uid = uid
+ self.pid = pid
+ self.tgid = tgid
+ self.is_process = is_process
+ self.task_name = task_name
+ self.children = []
+ self.tick = tick # time this task first appeared
+
+class Event(object):
+ def __init__(self, tick, task):
+ self.tick = tick
+ self.task = task
+
+############################################################
+# Types used in APC Protocol
+# - packed32, packed64
+# - int32
+# - string
+############################################################
+
+# variable length packed 4-byte signed value
+def packed32(x):
+ ret = []
+ if ((x & 0xffffff80) == 0):
+ ret.append(x & 0x7f)
+ elif ((x & 0xffffc000) == 0):
+ ret.append((x | 0x80) & 0xff)
+ ret.append((x >> 7) & 0x7f)
+ elif ((x & 0xffe00000) == 0):
+ ret.append((x | 0x80) & 0xff)
+ ret.append(((x >> 7) | 0x80) & 0xff)
+ ret.append((x >> 14) & 0x7f)
+ elif ((x & 0xf0000000) == 0):
+ ret.append((x | 0x80) & 0xff)
+ ret.append(((x >> 7) | 0x80) & 0xff)
+ ret.append(((x >> 14) | 0x80) & 0xff)
+ ret.append((x >> 21) & 0x7f)
+ else:
+ ret.append((x | 0x80) & 0xff)
+ ret.append(((x >> 7) | 0x80) & 0xff)
+ ret.append(((x >> 14) | 0x80) & 0xff)
+ ret.append(((x >> 21) | 0x80) & 0xff)
+ ret.append((x >> 28) & 0x0f)
+ return ret
+
+# variable length packed 8-byte signed value
+def packed64(x):
+ ret = []
+ if ((x & 0xffffffffffffff80) == 0):
+ ret.append(x & 0x7f)
+ elif ((x & 0xffffffffffffc000) == 0):
+ ret.append((x | 0x80) & 0xff)
+ ret.append((x >> 7) & 0x7f)
+ elif ((x & 0xffffffffffe00000) == 0):
+ ret.append((x | 0x80) & 0xff)
+ ret.append(((x >> 7) | 0x80) & 0xff)
+ ret.append((x >> 14) & 0x7f)
+ elif ((x & 0xfffffffff0000000) == 0):
+ ret.append((x | 0x80) & 0xff)
+ ret.append(((x >> 7) | 0x80) & 0xff)
+ ret.append(((x >> 14) | 0x80) & 0xff)
+ ret.append((x >> 21) & 0x7f)
+ elif ((x & 0xfffffff800000000) == 0):
+ ret.append((x | 0x80) & 0xff)
+ ret.append(((x >> 7) | 0x80) & 0xff)
+ ret.append(((x >> 14) | 0x80) & 0xff)
+ ret.append(((x >> 21) | 0x80) & 0xff)
+ ret.append((x >> 28) & 0x7f)
+ elif ((x & 0xfffffc0000000000) == 0):
+ ret.append((x | 0x80) & 0xff)
+ ret.append(((x >> 7) | 0x80) & 0xff)
+ ret.append(((x >> 14) | 0x80) & 0xff)
+ ret.append(((x >> 21) | 0x80) & 0xff)
+ ret.append(((x >> 28) | 0x80) & 0xff)
+ ret.append((x >> 35) & 0x7f)
+ elif ((x & 0xfffe000000000000) == 0):
+ ret.append((x | 0x80) & 0xff)
+ ret.append(((x >> 7) | 0x80) & 0xff)
+ ret.append(((x >> 14) | 0x80) & 0xff)
+ ret.append(((x >> 21) | 0x80) & 0xff)
+ ret.append(((x >> 28) | 0x80) & 0xff)
+ ret.append(((x >> 35) | 0x80) & 0xff)
+ ret.append((x >> 42) & 0x7f)
+ elif ((x & 0xff00000000000000) == 0):
+ ret.append((x | 0x80) & 0xff)
+ ret.append(((x >> 7) | 0x80) & 0xff)
+ ret.append(((x >> 14) | 0x80) & 0xff)
+ ret.append(((x >> 21) | 0x80) & 0xff)
+ ret.append(((x >> 28) | 0x80) & 0xff)
+ ret.append(((x >> 35) | 0x80) & 0xff)
+ ret.append(((x >> 42) | 0x80) & 0xff)
+ ret.append((x >> 49) & 0x7f)
+ elif ((x & 0x8000000000000000) == 0):
+ ret.append((x | 0x80) & 0xff)
+ ret.append(((x >> 7) | 0x80) & 0xff)
+ ret.append(((x >> 14) | 0x80) & 0xff)
+ ret.append(((x >> 21) | 0x80) & 0xff)
+ ret.append(((x >> 28) | 0x80) & 0xff)
+ ret.append(((x >> 35) | 0x80) & 0xff)
+ ret.append(((x >> 42) | 0x80) & 0xff)
+ ret.append(((x >> 49) | 0x80) & 0xff)
+ ret.append((x >> 56) & 0x7f)
+ else:
+ ret.append((x | 0x80) & 0xff)
+ ret.append(((x >> 7) | 0x80) & 0xff)
+ ret.append(((x >> 14) | 0x80) & 0xff)
+ ret.append(((x >> 21) | 0x80) & 0xff)
+ ret.append(((x >> 28) | 0x80) & 0xff)
+ ret.append(((x >> 35) | 0x80) & 0xff)
+ ret.append(((x >> 42) | 0x80) & 0xff)
+ ret.append(((x >> 49) | 0x80) & 0xff)
+ ret.append(((x >> 56) | 0x80) & 0xff)
+ ret.append((x >> 63) & 0x7f)
+ return ret
+
+# 4-byte signed little endian
+def int32(x):
+ ret = []
+ ret.append(x & 0xff)
+ ret.append((x >> 8) & 0xff)
+ ret.append((x >> 16) & 0xff)
+ ret.append((x >> 24) & 0xff)
+ return ret
+
+# 2-byte signed little endian
+def int16(x):
+ ret = []
+ ret.append(x & 0xff)
+ ret.append((x >> 8) & 0xff)
+ return ret
+
+# a packed32 length followed by the specified number of characters
+def stringList(x):
+ ret = []
+ ret += packed32(len(x))
+ for i in x:
+ ret.append(i)
+ return ret
+
+def utf8StringList(x):
+ ret = []
+ for i in x:
+ ret.append(ord(i))
+ return ret
+
+# packed64 time value in nanoseconds relative to the uptime from the
+# Summary message.
+def timestampList(x):
+ ret = packed64(x)
+ return ret
+
+
+############################################################
+# Write binary
+############################################################
+
+def writeBinary(outfile, binary_list):
+ for i in binary_list:
+ outfile.write("%c" % i)
+
+############################################################
+# APC Protocol Frame Types
+############################################################
+
+def addFrameHeader(frame_type, body, core):
+ ret = []
+
+ if frame_type == "Summary":
+ code = 1
+ elif frame_type == "Backtrace":
+ code = 2
+ elif frame_type == "Name":
+ code = 3
+ elif frame_type == "Counter":
+ code = 4
+ elif frame_type == "Block Counter":
+ code = 5
+ elif frame_type == "Annotate":
+ code = 6
+ elif frame_type == "Sched Trace":
+ code = 7
+ elif frame_type == "GPU Trace":
+ code = 8
+ elif frame_type == "Idle":
+ code = 9
+ else:
+ print "ERROR: Unknown frame type:", frame_type
+ sys.exit(1)
+
+ packed_code = packed32(code)
+
+ packed_core = packed32(core)
+
+ length = int32(len(packed_code) + len(packed_core) + len(body))
+
+ ret = length + packed_code + packed_core + body
+ return ret
+
+
+# Summary frame
+# - timestamp: packed64
+# - uptime: packed64
+def summaryFrame(timestamp, uptime):
+ frame_type = "Summary"
+ body = packed64(timestamp) + packed64(uptime)
+ ret = addFrameHeader(frame_type, body, 0)
+ return ret
+
+# Backtrace frame
+# - not implemented yet
+def backtraceFrame():
+ pass
+
+# Cookie name message
+# - cookie: packed32
+# - name: string
+def cookieNameFrame(cookie, name):
+ frame_type = "Name"
+ packed_code = packed32(1)
+ body = packed_code + packed32(cookie) + stringList(name)
+ ret = addFrameHeader(frame_type, body, 0)
+ return ret
+
+# Thread name message
+# - timestamp: timestamp
+# - thread id: packed32
+# - name: string
+def threadNameFrame(timestamp, thread_id, name):
+ frame_type = "Name"
+ packed_code = packed32(2)
+ body = packed_code + timestampList(timestamp) + \
+ packed32(thread_id) + stringList(name)
+ ret = addFrameHeader(frame_type, body, 0)
+ return ret
+
+# Core name message
+# - name: string
+def coreNameFrame(name):
+ frame_type = "Name"
+ packed_code = packed32(3)
+ body = packed_code + stringList(name)
+ ret = addFrameHeader(frame_type, body, 0)
+ return ret
+
+# Counter frame message
+# - timestamp: timestamp
+# - core: packed32
+# - key: packed32
+# - value: packed64
+def counterFrame(timestamp, core, key, value):
+ frame_type = "Counter"
+ body = timestampList(timestamp) + packed32(core) + packed32(key) + \
+ packed64(value)
+ ret = addFrameHeader(frame_type, body, core)
+ return ret
+
+# Block Counter frame message
+# - key: packed32
+# - value: packed64
+def blockCounterFrame(core, key, value):
+ frame_type = "Block Counter"
+ body = packed32(key) + packed64(value)
+ ret = addFrameHeader(frame_type, body, core)
+ return ret
+
+# Annotate frame messages
+# - core: packed32
+# - tid: packed32
+# - timestamp: timestamp
+# - size: packed32
+# - body
+def annotateFrame(core, tid, timestamp, size, userspace_body):
+ frame_type = "Annotate"
+ body = packed32(core) + packed32(tid) + timestampList(timestamp) + \
+ packed32(size) + userspace_body
+ ret = addFrameHeader(frame_type, body, core)
+ return ret
+
+# Scheduler Trace frame messages
+# Sched Switch
+# - Code: 1
+# - timestamp: timestamp
+# - pid: packed32
+# - tid: packed32
+# - cookie: packed32
+# - state: packed32
+def schedSwitchFrame(core, timestamp, pid, tid, cookie, state):
+ frame_type = "Sched Trace"
+ body = packed32(1) + timestampList(timestamp) + packed32(pid) + \
+ packed32(tid) + packed32(cookie) + packed32(state)
+ ret = addFrameHeader(frame_type, body, core)
+ return ret
+
+# Sched Thread Exit
+# - Code: 2
+# - timestamp: timestamp
+# - tid: packed32
+def schedThreadExitFrame(core, timestamp, pid, tid, cookie, state):
+ frame_type = "Sched Trace"
+ body = packed32(2) + timestampList(timestamp) + packed32(tid)
+ ret = addFrameHeader(frame_type, body, core)
+ return ret
+
+# GPU Trace frame messages
+# - Not implemented yet
+def gpuTraceFrame():
+ pass
+
+# Idle frame messages
+# Enter Idle
+# - code: 1
+# - timestamp: timestamp
+# - core: packed32
+def enterIdleFrame(timestamp, core):
+ frame_type = "Idle"
+ body = packed32(1) + timestampList(timestamp) + packed32(core)
+ ret = addFrameHeader(frame_type, body, core)
+ return ret
+
+# Exit Idle
+# - code: 2
+# - timestamp: timestamp
+# - core: packed32
+def exitIdleFrame(timestamp, core):
+ frame_type = "Idle"
+ body = packed32(2) + timestampList(timestamp) + packed32(core)
+ ret = addFrameHeader(frame_type, body, core)
+ return ret
+
+
+####################################################################
+def parseProcessInfo(task_file):
+ print "\n==============================="
+ print "Parsing Task file..."
+ print task_file
+ print "===============================\n"
+
+ global start_tick, end_tick, num_cpus
+ global process_dict, thread_dict, process_list
+ global event_list, unified_event_list
+ global idle_uid, kernel_uid
+
+ event_list = []
+ unified_event_list = []
+ for cpu in range(num_cpus):
+ event_list.append([])
+
+ uid = 1 # uid 0 is reserved for idle
+
+ # Dummy Tasks for frame buffers and system diagrams
+ process = Task(uid, 9999, 9999, "framebuffer", True, 0)
+ process_list.append(process)
+ uid += 1
+ thread = Task(uid, 9999, 9999, "framebuffer", False, 0)
+ process.children.append(thread)
+ uid += 1
+ process = Task(uid, 9998, 9998, "System", True, 0)
+ process_list.append(process)
+ # if we don't find the real kernel, use this to keep things going
+ kernel_uid = uid
+ uid += 1
+ thread = Task(uid, 9998, 9998, "System", False, 0)
+ process.children.append(thread)
+ uid += 1
+
+ ext = os.path.splitext(task_file)[1]
+
+ try:
+ if ext == ".gz":
+ process_file = gzip.open(task_file, 'rb')
+ else:
+ process_file = open(task_file, 'rb')
+ except:
+ print "ERROR opening task file:", task_file
+ print "Make sure context switch task dumping is enabled in gem5."
+ sys.exit(1)
+
+ process_re = re.compile("tick=(\d+)\s+(\d+)\s+cpu_id=(\d+)\s+" +
+ "next_pid=([-\d]+)\s+next_tgid=([-\d]+)\s+next_task=(.*)")
+
+ task_name_failure_warned = False
+
+ for line in process_file:
+ match = re.match(process_re, line)
+ if match:
+ tick = int(match.group(1))
+ if (start_tick < 0):
+ start_tick = tick
+ cpu_id = int(match.group(3))
+ pid = int(match.group(4))
+ tgid = int(match.group(5))
+ task_name = match.group(6)
+
+ if not task_name_failure_warned:
+ if task_name == "FailureIn_curTaskName":
+ print "-------------------------------------------------"
+ print "WARNING: Task name not set correctly!"
+ print "Process/Thread info will not be displayed correctly"
+ print "Perhaps forgot to apply m5struct.patch to kernel?"
+ print "-------------------------------------------------"
+ task_name_failure_warned = True
+
+ if not tgid in process_dict:
+ if tgid == pid:
+ # new task is parent as well
+ if args.verbose:
+ print "new process", uid, pid, tgid, task_name
+ if tgid == 0:
+ # new process is the "idle" task
+ process = Task(uid, pid, tgid, "idle", True, tick)
+ idle_uid = 0
+ else:
+ process = Task(uid, pid, tgid, task_name, True, tick)
+ else:
+ if tgid == 0:
+ process = Task(uid, tgid, tgid, "idle", True, tick)
+ idle_uid = 0
+ else:
+ # parent process name not known yet
+ process = Task(uid, tgid, tgid, "_Unknown_", True, tick)
+ if tgid == -1: # kernel
+ kernel_uid = 0
+ uid += 1
+ process_dict[tgid] = process
+ process_list.append(process)
+ else:
+ if tgid == pid:
+ if process_dict[tgid].task_name == "_Unknown_":
+ if args.verbose:
+ print "new process", \
+ process_dict[tgid].uid, pid, tgid, task_name
+ process_dict[tgid].task_name = task_name
+ if process_dict[tgid].task_name != task_name and tgid != 0:
+ process_dict[tgid].task_name = task_name
+
+ if not pid in thread_dict:
+ if args.verbose:
+ print "new thread", \
+ uid, process_dict[tgid].uid, pid, tgid, task_name
+ thread = Task(uid, pid, tgid, task_name, False, tick)
+ uid += 1
+ thread_dict[pid] = thread
+ process_dict[tgid].children.append(thread)
+ else:
+ if thread_dict[pid].task_name != task_name:
+ thread_dict[pid].task_name = task_name
+
+ if args.verbose:
+ print tick, uid, cpu_id, pid, tgid, task_name
+
+ task = thread_dict[pid]
+ event = Event(tick, task)
+ event_list[cpu_id].append(event)
+ unified_event_list.append(event)
+
+ if len(unified_event_list) == num_events:
+ print "Truncating at", num_events, "events!"
+ break
+ print "Found %d events." % len(unified_event_list)
+
+ for process in process_list:
+ if process.pid > 9990: # fix up framebuffer ticks
+ process.tick = start_tick
+ print process.uid, process.pid, process.tgid, \
+ process.task_name, str(process.tick)
+ for thread in process.children:
+ if thread.pid > 9990:
+ thread.tick = start_tick
+ print "\t", thread.uid, thread.pid, thread.tgid, \
+ thread.task_name, str(thread.tick)
+
+ end_tick = tick
+
+ print "Start tick:", start_tick
+ print "End tick: ", end_tick
+ print ""
+
+ return
+
+
+def initOutput(output_path):
+ if not os.path.exists(output_path):
+ os.mkdir(output_path)
+
+def ticksToNs(tick):
+ if ticks_in_ns < 0:
+ print "ticks_in_ns not set properly!"
+ sys.exit(1)
+
+ return tick / ticks_in_ns
+
+def writeXmlFile(xml, filename):
+ f = open(filename, "w")
+ txt = ET.tostring(xml)
+ f.write(minidom.parseString(txt).toprettyxml())
+ f.close()
+
+
+# StatsEntry that contains individual statistics
+class StatsEntry(object):
+ def __init__(self, name, group, group_index, per_cpu, per_switchcpu, key):
+
+ # Full name of statistics
+ self.name = name
+
+ # Streamline group name that statistic will belong to
+ self.group = group
+
+ # Index of statistics within group (used to change colors within groups)
+ self.group_index = group_index
+
+ # Shorter name with "system" stripped off
+ # and symbols converted to alphanumerics
+ self.short_name = re.sub("system\.", "", name)
+ self.short_name = re.sub(":", "_", name)
+
+ # Regex for this stat (string version used to construct union regex)
+ self.regex_string = "^" + name + "\s+([\d\.]+)"
+ self.regex = re.compile("^" + name + "\s+([\d\.e\-]+)\s+# (.*)$", re.M)
+ self.description = ""
+
+ # Whether this stat is use per CPU or not
+ self.per_cpu = per_cpu
+ self.per_switchcpu = per_switchcpu
+
+ # Key used in .apc protocol (as described in captured.xml)
+ self.key = key
+
+ # List of values of stat per timestamp
+ self.values = []
+
+ # Whether this stat has been found for the current timestamp
+ self.found = False
+
+ # Whether this stat has been found at least once
+ # (to suppress too many warnings)
+ self.not_found_at_least_once = False
+
+ # Field used to hold ElementTree subelement for this stat
+ self.ET_element = None
+
+ # Create per-CPU stat name and regex, etc.
+ if self.per_cpu:
+ self.per_cpu_regex_string = []
+ self.per_cpu_regex = []
+ self.per_cpu_name = []
+ self.per_cpu_found = []
+ for i in range(num_cpus):
+ # Resuming from checkpoints results in using "switch_cpus"
+ if per_switchcpu:
+ per_cpu_name = "system.switch_cpus"
+ else:
+ per_cpu_name = "system.cpu"
+
+ # No CPU number appends if num_cpus == 1
+ if num_cpus > 1:
+ per_cpu_name += str(i)
+ per_cpu_name += "." + self.name
+ self.per_cpu_name.append(per_cpu_name)
+ print "\t", per_cpu_name
+
+ self.per_cpu_regex_string.\
+ append("^" + per_cpu_name + "\s+[\d\.]+")
+ self.per_cpu_regex.append(re.compile("^" + per_cpu_name + \
+ "\s+([\d\.e\-]+)\s+# (.*)$", re.M))
+ self.values.append([])
+ self.per_cpu_found.append(False)
+
+ def append_value(self, val, per_cpu_index = None):
+ if self.per_cpu:
+ self.values[per_cpu_index].append(str(val))
+ else:
+ self.values.append(str(val))
+
+# Global stats object that contains the list of stats entries
+# and other utility functions
+class Stats(object):
+ def __init__(self):
+ self.stats_list = []
+ self.tick_list = []
+ self.next_key = 1
+
+ def register(self, name, group, group_index, per_cpu, per_switchcpu):
+ print "registering stat:", name, "group:", group, group_index
+ self.stats_list.append(StatsEntry(name, group, group_index, per_cpu, \
+ per_switchcpu, self.next_key))
+ self.next_key += 1
+
+ # Union of all stats to accelerate parsing speed
+ def createStatsRegex(self):
+ regex_strings = [];
+ print "\nnum entries in stats_list", len(self.stats_list)
+ for entry in self.stats_list:
+ if entry.per_cpu:
+ for i in range(num_cpus):
+ regex_strings.append(entry.per_cpu_regex_string[i])
+ else:
+ regex_strings.append(entry.regex_string)
+
+ self.regex = re.compile('|'.join(regex_strings))
+
+
+def registerStats(config_file):
+ print "==============================="
+ print "Parsing stats config.ini file..."
+ print config_file
+ print "==============================="
+
+ config = ConfigParser()
+ if not config.read(config_file):
+ print "ERROR: config file '", config_file, "' not found!"
+ sys.exit(1)
+
+ print "\nRegistering Stats..."
+
+ stats = Stats()
+
+ per_cpu_stat_groups = config.options('PER_CPU_STATS')
+ for group in per_cpu_stat_groups:
+ i = 0
+ per_cpu_stats_list = config.get('PER_CPU_STATS', group).split('\n')
+ for item in per_cpu_stats_list:
+ if item:
+ stats.register(item, group, i, True, False)
+ i += 1
+
+ per_cpu_stat_groups = config.options('PER_SWITCHCPU_STATS')
+ for group in per_cpu_stat_groups:
+ i = 0
+ per_cpu_stats_list = \
+ config.get('PER_SWITCHCPU_STATS', group).split('\n')
+ for item in per_cpu_stats_list:
+ if item:
+ stats.register(item, group, i, True, True)
+ i += 1
+
+ per_l2_stat_groups = config.options('PER_L2_STATS')
+ for group in per_l2_stat_groups:
+ i = 0
+ per_l2_stats_list = config.get('PER_L2_STATS', group).split('\n')
+ for item in per_l2_stats_list:
+ if item:
+ for l2 in range(num_l2):
+ name = item
+ prefix = "system.l2"
+ if num_l2 > 1:
+ prefix += str(l2)
+ prefix += "."
+ name = prefix + name
+ stats.register(name, group, i, False, False)
+ i += 1
+
+ other_stat_groups = config.options('OTHER_STATS')
+ for group in other_stat_groups:
+ i = 0
+ other_stats_list = config.get('OTHER_STATS', group).split('\n')
+ for item in other_stats_list:
+ if item:
+ stats.register(item, group, i, False, False)
+ i += 1
+
+ stats.createStatsRegex()
+
+ return stats
+
+# Parse and read in gem5 stats file
+# Streamline counters are organized per CPU
+def readGem5Stats(stats, gem5_stats_file):
+ print "\n==============================="
+ print "Parsing gem5 stats file..."
+ print gem5_stats_file
+ print "===============================\n"
+ ext = os.path.splitext(gem5_stats_file)[1]
+
+ window_start_regex = \
+ re.compile("^---------- Begin Simulation Statistics ----------")
+ window_end_regex = \
+ re.compile("^---------- End Simulation Statistics ----------")
+ final_tick_regex = re.compile("^final_tick\s+(\d+)")
+
+ global ticks_in_ns
+ sim_freq_regex = re.compile("^sim_freq\s+(\d+)")
+ sim_freq = -1
+
+ try:
+ if ext == ".gz":
+ f = gzip.open(gem5_stats_file, "r")
+ else:
+ f = open(gem5_stats_file, "r")
+ except:
+ print "ERROR opening stats file", gem5_stats_file, "!"
+ sys.exit(1)
+
+ stats_not_found_list = stats.stats_list[:]
+ window_num = 0
+
+ while (True):
+ error = False
+ try:
+ line = f.readline()
+ except IOError:
+ print ""
+ print "WARNING: IO error in stats file"
+ print "(gzip stream not closed properly?)...continuing for now"
+ error = True
+ if not line:
+ break
+
+ # Find out how many gem5 ticks in 1ns
+ if sim_freq < 0:
+ m = sim_freq_regex.match(line)
+ if m:
+ sim_freq = int(m.group(1)) # ticks in 1 sec
+ ticks_in_ns = int(sim_freq / 1e9)
+ print "Simulation frequency found! 1 tick == %e sec\n" \
+ % (1.0 / sim_freq)
+
+ # Final tick in gem5 stats: current absolute timestamp
+ m = final_tick_regex.match(line)
+ if m:
+ tick = int(m.group(1))
+ if tick > end_tick:
+ break
+ stats.tick_list.append(tick)
+
+
+ if (window_end_regex.match(line) or error):
+ if args.verbose:
+ print "new window"
+ for stat in stats.stats_list:
+ if stat.per_cpu:
+ for i in range(num_cpus):
+ if not stat.per_cpu_found[i]:
+ if not stat.not_found_at_least_once:
+ print "WARNING: stat not found in window #", \
+ window_num, ":", stat.per_cpu_name[i]
+ print "suppressing further warnings for " + \
+ "this stat"
+ stat.not_found_at_least_once = True
+ stat.values[i].append(str(0))
+ stat.per_cpu_found[i] = False
+ else:
+ if not stat.found:
+ if not stat.not_found_at_least_once:
+ print "WARNING: stat not found in window #", \
+ window_num, ":", stat.name
+ print "suppressing further warnings for this stat"
+ stat.not_found_at_least_once = True
+ stat.values.append(str(0))
+ stat.found = False
+ stats_not_found_list = stats.stats_list[:]
+ window_num += 1
+ if error:
+ break
+
+ # Do a single regex of the union of all stats first for speed
+ if stats.regex.match(line):
+ # Then loop through only the stats we haven't seen in this window
+ for stat in stats_not_found_list[:]:
+ if stat.per_cpu:
+ for i in range(num_cpus):
+ m = stat.per_cpu_regex[i].match(line)
+ if m:
+ if stat.name == "ipc":
+ value = str(int(float(m.group(1)) * 1000))
+ else:
+ value = str(int(float(m.group(1))))
+ if args.verbose:
+ print stat.per_cpu_name[i], value
+ stat.values[i].append(value)
+ stat.per_cpu_found[i] = True
+ all_found = True
+ for j in range(num_cpus):
+ if not stat.per_cpu_found[j]:
+ all_found = False
+ if all_found:
+ stats_not_found_list.remove(stat)
+ if stat.description == "":
+ stat.description = m.group(2)
+ else:
+ m = stat.regex.match(line)
+ if m:
+ value = str(int(float(m.group(1))))
+ if args.verbose:
+ print stat.name, value
+ stat.values.append(value)
+ stat.found = True
+ stats_not_found_list.remove(stat)
+ if stat.description == "":
+ stat.description = m.group(2)
+ f.close()
+
+
+# Create session.xml file in .apc folder
+def doSessionXML(output_path):
+ session_file = output_path + "/session.xml"
+
+ xml = ET.Element("session")
+
+ xml.set("version", "1")
+ xml.set("call_stack_unwinding", "no")
+ xml.set("parse_debug_info", "no")
+ xml.set("high_resolution", "yes")
+ xml.set("buffer_mode", "streaming")
+ xml.set("sample_rate", "low")
+
+ # Setting duration to zero for now. Doesn't affect visualization.
+ xml.set("duration", "0")
+
+ xml.set("target_host", "")
+ xml.set("target_port", "8080")
+
+ writeXmlFile(xml, session_file)
+
+
+# Create captured.xml file in .apc folder
+def doCapturedXML(output_path, stats):
+ captured_file = output_path + "/captured.xml"
+
+ xml = ET.Element("captured")
+ xml.set("version", "1")
+ xml.set("protocol", "12")
+
+ target = ET.SubElement(xml, "target")
+ target.set("name", "gem5")
+ target.set("sample_rate", "1000")
+ target.set("cores", str(num_cpus))
+
+ counters = ET.SubElement(xml, "counters")
+ for stat in stats.stats_list:
+ s = ET.SubElement(counters, "counter")
+ stat_name = re.sub("\.", "_", stat.short_name)
+ s.set("title", stat.group)
+ s.set("name", stat_name)
+ s.set("color", "0x00000000")
+ s.set("key", "0x%08x" % stat.key)
+ s.set("type", stat_name)
+ s.set("event", "0x00000000")
+ if stat.per_cpu:
+ s.set("per_cpu", "yes")
+ else:
+ s.set("per_cpu", "no")
+ s.set("display", "")
+ s.set("units", "")
+ s.set("average_selection", "no")
+ s.set("description", stat.description)
+
+ writeXmlFile(xml, captured_file)
+
+# Writes out Streamline cookies (unique IDs per process/thread)
+def writeCookiesThreads(blob):
+ thread_list = []
+ for process in process_list:
+ if process.uid > 0:
+ print "cookie", process.task_name, process.uid
+ writeBinary(blob, cookieNameFrame(process.uid, process.task_name))
+
+ # pid and tgid need to be positive values -- no longer true?
+ for thread in process.children:
+ thread_list.append(thread)
+
+ # Threads need to be sorted in timestamp order
+ thread_list.sort(key = lambda x: x.tick)
+ for thread in thread_list:
+ print "thread", thread.task_name, (ticksToNs(thread.tick)),\
+ thread.tgid, thread.pid
+ writeBinary(blob, threadNameFrame(ticksToNs(thread.tick),\
+ thread.pid, thread.task_name))
+
+# Writes context switch info as Streamline scheduling events
+def writeSchedEvents(blob):
+ for cpu in range(num_cpus):
+ for event in event_list[cpu]:
+ timestamp = ticksToNs(event.tick)
+ pid = event.task.tgid
+ tid = event.task.pid
+ if process_dict.has_key(event.task.tgid):
+ cookie = process_dict[event.task.tgid].uid
+ else:
+ cookie = 0
+
+ # State:
+ # 0: waiting on other event besides I/O
+ # 1: Contention/pre-emption
+ # 2: Waiting on I/O
+ # 3: Waiting on mutex
+ # Hardcoding to 0 for now. Other states not implemented yet.
+ state = 0
+
+ if args.verbose:
+ print cpu, timestamp, pid, tid, cookie
+
+ writeBinary(blob,\
+ schedSwitchFrame(cpu, timestamp, pid, tid, cookie, state))
+
+# Writes selected gem5 statistics as Streamline counters
+def writeCounters(blob, stats):
+ timestamp_list = []
+ for tick in stats.tick_list:
+ if tick > end_tick:
+ break
+ timestamp_list.append(ticksToNs(tick))
+
+ for stat in stats.stats_list:
+ if stat.per_cpu:
+ stat_length = len(stat.values[0])
+ else:
+ stat_length = len(stat.values)
+
+ for n in range(len(timestamp_list)):
+ for stat in stats.stats_list:
+ if stat.per_cpu:
+ for i in range(num_cpus):
+ writeBinary(blob, counterFrame(timestamp_list[n], i, \
+ stat.key, int(float(stat.values[i][n]))))
+ else:
+ writeBinary(blob, counterFrame(timestamp_list[n], 0, \
+ stat.key, int(float(stat.values[n]))))
+
+# Streamline can display LCD frame buffer dumps (gzipped bmp)
+# This function converts the frame buffer dumps to the Streamline format
+def writeVisualAnnotations(blob, input_path, output_path):
+ frame_path = input_path + "/frames_system.vncserver"
+ if not os.path.exists(frame_path):
+ return
+
+ frame_count = 0
+ file_list = os.listdir(frame_path)
+ file_list.sort()
+ re_fb = re.compile("fb\.(\d+)\.(\d+)\.bmp.gz")
+
+ # Use first non-negative pid to tag visual annotations
+ annotate_pid = -1
+ for e in unified_event_list:
+ pid = e.task.pid
+ if pid >= 0:
+ annotate_pid = pid
+ break
+
+ for fn in file_list:
+ m = re_fb.match(fn)
+ if m:
+ seq = m.group(1)
+ tick = int(m.group(2))
+ if tick > end_tick:
+ break
+ frame_count += 1
+
+ userspace_body = []
+ userspace_body += packed32(0x1C) # escape code
+ userspace_body += packed32(0x04) # visual code
+
+ text_annotation = "image_" + str(ticksToNs(tick)) + ".bmp.gz"
+ userspace_body += int16(len(text_annotation))
+ userspace_body += utf8StringList(text_annotation)
+
+ if gzipped_bmp_supported:
+ # copy gzipped bmp directly
+ bytes_read = open(frame_path + "/" + fn, "rb").read()
+ else:
+ # copy uncompressed bmp
+ bytes_read = gzip.open(frame_path + "/" + fn, "rb").read()
+
+ userspace_body += int32(len(bytes_read))
+ userspace_body += bytes_read
+
+ writeBinary(blob, annotateFrame(0, annotate_pid, ticksToNs(tick), \
+ len(userspace_body), userspace_body))
+
+ print "\nfound", frame_count, "frames for visual annotation.\n"
+
+
+def createApcProject(input_path, output_path, stats):
+ initOutput(output_path)
+
+ blob = open(output_path + "/0000000000", "wb")
+
+ # Summary frame takes current system time and system uptime.
+ # Filling in with random values for now.
+ writeBinary(blob, summaryFrame(1234, 5678))
+
+ writeCookiesThreads(blob)
+
+ print "writing Events"
+ writeSchedEvents(blob)
+
+ print "writing Counters"
+ writeCounters(blob, stats)
+
+ print "writing Visual Annotations"
+ writeVisualAnnotations(blob, input_path, output_path)
+
+ doSessionXML(output_path)
+ doCapturedXML(output_path, stats)
+
+ blob.close()
+
+
+
+#######################
+# Main Routine
+
+input_path = args.input_path
+output_path = args.output_path
+
+####
+# Make sure input path exists
+####
+if not os.path.exists(input_path):
+ print "ERROR: Input path %s does not exist!" % input_path
+ sys.exit(1)
+
+####
+# Parse gem5 configuration file to find # of CPUs and L2s
+####
+(num_cpus, num_l2) = parseConfig(input_path + "/config.ini")
+
+####
+# Parse task file to find process/thread info
+####
+parseProcessInfo(input_path + "/system.tasks.txt")
+
+####
+# Parse stat config file and register stats
+####
+stat_config_file = args.stat_config_file
+stats = registerStats(stat_config_file)
+
+####
+# Parse gem5 stats
+####
+# Check if both stats.txt and stats.txt.gz exist and warn if both exist
+if os.path.exists(input_path + "/stats.txt") and \
+ os.path.exists(input_path + "/stats.txt.gz"):
+ print "WARNING: Both stats.txt.gz and stats.txt exist. \
+ Using stats.txt.gz by default."
+
+gem5_stats_file = input_path + "/stats.txt.gz"
+if not os.path.exists(gem5_stats_file):
+ gem5_stats_file = input_path + "/stats.txt"
+if not os.path.exists(gem5_stats_file):
+ print "ERROR: stats.txt[.gz] file does not exist in %s!" % input_path
+ sys.exit(1)
+
+readGem5Stats(stats, gem5_stats_file)
+
+####
+# Create Streamline .apc project folder
+####
+createApcProject(input_path, output_path, stats)
+
+print "All done!"
projects / gem5.git / commitdiff