util/streamline/m5stats2streamline.py

   1 #!/usr/bin/env python2.7
   2
   3 # Copyright (c) 2012, 2014 ARM Limited
   4 # All rights reserved
   5 #
   6 # The license below extends only to copyright in the software and shall
   7 # not be construed as granting a license to any other intellectual
   8 # property including but not limited to intellectual property relating
   9 # to a hardware implementation of the functionality of the software
  10 # licensed hereunder.  You may use the software subject to the license
  11 # terms below provided that you ensure that this notice is replicated
  12 # unmodified and in its entirety in all distributions of the software,
  13 # modified or unmodified, in source code or in binary form.
  14 #
  15 # Redistribution and use in source and binary forms, with or without
  16 # modification, are permitted provided that the following conditions are
  17 # met: redistributions of source code must retain the above copyright
  18 # notice, this list of conditions and the following disclaimer;
  19 # redistributions in binary form must reproduce the above copyright
  20 # notice, this list of conditions and the following disclaimer in the
  21 # documentation and/or other materials provided with the distribution;
  22 # neither the name of the copyright holders nor the names of its
  23 # contributors may be used to endorse or promote products derived from
  24 # this software without specific prior written permission.
  25 #
  26 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  27 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  28 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  29 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  30 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  31 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  32 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  33 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  34 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  35 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  36 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  37 #
  38 # Author: Dam Sunwoo
  39 #
  40
  41 # This script converts gem5 output to ARM DS-5 Streamline .apc project file
  42 # (Requires the gem5 runs to be run with ContextSwitchStatsDump enabled and
  43 # some patches applied to target Linux kernel.)
  44 #
  45 # Usage:
  46 # m5stats2streamline.py <stat_config.ini> <gem5 run folder> <dest .apc folder>
  47 #
  48 # <stat_config.ini>: .ini file that describes which stats to be included
  49 #                    in conversion. Sample .ini files can be found in
  50 #                    util/streamline.
  51 #                    NOTE: this is NOT the gem5 config.ini file.
  52 #
  53 # <gem5 run folder>: Path to gem5 run folder (must contain config.ini,
  54 #                    stats.txt[.gz], and system.tasks.txt.)
  55 #
  56 # <dest .apc folder>: Destination .apc folder path
  57 #
  58 # APC project generation based on Gator v17 (DS-5 v5.17)
  59 # Subsequent versions should be backward compatible
  60
  61 import re, sys, os
  62 from ConfigParser import ConfigParser
  63 import gzip
  64 import xml.etree.ElementTree as ET
  65 import xml.dom.minidom as minidom
  66 import shutil
  67 import zlib
  68
  69 import argparse
  70
  71 parser = argparse.ArgumentParser(
  72         formatter_class=argparse.RawDescriptionHelpFormatter,
  73         description="""
  74         Converts gem5 runs to ARM DS-5 Streamline .apc project file.
  75         (NOTE: Requires gem5 runs to be run with ContextSwitchStatsDump
  76         enabled and some patches applied to the target Linux kernel.)
  77
  78         APC project generation based on Gator v17 (DS-5 v5.17)
  79         Subsequent versions should be backward compatible
  80         """)
  81
  82 parser.add_argument("stat_config_file", metavar="<stat_config.ini>",
  83                     help=".ini file that describes which stats to be included \
  84                     in conversion. Sample .ini files can be found in \
  85                     util/streamline. NOTE: this is NOT the gem5 config.ini \
  86                     file.")
  87
  88 parser.add_argument("input_path", metavar="<gem5 run folder>",
  89                     help="Path to gem5 run folder (must contain config.ini, \
  90                     stats.txt[.gz], and system.tasks.txt.)")
  91
  92 parser.add_argument("output_path", metavar="<dest .apc folder>",
  93                     help="Destination .apc folder path")
  94
  95 parser.add_argument("--num-events", action="store", type=int,
  96                     default=1000000,
  97                     help="Maximum number of scheduling (context switch) \
  98                     events to be processed. Set to truncate early. \
  99                     Default=1000000")
 100
 101 parser.add_argument("--gzipped-bmp-not-supported", action="store_true",
 102                     help="Do not use gzipped .bmp files for visual annotations. \
 103                     This option is only required when using Streamline versions \
 104                     older than 5.14")
 105
 106 parser.add_argument("--verbose", action="store_true",
 107                     help="Enable verbose output")
 108
 109 args = parser.parse_args()
 110
 111 if not re.match("(.*)\.apc", args.output_path):
 112     print "ERROR: <dest .apc folder> should end with '.apc'!"
 113     sys.exit(1)
 114
 115 # gzipped BMP files for visual annotation is supported in Streamline 5.14.
 116 # Setting this to True will significantly compress the .apc binary file that
 117 # includes frame buffer snapshots.
 118 gzipped_bmp_supported = not args.gzipped_bmp_not_supported
 119
 120 ticks_in_ns = -1
 121
 122 # Default max # of events. Increase this for longer runs.
 123 num_events = args.num_events
 124
 125 start_tick = -1
 126 end_tick = -1
 127
 128 # Parse gem5 config.ini file to determine some system configurations.
 129 # Number of CPUs, L2s, etc.
 130 def parseConfig(config_file):
 131     global num_cpus, num_l2
 132
 133     print "\n==============================="
 134     print "Parsing gem5 config.ini file..."
 135     print config_file
 136     print "===============================\n"
 137     config = ConfigParser()
 138     if not config.read(config_file):
 139         print "ERROR: config file '", config_file, "' not found"
 140         sys.exit(1)
 141
 142     if config.has_section("system.cpu"):
 143         num_cpus = 1
 144     else:
 145         num_cpus = 0
 146         while config.has_section("system.cpu" + str(num_cpus)):
 147             num_cpus += 1
 148
 149     if config.has_section("system.l2_cache"):
 150         num_l2 = 1
 151     else:
 152         num_l2 = 0
 153         while config.has_section("system.l2_cache" + str(num_l2)):
 154             num_l2 += 1
 155
 156     print "Num CPUs:", num_cpus
 157     print "Num L2s:", num_l2
 158     print ""
 159
 160     return (num_cpus, num_l2)
 161
 162
 163 process_dict = {}
 164 thread_dict = {}
 165
 166 process_list = []
 167
 168 idle_uid = -1
 169 kernel_uid = -1
 170
 171 class Task(object):
 172     def __init__(self, uid, pid, tgid, task_name, is_process, tick):
 173         if pid == 0: # Idle
 174             self.uid = 0
 175         elif pid == -1: # Kernel
 176             self.uid = 0
 177         else:
 178             self.uid = uid
 179         self.pid = pid
 180         self.tgid = tgid
 181         self.is_process = is_process
 182         self.task_name = task_name
 183         self.children = []
 184         self.tick = tick # time this task first appeared
 185
 186 class Event(object):
 187     def __init__(self, tick, task):
 188         self.tick = tick
 189         self.task = task
 190
 191 ############################################################
 192 # Types used in APC Protocol
 193 #  - packed32, packed64
 194 #  - int32
 195 #  - string
 196 ############################################################
 197
 198 def packed32(x):
 199     ret = []
 200     more = True
 201     while more:
 202         b = x & 0x7f
 203         x = x >> 7
 204         if (((x == 0) and ((b & 0x40) == 0)) or \
 205             ((x == -1) and ((b & 0x40) != 0))):
 206             more = False
 207         else:
 208             b = b | 0x80
 209         ret.append(b)
 210     return ret
 211
 212 # For historical reasons, 32/64-bit versions of functions are presevered
 213 def packed64(x):
 214     return packed32(x)
 215
 216 #  variable length packed 4-byte signed value
 217 def unsigned_packed32(x):
 218     ret = []
 219     if ((x & 0xffffff80) == 0):
 220         ret.append(x & 0x7f)
 221     elif ((x & 0xffffc000) == 0):
 222         ret.append((x | 0x80) & 0xff)
 223         ret.append((x >> 7) & 0x7f)
 224     elif ((x & 0xffe00000) == 0):
 225         ret.append((x | 0x80) & 0xff)
 226         ret.append(((x >> 7) | 0x80) & 0xff)
 227         ret.append((x >> 14) & 0x7f)
 228     elif ((x & 0xf0000000) == 0):
 229         ret.append((x | 0x80) & 0xff)
 230         ret.append(((x >> 7) | 0x80) & 0xff)
 231         ret.append(((x >> 14) | 0x80) & 0xff)
 232         ret.append((x >> 21) & 0x7f)
 233     else:
 234         ret.append((x | 0x80) & 0xff)
 235         ret.append(((x >> 7) | 0x80) & 0xff)
 236         ret.append(((x >> 14) | 0x80) & 0xff)
 237         ret.append(((x >> 21) | 0x80) & 0xff)
 238         ret.append((x >> 28) & 0x0f)
 239     return ret
 240
 241 #  variable length packed 8-byte signed value
 242 def unsigned_packed64(x):
 243     ret = []
 244     if ((x & 0xffffffffffffff80) == 0):
 245         ret.append(x & 0x7f)
 246     elif ((x & 0xffffffffffffc000) == 0):
 247         ret.append((x | 0x80) & 0xff)
 248         ret.append((x >> 7) & 0x7f)
 249     elif ((x & 0xffffffffffe00000) == 0):
 250         ret.append((x | 0x80) & 0xff)
 251         ret.append(((x >> 7) | 0x80) & 0xff)
 252         ret.append((x >> 14) & 0x7f)
 253     elif ((x & 0xfffffffff0000000) == 0):
 254         ret.append((x | 0x80) & 0xff)
 255         ret.append(((x >> 7) | 0x80) & 0xff)
 256         ret.append(((x >> 14) | 0x80) & 0xff)
 257         ret.append((x >> 21) & 0x7f)
 258     elif ((x & 0xfffffff800000000) == 0):
 259         ret.append((x | 0x80) & 0xff)
 260         ret.append(((x >> 7) | 0x80) & 0xff)
 261         ret.append(((x >> 14) | 0x80) & 0xff)
 262         ret.append(((x >> 21) | 0x80) & 0xff)
 263         ret.append((x >> 28) & 0x7f)
 264     elif ((x & 0xfffffc0000000000) == 0):
 265         ret.append((x | 0x80) & 0xff)
 266         ret.append(((x >> 7) | 0x80) & 0xff)
 267         ret.append(((x >> 14) | 0x80) & 0xff)
 268         ret.append(((x >> 21) | 0x80) & 0xff)
 269         ret.append(((x >> 28) | 0x80) & 0xff)
 270         ret.append((x >> 35) & 0x7f)
 271     elif ((x & 0xfffe000000000000) == 0):
 272         ret.append((x | 0x80) & 0xff)
 273         ret.append(((x >> 7) | 0x80) & 0xff)
 274         ret.append(((x >> 14) | 0x80) & 0xff)
 275         ret.append(((x >> 21) | 0x80) & 0xff)
 276         ret.append(((x >> 28) | 0x80) & 0xff)
 277         ret.append(((x >> 35) | 0x80) & 0xff)
 278         ret.append((x >> 42) & 0x7f)
 279     elif ((x & 0xff00000000000000) == 0):
 280         ret.append((x | 0x80) & 0xff)
 281         ret.append(((x >> 7) | 0x80) & 0xff)
 282         ret.append(((x >> 14) | 0x80) & 0xff)
 283         ret.append(((x >> 21) | 0x80) & 0xff)
 284         ret.append(((x >> 28) | 0x80) & 0xff)
 285         ret.append(((x >> 35) | 0x80) & 0xff)
 286         ret.append(((x >> 42) | 0x80) & 0xff)
 287         ret.append((x >> 49) & 0x7f)
 288     elif ((x & 0x8000000000000000) == 0):
 289         ret.append((x | 0x80) & 0xff)
 290         ret.append(((x >> 7) | 0x80) & 0xff)
 291         ret.append(((x >> 14) | 0x80) & 0xff)
 292         ret.append(((x >> 21) | 0x80) & 0xff)
 293         ret.append(((x >> 28) | 0x80) & 0xff)
 294         ret.append(((x >> 35) | 0x80) & 0xff)
 295         ret.append(((x >> 42) | 0x80) & 0xff)
 296         ret.append(((x >> 49) | 0x80) & 0xff)
 297         ret.append((x >> 56) & 0x7f)
 298     else:
 299         ret.append((x | 0x80) & 0xff)
 300         ret.append(((x >> 7) | 0x80) & 0xff)
 301         ret.append(((x >> 14) | 0x80) & 0xff)
 302         ret.append(((x >> 21) | 0x80) & 0xff)
 303         ret.append(((x >> 28) | 0x80) & 0xff)
 304         ret.append(((x >> 35) | 0x80) & 0xff)
 305         ret.append(((x >> 42) | 0x80) & 0xff)
 306         ret.append(((x >> 49) | 0x80) & 0xff)
 307         ret.append(((x >> 56) | 0x80) & 0xff)
 308         ret.append((x >> 63) & 0x7f)
 309     return ret
 310
 311 # 4-byte signed little endian
 312 def int32(x):
 313     ret = []
 314     ret.append(x & 0xff)
 315     ret.append((x >> 8) & 0xff)
 316     ret.append((x >> 16) & 0xff)
 317     ret.append((x >> 24) & 0xff)
 318     return ret
 319
 320 # 2-byte signed little endian
 321 def int16(x):
 322     ret = []
 323     ret.append(x & 0xff)
 324     ret.append((x >> 8) & 0xff)
 325     return ret
 326
 327 # a packed32 length followed by the specified number of characters
 328 def stringList(x):
 329     ret = []
 330     ret += packed32(len(x))
 331     for i in x:
 332         ret.append(i)
 333     return ret
 334
 335 def utf8StringList(x):
 336     ret = []
 337     for i in x:
 338         ret.append(ord(i))
 339     return ret
 340
 341 # packed64 time value in nanoseconds relative to the uptime from the
 342 # Summary message.
 343 def timestampList(x):
 344     ret = packed64(x)
 345     return ret
 346
 347
 348 ############################################################
 349 # Write binary
 350 ############################################################
 351
 352 def writeBinary(outfile, binary_list):
 353     for i in binary_list:
 354         outfile.write("%c" % i)
 355
 356 ############################################################
 357 # APC Protocol Frame Types
 358 ############################################################
 359
 360 def addFrameHeader(frame_type, body, core):
 361     ret = []
 362
 363     if frame_type == "Summary":
 364         code = 1
 365     elif frame_type == "Backtrace":
 366         code = 2
 367     elif frame_type == "Name":
 368         code = 3
 369     elif frame_type == "Counter":
 370         code = 4
 371     elif frame_type == "Block Counter":
 372         code = 5
 373     elif frame_type == "Annotate":
 374         code = 6
 375     elif frame_type == "Sched Trace":
 376         code = 7
 377     elif frame_type == "GPU Trace":
 378         code = 8
 379     elif frame_type == "Idle":
 380         code = 9
 381     else:
 382         print "ERROR: Unknown frame type:", frame_type
 383         sys.exit(1)
 384
 385     packed_code = packed32(code)
 386
 387     packed_core = packed32(core)
 388
 389     length = int32(len(packed_code) + len(packed_core) + len(body))
 390
 391     ret = length + packed_code + packed_core + body
 392     return ret
 393
 394
 395 # Summary frame
 396 #  - timestamp: packed64
 397 #  - uptime: packed64
 398 def summaryFrame(timestamp, uptime):
 399     frame_type = "Summary"
 400     newline_canary = stringList("1\n2\r\n3\r4\n\r5")
 401     monotonic_delta = packed64(0)
 402     end_of_attr = stringList("")
 403     body = newline_canary + packed64(timestamp) + packed64(uptime)
 404     body += monotonic_delta + end_of_attr
 405     ret = addFrameHeader(frame_type, body, 0)
 406     return ret
 407
 408 # Backtrace frame
 409 #  - not implemented yet
 410 def backtraceFrame():
 411     pass
 412
 413 # Cookie name message
 414 #  - cookie: packed32
 415 #  - name: string
 416 def cookieNameFrame(cookie, name):
 417     frame_type = "Name"
 418     packed_code = packed32(1)
 419     body = packed_code + packed32(cookie) + stringList(name)
 420     ret = addFrameHeader(frame_type, body, 0)
 421     return ret
 422
 423 # Thread name message
 424 #  - timestamp: timestamp
 425 #  - thread id: packed32
 426 #  - name: string
 427 def threadNameFrame(timestamp, thread_id, name):
 428     frame_type = "Name"
 429     packed_code = packed32(2)
 430     body = packed_code + timestampList(timestamp) + \
 431             packed32(thread_id) + stringList(name)
 432     ret = addFrameHeader(frame_type, body, 0)
 433     return ret
 434
 435 # Core name message
 436 #  - name: string
 437 #  - core_id: packed32
 438 #  - cpuid: packed32
 439 def coreNameFrame(name, core_id, cpuid):
 440     frame_type = "Name"
 441     packed_code = packed32(3)
 442     body = packed_code + packed32(core_id) + packed32(cpuid) + stringList(name)
 443     ret = addFrameHeader(frame_type, body, 0)
 444     return ret
 445
 446 # IRQ Cookie name message
 447 #  - cookie: packed32
 448 #  - name: string
 449 #  - irq: packed32
 450 def irqCookieNameFrame(cookie, name, irq):
 451     frame_type = "Name"
 452     packed_code = packed32(5)
 453     body = packed_code + packed32(cookie) + stringList(name) + packed32(irq)
 454     ret = addFrameHeader(frame_type, body, 0)
 455     return ret
 456
 457 # Counter frame message
 458 #  - timestamp: timestamp
 459 #  - core: packed32
 460 #  - key: packed32
 461 #  - value: packed64
 462 def counterFrame(timestamp, core, key, value):
 463     frame_type = "Counter"
 464     body = timestampList(timestamp) + packed32(core) + packed32(key) + \
 465             packed64(value)
 466     ret = addFrameHeader(frame_type, body, core)
 467     return ret
 468
 469 # Block Counter frame message
 470 #  - key: packed32
 471 #  - value: packed64
 472 def blockCounterFrame(core, key, value):
 473     frame_type = "Block Counter"
 474     body = packed32(key) + packed64(value)
 475     ret = addFrameHeader(frame_type, body, core)
 476     return ret
 477
 478 # Annotate frame messages
 479 #  - core: packed32
 480 #  - tid: packed32
 481 #  - timestamp: timestamp
 482 #  - size: packed32
 483 #  - body
 484 def annotateFrame(core, tid, timestamp, size, userspace_body):
 485     frame_type = "Annotate"
 486     body = packed32(core) + packed32(tid) + timestampList(timestamp) + \
 487             packed32(size) + userspace_body
 488     ret = addFrameHeader(frame_type, body, core)
 489     return ret
 490
 491 # Scheduler Trace frame messages
 492 # Sched Switch
 493 #  - Code: 1
 494 #  - timestamp: timestamp
 495 #  - pid: packed32
 496 #  - tid: packed32
 497 #  - cookie: packed32
 498 #  - state: packed32
 499 def schedSwitchFrame(core, timestamp, pid, tid, cookie, state):
 500     frame_type = "Sched Trace"
 501     body = packed32(1) + timestampList(timestamp) + packed32(pid) + \
 502             packed32(tid) + packed32(cookie) + packed32(state)
 503     ret = addFrameHeader(frame_type, body, core)
 504     return ret
 505
 506 # Sched Thread Exit
 507 #  - Code: 2
 508 #  - timestamp: timestamp
 509 #  - tid: packed32
 510 def schedThreadExitFrame(core, timestamp, pid, tid, cookie, state):
 511     frame_type = "Sched Trace"
 512     body = packed32(2) + timestampList(timestamp) + packed32(tid)
 513     ret = addFrameHeader(frame_type, body, core)
 514     return ret
 515
 516 # GPU Trace frame messages
 517 #  - Not implemented yet
 518 def gpuTraceFrame():
 519     pass
 520
 521 # Idle frame messages
 522 # Enter Idle
 523 #  - code: 1
 524 #  - timestamp: timestamp
 525 #  - core: packed32
 526 def enterIdleFrame(timestamp, core):
 527     frame_type = "Idle"
 528     body = packed32(1) + timestampList(timestamp) + packed32(core)
 529     ret = addFrameHeader(frame_type, body, core)
 530     return ret
 531
 532 # Exit Idle
 533 #  - code: 2
 534 #  - timestamp: timestamp
 535 #  - core: packed32
 536 def exitIdleFrame(timestamp, core):
 537     frame_type = "Idle"
 538     body = packed32(2) + timestampList(timestamp) + packed32(core)
 539     ret = addFrameHeader(frame_type, body, core)
 540     return ret
 541
 542
 543 ####################################################################
 544 def parseProcessInfo(task_file):
 545     print "\n==============================="
 546     print "Parsing Task file..."
 547     print task_file
 548     print "===============================\n"
 549
 550     global start_tick, end_tick, num_cpus
 551     global process_dict, thread_dict, process_list
 552     global event_list, unified_event_list
 553     global idle_uid, kernel_uid
 554
 555     event_list = []
 556     unified_event_list = []
 557     for cpu in range(num_cpus):
 558         event_list.append([])
 559
 560     uid = 1 # uid 0 is reserved for idle
 561
 562     # Dummy Tasks for frame buffers and system diagrams
 563     process = Task(uid, 9999, 9999, "framebuffer", True, 0)
 564     process_list.append(process)
 565     uid += 1
 566     thread = Task(uid, 9999, 9999, "framebuffer", False, 0)
 567     process.children.append(thread)
 568     uid += 1
 569     process = Task(uid, 9998, 9998, "System", True, 0)
 570     process_list.append(process)
 571     # if we don't find the real kernel, use this to keep things going
 572     kernel_uid = uid
 573     uid += 1
 574     thread = Task(uid, 9998, 9998, "System", False, 0)
 575     process.children.append(thread)
 576     uid += 1
 577
 578     ext = os.path.splitext(task_file)[1]
 579
 580     try:
 581         if ext == ".gz":
 582             process_file = gzip.open(task_file, 'rb')
 583         else:
 584             process_file = open(task_file, 'rb')
 585     except:
 586         print "ERROR opening task file:", task_file
 587         print "Make sure context switch task dumping is enabled in gem5."
 588         sys.exit(1)
 589
 590     process_re = re.compile("tick=(\d+)\s+(\d+)\s+cpu_id=(\d+)\s+" +
 591         "next_pid=([-\d]+)\s+next_tgid=([-\d]+)\s+next_task=(.*)")
 592
 593     task_name_failure_warned = False
 594
 595     for line in process_file:
 596         match = re.match(process_re, line)
 597         if match:
 598             tick = int(match.group(1))
 599             if (start_tick < 0):
 600                 start_tick = tick
 601             cpu_id = int(match.group(3))
 602             pid = int(match.group(4))
 603             tgid = int(match.group(5))
 604             task_name = match.group(6)
 605
 606             if not task_name_failure_warned:
 607                 if task_name == "FailureIn_curTaskName":
 608                     print "-------------------------------------------------"
 609                     print "WARNING: Task name not set correctly!"
 610                     print "Process/Thread info will not be displayed correctly"
 611                     print "Perhaps forgot to apply m5struct.patch to kernel?"
 612                     print "-------------------------------------------------"
 613                     task_name_failure_warned = True
 614
 615             if not tgid in process_dict:
 616                 if tgid == pid:
 617                     # new task is parent as well
 618                     if args.verbose:
 619                         print "new process", uid, pid, tgid, task_name
 620                     if tgid == 0:
 621                         # new process is the "idle" task
 622                         process = Task(uid, pid, tgid, "idle", True, tick)
 623                         idle_uid = 0
 624                     else:
 625                         process = Task(uid, pid, tgid, task_name, True, tick)
 626                 else:
 627                     if tgid == 0:
 628                         process = Task(uid, tgid, tgid, "idle", True, tick)
 629                         idle_uid = 0
 630                     else:
 631                         # parent process name not known yet
 632                         process = Task(uid, tgid, tgid, "_Unknown_", True, tick)
 633                 if tgid == -1: # kernel
 634                     kernel_uid = 0
 635                 uid += 1
 636                 process_dict[tgid] = process
 637                 process_list.append(process)
 638             else:
 639                 if tgid == pid:
 640                     if process_dict[tgid].task_name == "_Unknown_":
 641                         if args.verbose:
 642                             print "new process", \
 643                                 process_dict[tgid].uid, pid, tgid, task_name
 644                         process_dict[tgid].task_name = task_name
 645                     if process_dict[tgid].task_name != task_name and tgid != 0:
 646                         process_dict[tgid].task_name = task_name
 647
 648             if not pid in thread_dict:
 649                 if args.verbose:
 650                     print "new thread", \
 651                        uid, process_dict[tgid].uid, pid, tgid, task_name
 652                 thread = Task(uid, pid, tgid, task_name, False, tick)
 653                 uid += 1
 654                 thread_dict[pid] = thread
 655                 process_dict[tgid].children.append(thread)
 656             else:
 657                 if thread_dict[pid].task_name != task_name:
 658                     thread_dict[pid].task_name = task_name
 659
 660             if args.verbose:
 661                 print tick, uid, cpu_id, pid, tgid, task_name
 662
 663             task = thread_dict[pid]
 664             event = Event(tick, task)
 665             event_list[cpu_id].append(event)
 666             unified_event_list.append(event)
 667
 668             if len(unified_event_list) == num_events:
 669                 print "Truncating at", num_events, "events!"
 670                 break
 671     print "Found %d events." % len(unified_event_list)
 672
 673     for process in process_list:
 674         if process.pid > 9990: # fix up framebuffer ticks
 675             process.tick = start_tick
 676         print process.uid, process.pid, process.tgid, \
 677             process.task_name, str(process.tick)
 678         for thread in process.children:
 679             if thread.pid > 9990:
 680                 thread.tick = start_tick
 681             print "\t", thread.uid, thread.pid, thread.tgid, \
 682                 thread.task_name, str(thread.tick)
 683
 684     end_tick = tick
 685
 686     print "Start tick:", start_tick
 687     print "End tick:  ", end_tick
 688     print ""
 689
 690     return
 691
 692
 693 def initOutput(output_path):
 694     if not os.path.exists(output_path):
 695         os.mkdir(output_path)
 696
 697 def ticksToNs(tick):
 698     if ticks_in_ns < 0:
 699         print "ticks_in_ns not set properly!"
 700         sys.exit(1)
 701
 702     return tick / ticks_in_ns
 703
 704 def writeXmlFile(xml, filename):
 705     f = open(filename, "w")
 706     txt = ET.tostring(xml)
 707     f.write(minidom.parseString(txt).toprettyxml())
 708     f.close()
 709
 710
 711 # StatsEntry that contains individual statistics
 712 class StatsEntry(object):
 713     def __init__(self, name, group, group_index, per_cpu, key):
 714
 715         # Full name of statistics
 716         self.name = name
 717
 718         # Streamline group name that statistic will belong to
 719         self.group = group
 720
 721         # Index of statistics within group (used to change colors within groups)
 722         self.group_index = group_index
 723
 724         # Shorter name with "system" stripped off
 725         # and symbols converted to alphanumerics
 726         self.short_name = re.sub("system\.", "", name)
 727         self.short_name = re.sub(":", "_", name)
 728
 729         # Regex for this stat (string version used to construct union regex)
 730         self.regex_string = "^" + name + "\s+([\d\.]+)"
 731         self.regex = re.compile("^" + name + "\s+([\d\.e\-]+)\s+# (.*)$", re.M)
 732         self.description = ""
 733
 734         # Whether this stat is use per CPU or not
 735         self.per_cpu = per_cpu
 736
 737         # Key used in .apc protocol (as described in captured.xml)
 738         self.key = key
 739
 740         # List of values of stat per timestamp
 741         self.values = []
 742
 743         # Whether this stat has been found for the current timestamp
 744         self.found = False
 745
 746         # Whether this stat has been found at least once
 747         # (to suppress too many warnings)
 748         self.not_found_at_least_once = False
 749
 750         # Field used to hold ElementTree subelement for this stat
 751         self.ET_element = None
 752
 753         # Create per-CPU stat name and regex, etc.
 754         if self.per_cpu:
 755             self.per_cpu_regex_string = []
 756             self.per_cpu_regex = []
 757             self.per_cpu_name = []
 758             self.per_cpu_found = []
 759             for i in range(num_cpus):
 760                 if num_cpus > 1:
 761                     per_cpu_name = re.sub("#", str(i), self.name)
 762                 else:
 763                     per_cpu_name = re.sub("#", "", self.name)
 764
 765                 self.per_cpu_name.append(per_cpu_name)
 766                 print "\t", per_cpu_name
 767
 768                 self.per_cpu_regex_string.\
 769                     append("^" + per_cpu_name + "\s+[\d\.]+")
 770                 self.per_cpu_regex.append(re.compile("^" + per_cpu_name + \
 771                     "\s+([\d\.e\-]+)\s+# (.*)$", re.M))
 772                 self.values.append([])
 773                 self.per_cpu_found.append(False)
 774
 775     def append_value(self, val, per_cpu_index = None):
 776         if self.per_cpu:
 777             self.values[per_cpu_index].append(str(val))
 778         else:
 779             self.values.append(str(val))
 780
 781 # Global stats object that contains the list of stats entries
 782 # and other utility functions
 783 class Stats(object):
 784     def __init__(self):
 785         self.stats_list = []
 786         self.tick_list = []
 787         self.next_key = 1
 788
 789     def register(self, name, group, group_index, per_cpu):
 790         print "registering stat:", name, "group:", group, group_index
 791         self.stats_list.append(StatsEntry(name, group, group_index, per_cpu, \
 792             self.next_key))
 793         self.next_key += 1
 794
 795     # Union of all stats to accelerate parsing speed
 796     def createStatsRegex(self):
 797         regex_strings = [];
 798         print "\nnum entries in stats_list", len(self.stats_list)
 799         for entry in self.stats_list:
 800             if entry.per_cpu:
 801                 for i in range(num_cpus):
 802                     regex_strings.append(entry.per_cpu_regex_string[i])
 803             else:
 804                 regex_strings.append(entry.regex_string)
 805
 806         self.regex = re.compile('|'.join(regex_strings))
 807
 808
 809 def registerStats(config_file):
 810     print "==============================="
 811     print "Parsing stats config.ini file..."
 812     print config_file
 813     print "==============================="
 814
 815     config = ConfigParser()
 816     if not config.read(config_file):
 817         print "ERROR: config file '", config_file, "' not found!"
 818         sys.exit(1)
 819
 820     print "\nRegistering Stats..."
 821
 822     stats = Stats()
 823
 824     per_cpu_stat_groups = config.options('PER_CPU_STATS')
 825     for group in per_cpu_stat_groups:
 826         i = 0
 827         per_cpu_stats_list = config.get('PER_CPU_STATS', group).split('\n')
 828         for item in per_cpu_stats_list:
 829             if item:
 830                 stats.register(item, group, i, True)
 831                 i += 1
 832
 833     per_l2_stat_groups = config.options('PER_L2_STATS')
 834     for group in per_l2_stat_groups:
 835         i = 0
 836         per_l2_stats_list = config.get('PER_L2_STATS', group).split('\n')
 837         for item in per_l2_stats_list:
 838             if item:
 839                 for l2 in range(num_l2):
 840                     if num_l2 > 1:
 841                         name = re.sub("#", str(l2), item)
 842                     else:
 843                         name = re.sub("#", "", item)
 844                     stats.register(name, group, i, False)
 845                 i += 1
 846
 847     other_stat_groups = config.options('OTHER_STATS')
 848     for group in other_stat_groups:
 849         i = 0
 850         other_stats_list = config.get('OTHER_STATS', group).split('\n')
 851         for item in other_stats_list:
 852             if item:
 853                 stats.register(item, group, i, False)
 854                 i += 1
 855
 856     stats.createStatsRegex()
 857
 858     return stats
 859
 860 # Parse and read in gem5 stats file
 861 # Streamline counters are organized per CPU
 862 def readGem5Stats(stats, gem5_stats_file):
 863     print "\n==============================="
 864     print "Parsing gem5 stats file..."
 865     print gem5_stats_file
 866     print "===============================\n"
 867     ext = os.path.splitext(gem5_stats_file)[1]
 868
 869     window_start_regex = \
 870         re.compile("^---------- Begin Simulation Statistics ----------")
 871     window_end_regex = \
 872         re.compile("^---------- End Simulation Statistics   ----------")
 873     final_tick_regex = re.compile("^final_tick\s+(\d+)")
 874
 875     global ticks_in_ns
 876     sim_freq_regex = re.compile("^sim_freq\s+(\d+)")
 877     sim_freq = -1
 878
 879     try:
 880         if ext == ".gz":
 881             f = gzip.open(gem5_stats_file, "r")
 882         else:
 883             f = open(gem5_stats_file, "r")
 884     except:
 885         print "ERROR opening stats file", gem5_stats_file, "!"
 886         sys.exit(1)
 887
 888     stats_not_found_list = stats.stats_list[:]
 889     window_num = 0
 890
 891     while (True):
 892         error = False
 893         try:
 894             line = f.readline()
 895         except IOError:
 896             print ""
 897             print "WARNING: IO error in stats file"
 898             print "(gzip stream not closed properly?)...continuing for now"
 899             error = True
 900         if not line:
 901             break
 902
 903         # Find out how many gem5 ticks in 1ns
 904         if sim_freq < 0:
 905             m = sim_freq_regex.match(line)
 906             if m:
 907                 sim_freq = int(m.group(1)) # ticks in 1 sec
 908                 ticks_in_ns = int(sim_freq / 1e9)
 909                 print "Simulation frequency found! 1 tick == %e sec\n" \
 910                         % (1.0 / sim_freq)
 911
 912         # Final tick in gem5 stats: current absolute timestamp
 913         m = final_tick_regex.match(line)
 914         if m:
 915             tick = int(m.group(1))
 916             if tick > end_tick:
 917                 break
 918             stats.tick_list.append(tick)
 919
 920
 921         if (window_end_regex.match(line) or error):
 922             if args.verbose:
 923                 print "new window"
 924             for stat in stats.stats_list:
 925                 if stat.per_cpu:
 926                     for i in range(num_cpus):
 927                         if not stat.per_cpu_found[i]:
 928                             if not stat.not_found_at_least_once:
 929                                 print "WARNING: stat not found in window #", \
 930                                     window_num, ":", stat.per_cpu_name[i]
 931                                 print "suppressing further warnings for " + \
 932                                     "this stat"
 933                                 stat.not_found_at_least_once = True
 934                             stat.values[i].append(str(0))
 935                         stat.per_cpu_found[i] = False
 936                 else:
 937                     if not stat.found:
 938                         if not stat.not_found_at_least_once:
 939                             print "WARNING: stat not found in window #", \
 940                                 window_num, ":", stat.name
 941                             print "suppressing further warnings for this stat"
 942                             stat.not_found_at_least_once = True
 943                         stat.values.append(str(0))
 944                     stat.found = False
 945             stats_not_found_list = stats.stats_list[:]
 946             window_num += 1
 947             if error:
 948                 break
 949
 950         # Do a single regex of the union of all stats first for speed
 951         if stats.regex.match(line):
 952             # Then loop through only the stats we haven't seen in this window
 953             for stat in stats_not_found_list[:]:
 954                 if stat.per_cpu:
 955                     for i in range(num_cpus):
 956                         m = stat.per_cpu_regex[i].match(line)
 957                         if m:
 958                             if stat.name == "ipc":
 959                                 value = str(int(float(m.group(1)) * 1000))
 960                             else:
 961                                 value = str(int(float(m.group(1))))
 962                             if args.verbose:
 963                                 print stat.per_cpu_name[i], value
 964                             stat.values[i].append(value)
 965                             stat.per_cpu_found[i] = True
 966                             all_found = True
 967                             for j in range(num_cpus):
 968                                 if not stat.per_cpu_found[j]:
 969                                     all_found = False
 970                             if all_found:
 971                                 stats_not_found_list.remove(stat)
 972                             if stat.description == "":
 973                                 stat.description = m.group(2)
 974                 else:
 975                     m = stat.regex.match(line)
 976                     if m:
 977                         value = str(int(float(m.group(1))))
 978                         if args.verbose:
 979                             print stat.name, value
 980                         stat.values.append(value)
 981                         stat.found = True
 982                         stats_not_found_list.remove(stat)
 983                         if stat.description == "":
 984                             stat.description = m.group(2)
 985     f.close()
 986
 987
 988 # Create session.xml file in .apc folder
 989 def doSessionXML(output_path):
 990     session_file = output_path + "/session.xml"
 991
 992     xml = ET.Element("session")
 993
 994     xml.set("version", "1")
 995     xml.set("call_stack_unwinding", "no")
 996     xml.set("parse_debug_info", "no")
 997     xml.set("high_resolution", "yes")
 998     xml.set("buffer_mode", "streaming")
 999     xml.set("sample_rate", "low")
1000
1001     # Setting duration to zero for now. Doesn't affect visualization.
1002     xml.set("duration", "0")
1003
1004     xml.set("target_host", "")
1005     xml.set("target_port", "8080")
1006
1007     writeXmlFile(xml, session_file)
1008
1009
1010 # Create captured.xml file in .apc folder
1011 def doCapturedXML(output_path, stats):
1012     captured_file = output_path + "/captured.xml"
1013
1014     xml = ET.Element("captured")
1015     xml.set("version", "1")
1016     xml.set("protocol", "17")
1017     xml.set("backtrace_processing", "none")
1018
1019     target = ET.SubElement(xml, "target")
1020     target.set("name", "gem5")
1021     target.set("sample_rate", "1000")
1022     target.set("cores", str(num_cpus))
1023
1024     counters = ET.SubElement(xml, "counters")
1025     for stat in stats.stats_list:
1026         s = ET.SubElement(counters, "counter")
1027         stat_name = re.sub("\.", "_", stat.short_name)
1028         stat_name = re.sub("#", "", stat_name)
1029         s.set("title", stat.group)
1030         s.set("name", stat_name)
1031         s.set("color", "0x00000000")
1032         s.set("key", "0x%08x" % stat.key)
1033         s.set("type", stat_name)
1034         s.set("event", "0x00000000")
1035         if stat.per_cpu:
1036             s.set("per_cpu", "yes")
1037         else:
1038             s.set("per_cpu", "no")
1039         s.set("display", "")
1040         s.set("units", "")
1041         s.set("average_selection", "no")
1042         s.set("description", stat.description)
1043
1044     writeXmlFile(xml, captured_file)
1045
1046 # Writes out Streamline cookies (unique IDs per process/thread)
1047 def writeCookiesThreads(blob):
1048     thread_list = []
1049     for process in process_list:
1050         if process.uid > 0:
1051             print "cookie", process.task_name, process.uid
1052             writeBinary(blob, cookieNameFrame(process.uid, process.task_name))
1053
1054         # pid and tgid need to be positive values -- no longer true?
1055         for thread in process.children:
1056             thread_list.append(thread)
1057
1058     # Threads need to be sorted in timestamp order
1059     thread_list.sort(key = lambda x: x.tick)
1060     for thread in thread_list:
1061         print "thread", thread.task_name, (ticksToNs(thread.tick)),\
1062                 thread.tgid, thread.pid
1063         writeBinary(blob, threadNameFrame(ticksToNs(thread.tick),\
1064                 thread.pid, thread.task_name))
1065
1066 # Writes context switch info as Streamline scheduling events
1067 def writeSchedEvents(blob):
1068     for cpu in range(num_cpus):
1069         for event in event_list[cpu]:
1070             timestamp = ticksToNs(event.tick)
1071             pid = event.task.tgid
1072             tid = event.task.pid
1073             if process_dict.has_key(event.task.tgid):
1074                 cookie = process_dict[event.task.tgid].uid
1075             else:
1076                 cookie = 0
1077
1078             # State:
1079             #   0: waiting on other event besides I/O
1080             #   1: Contention/pre-emption
1081             #   2: Waiting on I/O
1082             #   3: Waiting on mutex
1083             # Hardcoding to 0 for now. Other states not implemented yet.
1084             state = 0
1085
1086             if args.verbose:
1087                 print cpu, timestamp, pid, tid, cookie
1088
1089             writeBinary(blob,\
1090                 schedSwitchFrame(cpu, timestamp, pid, tid, cookie, state))
1091
1092 # Writes selected gem5 statistics as Streamline counters
1093 def writeCounters(blob, stats):
1094     timestamp_list = []
1095     for tick in stats.tick_list:
1096         if tick > end_tick:
1097             break
1098         timestamp_list.append(ticksToNs(tick))
1099
1100     for stat in stats.stats_list:
1101         if stat.per_cpu:
1102             stat_length = len(stat.values[0])
1103         else:
1104             stat_length = len(stat.values)
1105
1106     for n in range(len(timestamp_list)):
1107         for stat in stats.stats_list:
1108             if stat.per_cpu:
1109                 for i in range(num_cpus):
1110                     writeBinary(blob, counterFrame(timestamp_list[n], i, \
1111                                     stat.key, int(float(stat.values[i][n]))))
1112             else:
1113                 writeBinary(blob, counterFrame(timestamp_list[n], 0, \
1114                                     stat.key, int(float(stat.values[n]))))
1115
1116 # Streamline can display LCD frame buffer dumps (gzipped bmp)
1117 # This function converts the frame buffer dumps to the Streamline format
1118 def writeVisualAnnotations(blob, input_path, output_path):
1119     frame_path = input_path + "/frames_system.vncserver"
1120     if not os.path.exists(frame_path):
1121         return
1122
1123     frame_count = 0
1124     file_list = os.listdir(frame_path)
1125     file_list.sort()
1126     re_fb = re.compile("fb\.(\d+)\.(\d+)\.bmp.gz")
1127
1128     # Use first non-negative pid to tag visual annotations
1129     annotate_pid = -1
1130     for e in unified_event_list:
1131         pid = e.task.pid
1132         if pid >= 0:
1133             annotate_pid = pid
1134             break
1135
1136     for fn in file_list:
1137         m = re_fb.match(fn)
1138         if m:
1139             seq = m.group(1)
1140             tick = int(m.group(2))
1141             if tick > end_tick:
1142                 break
1143             frame_count += 1
1144
1145             userspace_body = []
1146             userspace_body += packed32(0x1C) # escape code
1147             userspace_body += packed32(0x04) # visual code
1148
1149             text_annotation = "image_" + str(ticksToNs(tick)) + ".bmp.gz"
1150             userspace_body += int16(len(text_annotation))
1151             userspace_body += utf8StringList(text_annotation)
1152
1153             if gzipped_bmp_supported:
1154                 # copy gzipped bmp directly
1155                 bytes_read = open(frame_path + "/" + fn, "rb").read()
1156             else:
1157                 # copy uncompressed bmp
1158                 bytes_read = gzip.open(frame_path + "/" + fn, "rb").read()
1159
1160             userspace_body += int32(len(bytes_read))
1161             userspace_body += bytes_read
1162
1163             writeBinary(blob, annotateFrame(0, annotate_pid, ticksToNs(tick), \
1164                                 len(userspace_body), userspace_body))
1165
1166     print "\nfound", frame_count, "frames for visual annotation.\n"
1167
1168
1169 def createApcProject(input_path, output_path, stats):
1170     initOutput(output_path)
1171
1172     blob = open(output_path + "/0000000000", "wb")
1173
1174     # Summary frame takes current system time and system uptime.
1175     # Filling in with random values for now.
1176     writeBinary(blob, summaryFrame(1234, 5678))
1177
1178     writeCookiesThreads(blob)
1179
1180     print "writing Events"
1181     writeSchedEvents(blob)
1182
1183     print "writing Counters"
1184     writeCounters(blob, stats)
1185
1186     print "writing Visual Annotations"
1187     writeVisualAnnotations(blob, input_path, output_path)
1188
1189     doSessionXML(output_path)
1190     doCapturedXML(output_path, stats)
1191
1192     blob.close()
1193
1194
1195
1196 #######################
1197 # Main Routine
1198
1199 input_path = args.input_path
1200 output_path = args.output_path
1201
1202 ####
1203 # Make sure input path exists
1204 ####
1205 if not os.path.exists(input_path):
1206     print "ERROR: Input path %s does not exist!" % input_path
1207     sys.exit(1)
1208
1209 ####
1210 # Parse gem5 configuration file to find # of CPUs and L2s
1211 ####
1212 (num_cpus, num_l2) = parseConfig(input_path + "/config.ini")
1213
1214 ####
1215 # Parse task file to find process/thread info
1216 ####
1217 parseProcessInfo(input_path + "/system.tasks.txt")
1218
1219 ####
1220 # Parse stat config file and register stats
1221 ####
1222 stat_config_file = args.stat_config_file
1223 stats = registerStats(stat_config_file)
1224
1225 ####
1226 # Parse gem5 stats
1227 ####
1228 # Check if both stats.txt and stats.txt.gz exist and warn if both exist
1229 if os.path.exists(input_path + "/stats.txt") and \
1230     os.path.exists(input_path + "/stats.txt.gz"):
1231     print "WARNING: Both stats.txt.gz and stats.txt exist. \
1232             Using stats.txt.gz by default."
1233
1234 gem5_stats_file = input_path + "/stats.txt.gz"
1235 if not os.path.exists(gem5_stats_file):
1236     gem5_stats_file = input_path + "/stats.txt"
1237 if not os.path.exists(gem5_stats_file):
1238     print "ERROR: stats.txt[.gz] file does not exist in %s!" % input_path
1239     sys.exit(1)
1240
1241 readGem5Stats(stats, gem5_stats_file)
1242
1243 ####
1244 # Create Streamline .apc project folder
1245 ####
1246 createApcProject(input_path, output_path, stats)
1247
1248 print "All done!"