util/minorview/model.py

   1 # Copyright (c) 2013 ARM Limited
   2 # All rights reserved
   3 #
   4 # The license below extends only to copyright in the software and shall
   5 # not be construed as granting a license to any other intellectual
   6 # property including but not limited to intellectual property relating
   7 # to a hardware implementation of the functionality of the software
   8 # licensed hereunder.  You may use the software subject to the license
   9 # terms below provided that you ensure that this notice is replicated
  10 # unmodified and in its entirety in all distributions of the software,
  11 # modified or unmodified, in source code or in binary form.
  12 #
  13 # Redistribution and use in source and binary forms, with or without
  14 # modification, are permitted provided that the following conditions are
  15 # met: redistributions of source code must retain the above copyright
  16 # notice, this list of conditions and the following disclaimer;
  17 # redistributions in binary form must reproduce the above copyright
  18 # notice, this list of conditions and the following disclaimer in the
  19 # documentation and/or other materials provided with the distribution;
  20 # neither the name of the copyright holders nor the names of its
  21 # contributors may be used to endorse or promote products derived from
  22 # this software without specific prior written permission.
  23 #
  24 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  25 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  26 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  27 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  28 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  29 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  30 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  31 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  32 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  33 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  34 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  35
  36 import parse
  37 import colours
  38 from colours import unknownColour
  39 from point import Point
  40 import re
  41 import blobs
  42 from time import time as wall_time
  43 import os
  44
  45 id_parts = "TSPLFE"
  46
  47 all_ids = set(id_parts)
  48 no_ids = set([])
  49
  50 class BlobDataSelect(object):
  51     """Represents which data is displayed for Ided object"""
  52     def __init__(self):
  53         # Copy all_ids
  54         self.ids = set(all_ids)
  55
  56     def __and__(self, rhs):
  57         """And for filtering"""
  58         ret = BlobDataSelect()
  59         ret.ids = self.ids.intersection(rhs.ids)
  60         return ret
  61
  62 class BlobVisualData(object):
  63     """Super class for block data colouring"""
  64     def to_striped_block(self, select):
  65         """Return an array of colours to use for a striped block"""
  66         return unknownColour
  67
  68     def get_inst(self):
  69         """Get an instruction Id (if any) from this data"""
  70         return None
  71
  72     def get_line(self):
  73         """Get a line Id (if any) from this data"""
  74         return None
  75
  76     def __repr__(self):
  77         return self.__class__.__name__ + '().from_string(' + \
  78             self.__str__() + ')'
  79
  80     def __str__(self):
  81         return ''
  82
  83 class Id(BlobVisualData):
  84     """A line or instruction id"""
  85     def __init__(self):
  86         self.isFault = False
  87         self.threadId = 0
  88         self.streamSeqNum = 0
  89         self.predictionSeqNum = 0
  90         self.lineSeqNum = 0
  91         self.fetchSeqNum = 0
  92         self.execSeqNum = 0
  93
  94     def as_list(self):
  95         return [self.threadId, self.streamSeqNum, self.predictionSeqNum,
  96             self.lineSeqNum, self.fetchSeqNum, self.execSeqNum]
  97
  98     def __cmp__(self, right):
  99         return cmp(self.as_list(), right.as_list())
 100
 101     def from_string(self, string):
 102         m = re.match('^(F;)?(\d+)/(\d+)\.(\d+)/(\d+)(/(\d+)(\.(\d+))?)?',
 103             string)
 104
 105         def seqnum_from_string(string):
 106             if string is None:
 107                 return 0
 108             else:
 109                 return int(string)
 110
 111         if m is None:
 112             print 'Invalid Id string', string
 113         else:
 114             elems = m.groups()
 115
 116             if elems[0] is not None:
 117                 self.isFault = True
 118             else:
 119                 self.isFault = False
 120
 121             self.threadId = seqnum_from_string(elems[1])
 122             self.streamSeqNum = seqnum_from_string(elems[2])
 123             self.predictionSeqNum = seqnum_from_string(elems[3])
 124             self.lineSeqNum = seqnum_from_string(elems[4])
 125             self.fetchSeqNum = seqnum_from_string(elems[6])
 126             self.execSeqNum = seqnum_from_string(elems[8])
 127         return self
 128
 129     def get_inst(self):
 130         if self.fetchSeqNum != 0:
 131             return self
 132         else:
 133             return None
 134
 135     def get_line(self):
 136         return self
 137
 138     def __str__(self):
 139         """Returns the usual id T/S.P/L/F.E string"""
 140         return (
 141             str(self.threadId) + '/' +
 142             str(self.streamSeqNum) + '.' +
 143             str(self.predictionSeqNum) + '/' +
 144             str(self.lineSeqNum) + '/' +
 145             str(self.fetchSeqNum) + '.' +
 146             str(self.execSeqNum))
 147
 148     def to_striped_block(self, select):
 149         ret = []
 150
 151         if self.isFault:
 152             ret.append(colours.faultColour)
 153
 154         if 'T' in select.ids:
 155             ret.append(colours.number_to_colour(self.threadId))
 156         if 'S' in select.ids:
 157             ret.append(colours.number_to_colour(self.streamSeqNum))
 158         if 'P' in select.ids:
 159             ret.append(colours.number_to_colour(self.predictionSeqNum))
 160         if 'L' in select.ids:
 161             ret.append(colours.number_to_colour(self.lineSeqNum))
 162         if self.fetchSeqNum != 0 and 'F' in select.ids:
 163             ret.append(colours.number_to_colour(self.fetchSeqNum))
 164         if self.execSeqNum != 0 and 'E' in select.ids:
 165             ret.append(colours.number_to_colour(self.execSeqNum))
 166
 167         if len(ret) == 0:
 168             ret = [colours.unknownColour]
 169
 170         if self.isFault:
 171             ret.append(colours.faultColour)
 172
 173         return ret
 174
 175 class Branch(BlobVisualData):
 176     """Branch data new stream and prediction sequence numbers, a branch
 177     reason and a new PC"""
 178     def __init__(self):
 179         self.newStreamSeqNum = 0
 180         self.newPredictionSeqNum = 0
 181         self.newPC = 0
 182         self.reason = "NoBranch"
 183         self.id = Id()
 184
 185     def from_string(self, string):
 186         m = re.match('^(\w+);(\d+)\.(\d+);([0-9a-fA-Fx]+);(.*)$', string)
 187
 188         if m is not None:
 189             self.reason, newStreamSeqNum, newPredictionSeqNum, \
 190                 newPC, id = m.groups()
 191
 192             self.newStreamSeqNum = int(newStreamSeqNum)
 193             self.newPredictionSeqNum = int(newPredictionSeqNum)
 194             self.newPC = int(newPC, 0)
 195             self.id = special_view_decoder(Id)(id)
 196             # self.branch = special_view_decoder(Branch)(branch)
 197         else:
 198             print "Bad Branch data:", string
 199         return self
 200
 201     def to_striped_block(self, select):
 202         return [colours.number_to_colour(self.newStreamSeqNum),
 203             colours.number_to_colour(self.newPredictionSeqNum),
 204             colours.number_to_colour(self.newPC)]
 205
 206 class Counts(BlobVisualData):
 207     """Treat the input data as just a /-separated list of count values (or
 208     just a single value)"""
 209     def __init__(self):
 210         self.counts = []
 211
 212     def from_string(self, string):
 213         self.counts = map(int, re.split('/', string))
 214         return self
 215
 216     def to_striped_block(self, select):
 217         return map(colours.number_to_colour, self.counts)
 218
 219 class Colour(BlobVisualData):
 220     """A fixed colour block, used for special colour decoding"""
 221     def __init__(self, colour):
 222         self.colour = colour
 223
 224     def to_striped_block(self, select):
 225         return [self.colour]
 226
 227 class DcacheAccess(BlobVisualData):
 228     """Data cache accesses [RW];id"""
 229     def __init__(self):
 230         self.direc = 'R'
 231         self.id = Id()
 232
 233     def from_string(self, string):
 234         self.direc, id = re.match('^([RW]);([^;]*);.*$', string).groups()
 235         self.id.from_string(id)
 236         return self
 237
 238     def get_inst(self):
 239         return self.id
 240
 241     def to_striped_block(self, select):
 242         if self.direc == 'R':
 243             direc_colour = colours.readColour
 244         elif self.direc == 'R':
 245             direc_colour = colours.writeColour
 246         else:
 247             direc_colour = colours.errorColour
 248         return [direc_colour] + self.id.to_striped_block(select)
 249
 250 class ColourPattern(object):
 251     """Super class for decoders that make 2D grids rather than just single
 252     striped blocks"""
 253     def elems(self):
 254         return []
 255
 256     def to_striped_block(self, select):
 257         return [[[colours.errorColour]]]
 258
 259 def special_view_decoder(class_):
 260     """Generate a decode function that checks for special character
 261     arguments first (and generates a fixed colour) before building a
 262     BlobVisualData of the given class"""
 263     def decode(symbol):
 264         if symbol in special_state_colours:
 265             return Colour(special_state_colours[symbol])
 266         else:
 267             return class_().from_string(symbol)
 268     return decode
 269
 270 class TwoDColours(ColourPattern):
 271     """A 2D grid pattern decoder"""
 272     def __init__(self, blockss):
 273         self.blockss = blockss
 274
 275     @classmethod
 276     def decoder(class_, elemClass, dataName):
 277         """Factory for making decoders for particular block types"""
 278         def decode(pairs):
 279             if dataName not in pairs:
 280                 print 'TwoDColours: no event data called:', \
 281                     dataName, 'in:', pairs
 282                 return class_([[Colour(colours.errorColour)]])
 283             else:
 284                 parsed = parse.list_parser(pairs[dataName])
 285                 return class_(parse.map2(special_view_decoder(elemClass), \
 286                     parsed))
 287         return decode
 288
 289     @classmethod
 290     def indexed_decoder(class_, elemClass, dataName, picPairs):
 291         """Factory for making decoders for particular block types but
 292         where the list elements are pairs of (index, data) and
 293         strip and stripelems counts are picked up from the pair
 294         data on the decoder's picture file.  This gives a 2D layout
 295         of the values with index 0 at strip=0, elem=0 and index 1
 296         at strip=0, elem=1"""
 297         def decode(pairs):
 298             if dataName not in pairs:
 299                 print 'TwoDColours: no event data called:', \
 300                     dataName, 'in:', pairs
 301                 return class_([[Colour(colours.errorColour)]])
 302             else:
 303                 strips = int(picPairs['strips'])
 304                 strip_elems = int(picPairs['stripelems'])
 305
 306                 raw_iv_pairs = pairs[dataName]
 307
 308                 parsed = parse.parse_indexed_list(raw_iv_pairs)
 309
 310                 array = [[Colour(colours.emptySlotColour)
 311                     for i in xrange(0, strip_elems)]
 312                     for j in xrange(0, strips)]
 313
 314                 for index, value in parsed:
 315                     try:
 316                         array[index % strips][index / strips] = \
 317                             special_view_decoder(elemClass)(value)
 318                     except:
 319                         print "Element out of range strips: %d," \
 320                             " stripelems %d, index: %d" % (strips,
 321                             strip_elems, index)
 322
 323                 # return class_(array)
 324                 return class_(array)
 325         return decode
 326
 327     def elems(self):
 328         """Get a flat list of all elements"""
 329         ret = []
 330         for blocks in self.blockss:
 331             ret += blocks
 332         return ret
 333
 334     def to_striped_block(self, select):
 335         return parse.map2(lambda d: d.to_striped_block(select), self.blockss)
 336
 337 class FrameColours(ColourPattern):
 338     """Decode to a 2D grid which has a single occupied row from the event
 339     data and some blank rows forming a frame with the occupied row as a
 340     'title' coloured stripe"""
 341     def __init__(self, block, numBlankSlots):
 342         self.numBlankSlots = numBlankSlots
 343         self.block = block
 344
 345     @classmethod
 346     def decoder(class_, elemClass, numBlankSlots, dataName):
 347         """Factory for element type"""
 348         def decode(pairs):
 349             if dataName not in pairs:
 350                 print 'FrameColours: no event data called:', dataName, \
 351                     'in:', pairs
 352                 return class_([Colour(colours.errorColour)])
 353             else:
 354                 parsed = parse.list_parser(pairs[dataName])
 355                 return class_(special_view_decoder(elemClass)
 356                     (parsed[0][0]), numBlankSlots)
 357         return decode
 358
 359     def elems(self):
 360         return [self.block]
 361
 362     def to_striped_block(self, select):
 363         return ([[self.block.to_striped_block(select)]] +
 364             (self.numBlankSlots * [[[colours.backgroundColour]]]))
 365
 366 special_state_colours = {
 367     'U': colours.unknownColour,
 368     'B': colours.blockedColour,
 369     '-': colours.bubbleColour,
 370     '': colours.emptySlotColour,
 371     'E': colours.emptySlotColour,
 372     'R': colours.reservedSlotColour,
 373     'X': colours.errorColour,
 374     'F': colours.faultColour,
 375     'r': colours.readColour,
 376     'w': colours.writeColour
 377     }
 378
 379 special_state_names = {
 380     'U': '(U)nknown',
 381     'B': '(B)locked',
 382     '-': '(-)Bubble',
 383     '': '()Empty',
 384     'E': '(E)mpty',
 385     'R': '(R)eserved',
 386     'X': '(X)Error',
 387     'F': '(F)ault',
 388     'r': '(r)ead',
 389     'w': '(w)rite'
 390     }
 391
 392 special_state_chars = special_state_colours.keys()
 393
 394 # The complete set of available block data types
 395 decoder_element_classes = {
 396     'insts': Id,
 397     'lines': Id,
 398     'branch': Branch,
 399     'dcache': DcacheAccess,
 400     'counts': Counts
 401     }
 402
 403 indexed_decoder_element_classes = {
 404     'indexedCounts' : Counts
 405     }
 406
 407 def find_colour_decoder(stripSpace, decoderName, dataName, picPairs):
 408     """Make a colour decoder from some picture file blob attributes"""
 409     if decoderName == 'frame':
 410         return FrameColours.decoder(Counts, stripSpace, dataName)
 411     elif decoderName in decoder_element_classes:
 412         return TwoDColours.decoder(decoder_element_classes[decoderName],
 413             dataName)
 414     elif decoderName in indexed_decoder_element_classes:
 415         return TwoDColours.indexed_decoder(
 416             indexed_decoder_element_classes[decoderName], dataName, picPairs)
 417     else:
 418         return None
 419
 420 class IdedObj(object):
 421     """An object identified by an Id carrying paired data.
 422     The super class for Inst and Line"""
 423
 424     def __init__(self, id, pairs={}):
 425         self.id = id
 426         self.pairs = pairs
 427
 428     def __cmp__(self, right):
 429         return cmp(self.id, right.id)
 430
 431     def table_line(self):
 432         """Represent the object as a list of table row data"""
 433         return []
 434
 435     # FIXME, add a table column titles?
 436
 437     def __repr__(self):
 438         return ' '.join(self.table_line())
 439
 440 class Inst(IdedObj):
 441     """A non-fault instruction"""
 442     def __init__(self, id, disassembly, addr, pairs={}):
 443         super(Inst,self).__init__(id, pairs)
 444         if 'nextAddr' in pairs:
 445             self.nextAddr = int(pairs['nextAddr'], 0)
 446             del pairs['nextAddr']
 447         else:
 448             self.nextAddr = None
 449         self.disassembly = disassembly
 450         self.addr = addr
 451
 452     def table_line(self):
 453         if self.nextAddr is not None:
 454             addrStr = '0x%x->0x%x' % (self.addr, self.nextAddr)
 455         else:
 456             addrStr = '0x%x' % self.addr
 457         ret = [addrStr, self.disassembly]
 458         for name, value in self.pairs.iteritems():
 459             ret.append("%s=%s" % (name, str(value)))
 460         return ret
 461
 462 class InstFault(IdedObj):
 463     """A fault instruction"""
 464     def __init__(self, id, fault, addr, pairs={}):
 465         super(InstFault,self).__init__(id, pairs)
 466         self.fault = fault
 467         self.addr = addr
 468
 469     def table_line(self):
 470         ret = ["0x%x" % self.addr, self.fault]
 471         for name, value in self.pairs:
 472             ret.append("%s=%s", name, str(value))
 473         return ret
 474
 475 class Line(IdedObj):
 476     """A fetched line"""
 477     def __init__(self, id, vaddr, paddr, size, pairs={}):
 478         super(Line,self).__init__(id, pairs)
 479         self.vaddr = vaddr
 480         self.paddr = paddr
 481         self.size = size
 482
 483     def table_line(self):
 484         ret = ["0x%x/0x%x" % (self.vaddr, self.paddr), "%d" % self.size]
 485         for name, value in self.pairs:
 486             ret.append("%s=%s", name, str(value))
 487         return ret
 488
 489 class LineFault(IdedObj):
 490     """A faulting line"""
 491     def __init__(self, id, fault, vaddr, pairs={}):
 492         super(LineFault,self).__init__(id, pairs)
 493         self.vaddr = vaddr
 494         self.fault = fault
 495
 496     def table_line(self):
 497         ret = ["0x%x" % self.vaddr, self.fault]
 498         for name, value in self.pairs:
 499             ret.append("%s=%s", name, str(value))
 500         return ret
 501
 502 class BlobEvent(object):
 503     """Time event for a single blob"""
 504     def __init__(self, unit, time, pairs = {}):
 505         # blob's unit name
 506         self.unit = unit
 507         self.time = time
 508         # dict of picChar (blob name) to visual data
 509         self.visuals = {}
 510         # Miscellaneous unparsed MinorTrace line data
 511         self.pairs = pairs
 512         # Non-MinorTrace debug printout for this unit at this time
 513         self.comments = []
 514
 515     def find_ided_objects(self, model, picChar, includeInstLines):
 516         """Find instructions/lines mentioned in the blob's event
 517         data"""
 518         ret = []
 519         if picChar in self.visuals:
 520             blocks = self.visuals[picChar].elems()
 521             def find_inst(data):
 522                 instId = data.get_inst()
 523                 lineId = data.get_line()
 524                 if instId is not None:
 525                     inst = model.find_inst(instId)
 526                     line = model.find_line(instId)
 527                     if inst is not None:
 528                         ret.append(inst)
 529                     if includeInstLines and line is not None:
 530                         ret.append(line)
 531                 elif lineId is not None:
 532                     line = model.find_line(lineId)
 533                     if line is not None:
 534                         ret.append(line)
 535             map(find_inst, blocks)
 536         return sorted(ret)
 537
 538 class BlobModel(object):
 539     """Model bringing together blob definitions and parsed events"""
 540     def __init__(self, unitNamePrefix=''):
 541         self.blobs = []
 542         self.unitNameToBlobs = {}
 543         self.unitEvents = {}
 544         self.clear_events()
 545         self.picSize = Point(20,10)
 546         self.lastTime = 0
 547         self.unitNamePrefix = unitNamePrefix
 548
 549     def clear_events(self):
 550         """Drop all events and times"""
 551         self.lastTime = 0
 552         self.times = []
 553         self.insts = {}
 554         self.lines = {}
 555         self.numEvents = 0
 556
 557         for unit, events in self.unitEvents.iteritems():
 558             self.unitEvents[unit] = []
 559
 560     def add_blob(self, blob):
 561         """Add a parsed blob to the model"""
 562         self.blobs.append(blob)
 563         if blob.unit not in self.unitNameToBlobs:
 564             self.unitNameToBlobs[blob.unit] = []
 565
 566         self.unitNameToBlobs[blob.unit].append(blob)
 567
 568     def add_inst(self, inst):
 569         """Add a MinorInst instruction definition to the model"""
 570         # Is this a non micro-op instruction.  Microops (usually) get their
 571         #   fetchSeqNum == 0 varient stored first
 572         macroop_key = (inst.id.fetchSeqNum, 0)
 573         full_key = (inst.id.fetchSeqNum, inst.id.execSeqNum)
 574
 575         if inst.id.execSeqNum != 0 and macroop_key not in self.insts:
 576             self.insts[macroop_key] = inst
 577
 578         self.insts[full_key] = inst
 579
 580     def find_inst(self, id):
 581         """Find an instruction either as a microop or macroop"""
 582         macroop_key = (id.fetchSeqNum, 0)
 583         full_key = (id.fetchSeqNum, id.execSeqNum)
 584
 585         if full_key in self.insts:
 586             return self.insts[full_key]
 587         elif macroop_key in self.insts:
 588             return self.insts[macroop_key]
 589         else:
 590             return None
 591
 592     def add_line(self, line):
 593         """Add a MinorLine line to the model"""
 594         self.lines[line.id.lineSeqNum] = line
 595
 596     def add_unit_event(self, event):
 597         """Add a single event to the model.  This must be an event at a
 598         time >= the current maximum time"""
 599         if event.unit in self.unitEvents:
 600             events = self.unitEvents[event.unit]
 601             if len(events) > 0 and events[len(events)-1].time > event.time:
 602                 print "Bad event ordering"
 603             events.append(event)
 604         self.numEvents += 1
 605         self.lastTime = max(self.lastTime, event.time)
 606
 607     def extract_times(self):
 608         """Extract a list of all the times from the seen events.  Call after
 609         reading events to give a safe index list to use for time indices"""
 610         times = {}
 611         for unitEvents in self.unitEvents.itervalues():
 612             for event in unitEvents:
 613                 times[event.time] = 1
 614         self.times = times.keys()
 615         self.times.sort()
 616
 617     def find_line(self, id):
 618         """Find a line by id"""
 619         key = id.lineSeqNum
 620         return self.lines.get(key, None)
 621
 622     def find_event_bisection(self, unit, time, events,
 623         lower_index, upper_index):
 624         """Find an event by binary search on time indices"""
 625         while lower_index <= upper_index:
 626             pivot = (upper_index + lower_index) / 2
 627             pivotEvent = events[pivot]
 628             event_equal = (pivotEvent.time == time or
 629                 (pivotEvent.time < time and
 630                     (pivot == len(events) - 1 or
 631                         events[pivot + 1].time > time)))
 632
 633             if event_equal:
 634                 return pivotEvent
 635             elif time > pivotEvent.time:
 636                 if pivot == upper_index:
 637                     return None
 638                 else:
 639                     lower_index = pivot + 1
 640             elif time < pivotEvent.time:
 641                 if pivot == lower_index:
 642                     return None
 643                 else:
 644                     upper_index = pivot - 1
 645             else:
 646                 return None
 647         return None
 648
 649     def find_unit_event_by_time(self, unit, time):
 650         """Find the last event for the given unit at time <= time"""
 651         if unit in self.unitEvents:
 652             events = self.unitEvents[unit]
 653             ret = self.find_event_bisection(unit, time, events,
 654                 0, len(events)-1)
 655
 656             return ret
 657         else:
 658             return None
 659
 660     def find_time_index(self, time):
 661         """Find a time index close to the given time (where
 662         times[return] <= time and times[return+1] > time"""
 663         ret = 0
 664         lastIndex = len(self.times) - 1
 665         while ret < lastIndex and self.times[ret + 1] <= time:
 666             ret += 1
 667         return ret
 668
 669     def add_minor_inst(self, rest):
 670         """Parse and add a MinorInst line to the model"""
 671         pairs = parse.parse_pairs(rest)
 672         other_pairs = dict(pairs)
 673
 674         id = Id().from_string(pairs['id'])
 675         del other_pairs['id']
 676
 677         addr = int(pairs['addr'], 0)
 678         del other_pairs['addr']
 679
 680         if 'inst' in other_pairs:
 681             del other_pairs['inst']
 682
 683             # Collapse unnecessary spaces in disassembly
 684             disassembly = re.sub('  *', ' ',
 685                 re.sub('^ *', '', pairs['inst']))
 686
 687             inst = Inst(id, disassembly, addr, other_pairs)
 688             self.add_inst(inst)
 689         elif 'fault' in other_pairs:
 690             del other_pairs['fault']
 691
 692             inst = InstFault(id, pairs['fault'], addr, other_pairs)
 693
 694             self.add_inst(inst)
 695
 696     def add_minor_line(self, rest):
 697         """Parse and add a MinorLine line to the model"""
 698         pairs = parse.parse_pairs(rest)
 699         other_pairs = dict(pairs)
 700
 701         id = Id().from_string(pairs['id'])
 702         del other_pairs['id']
 703
 704         vaddr = int(pairs['vaddr'], 0)
 705         del other_pairs['vaddr']
 706
 707         if 'paddr' in other_pairs:
 708             del other_pairs['paddr']
 709             del other_pairs['size']
 710             paddr = int(pairs['paddr'], 0)
 711             size = int(pairs['size'], 0)
 712
 713             self.add_line(Line(id,
 714                 vaddr, paddr, size, other_pairs))
 715         elif 'fault' in other_pairs:
 716             del other_pairs['fault']
 717
 718             self.add_line(LineFault(id, pairs['fault'], vaddr, other_pairs))
 719
 720     def load_events(self, file, startTime=0, endTime=None):
 721         """Load an event file and add everything to this model"""
 722         def update_comments(comments, time):
 723             # Add a list of comments to an existing event, if there is one at
 724             #   the given time, or create a new, correctly-timed, event from
 725             #   the last event and attach the comments to that
 726             for commentUnit, commentRest in comments:
 727                 event = self.find_unit_event_by_time(commentUnit, time)
 728                 # Find an event to which this comment can be attached
 729                 if event is None:
 730                     # No older event, make a new empty one
 731                     event = BlobEvent(commentUnit, time, {})
 732                     self.add_unit_event(event)
 733                 elif event.time != time:
 734                     # Copy the old event and make a new one with the right
 735                     #   time and comment
 736                     newEvent = BlobEvent(commentUnit, time, event.pairs)
 737                     newEvent.visuals = dict(event.visuals)
 738                     event = newEvent
 739                     self.add_unit_event(event)
 740                 event.comments.append(commentRest)
 741
 742         self.clear_events()
 743
 744         # A negative time will *always* be different from an event time
 745         time = -1
 746         time_events = {}
 747         last_time_lines = {}
 748         minor_trace_line_count = 0
 749         comments = []
 750
 751         default_colour = [[colours.unknownColour]]
 752         next_progress_print_event_count = 1000
 753
 754         if not os.access(file, os.R_OK):
 755             print 'Can\'t open file', file
 756             exit(1)
 757         else:
 758             print 'Opening file', file
 759
 760         f = open(file)
 761
 762         start_wall_time = wall_time()
 763
 764         # Skip leading events
 765         still_skipping = True
 766         l = f.readline()
 767         while l and still_skipping:
 768             match = re.match('^\s*(\d+):', l)
 769             if match is not None:
 770                 event_time = match.groups()
 771                 if int(event_time[0]) >= startTime:
 772                     still_skipping = False
 773                 else:
 774                     l = f.readline()
 775             else:
 776                 l = f.readline()
 777
 778         match_line_re = re.compile(
 779             '^\s*(\d+):\s*([\w\.]+):\s*(Minor\w+:)?\s*(.*)$')
 780
 781         # Parse each line of the events file, accumulating comments to be
 782         #   attached to MinorTrace events when the time changes
 783         reached_end_time = False
 784         while not reached_end_time and l:
 785             match = match_line_re.match(l)
 786             if match is not None:
 787                 event_time, unit, line_type, rest = match.groups()
 788                 event_time = int(event_time)
 789
 790                 unit = re.sub('^' + self.unitNamePrefix + '\.?(.*)$',
 791                     '\\1', unit)
 792
 793                 # When the time changes, resolve comments
 794                 if event_time != time:
 795                     if self.numEvents > next_progress_print_event_count:
 796                         print ('Parsed to time: %d' % event_time)
 797                         next_progress_print_event_count = (
 798                             self.numEvents + 1000)
 799                     update_comments(comments, time)
 800                     comments = []
 801                     time = event_time
 802
 803                 if line_type is None:
 804                     # Treat this line as just a 'comment'
 805                     comments.append((unit, rest))
 806                 elif line_type == 'MinorTrace:':
 807                     minor_trace_line_count += 1
 808
 809                     # Only insert this event if it's not the same as
 810                     #   the last event we saw for this unit
 811                     if last_time_lines.get(unit, None) != rest:
 812                         event = BlobEvent(unit, event_time, {})
 813                         pairs = parse.parse_pairs(rest)
 814                         event.pairs = pairs
 815
 816                         # Try to decode the colour data for this event
 817                         blobs = self.unitNameToBlobs.get(unit, [])
 818                         for blob in blobs:
 819                             if blob.visualDecoder is not None:
 820                                 event.visuals[blob.picChar] = (
 821                                     blob.visualDecoder(pairs))
 822
 823                         self.add_unit_event(event)
 824                         last_time_lines[unit] = rest
 825                 elif line_type == 'MinorInst:':
 826                     self.add_minor_inst(rest)
 827                 elif line_type == 'MinorLine:':
 828                     self.add_minor_line(rest)
 829
 830             if endTime is not None and time > endTime:
 831                 reached_end_time = True
 832
 833             l = f.readline()
 834
 835         update_comments(comments, time)
 836         self.extract_times()
 837         f.close()
 838
 839         end_wall_time = wall_time()
 840
 841         print 'Total events:', minor_trace_line_count, 'unique events:', \
 842             self.numEvents
 843         print 'Time to parse:', end_wall_time - start_wall_time
 844
 845     def add_blob_picture(self, offset, pic, nameDict):
 846         """Add a parsed ASCII-art pipeline markup to the model"""
 847         pic_width = 0
 848         for line in pic:
 849             pic_width = max(pic_width, len(line))
 850         pic_height = len(pic)
 851
 852         # Number of horizontal characters per 'pixel'.  Should be 2
 853         charsPerPixel = 2
 854
 855         # Clean up pic_width to a multiple of charsPerPixel
 856         pic_width = (pic_width + charsPerPixel - 1) // 2
 857
 858         self.picSize = Point(pic_width, pic_height)
 859
 860         def pic_at(point):
 861             """Return the char pair at the given point.
 862             Returns None for characters off the picture"""
 863             x, y = point.to_pair()
 864             x *= 2
 865             if y >= len(pic) or x >= len(pic[y]):
 866                 return None
 867             else:
 868                 return pic[y][x:x + charsPerPixel]
 869
 870         def clear_pic_at(point):
 871             """Clear the chars at point so we don't trip over them again"""
 872             line = pic[point.y]
 873             x = point.x * charsPerPixel
 874             pic[point.y] = line[0:x] + (' ' * charsPerPixel) + \
 875                 line[x + charsPerPixel:]
 876
 877         def skip_same_char(start, increment):
 878             """Skip characters which match pic_at(start)"""
 879             char = pic_at(start)
 880             hunt = start
 881             while pic_at(hunt) == char:
 882                 hunt += increment
 883             return hunt
 884
 885         def find_size(start):
 886             """Find the size of a rectangle with top left hand corner at
 887             start consisting of (at least) a -. shaped corner describing
 888             the top right corner of a rectangle of the same char"""
 889             char = pic_at(start)
 890             hunt_x = skip_same_char(start, Point(1,0))
 891             hunt_y = skip_same_char(start, Point(0,1))
 892             off_bottom_right = (hunt_x * Point(1,0)) + (hunt_y * Point(0,1))
 893             return off_bottom_right - start
 894
 895         def point_return(point):
 896             """Carriage return, line feed"""
 897             return Point(0, point.y + 1)
 898
 899         def find_arrow(start):
 900             """Find a simple 1-char wide arrow"""
 901
 902             def body(endChar, contChar, direc):
 903                 arrow_point = start
 904                 arrow_point += Point(0, 1)
 905                 clear_pic_at(start)
 906                 while pic_at(arrow_point) == contChar:
 907                     clear_pic_at(arrow_point)
 908                     arrow_point += Point(0, 1)
 909
 910                 if pic_at(arrow_point) == endChar:
 911                     clear_pic_at(arrow_point)
 912                     self.add_blob(blobs.Arrow('_', start + offset,
 913                         direc = direc,
 914                         size = (Point(1, 1) + arrow_point - start)))
 915                 else:
 916                     print 'Bad arrow', start
 917
 918             char = pic_at(start)
 919             if char == '-\\':
 920                 body('-/', ' :', 'right')
 921             elif char == '/-':
 922                 body('\\-', ': ', 'left')
 923
 924         blank_chars = ['  ', ' :', ': ']
 925
 926         # Traverse the picture left to right, top to bottom to find blobs
 927         seen_dict = {}
 928         point = Point(0,0)
 929         while pic_at(point) is not None:
 930             while pic_at(point) is not None:
 931                 char = pic_at(point)
 932                 if char == '->':
 933                     self.add_blob(blobs.Arrow('_', point + offset,
 934                         direc = 'right'))
 935                 elif char == '<-':
 936                     self.add_blob(blobs.Arrow('_', point + offset,
 937                         direc = 'left'))
 938                 elif char == '-\\' or char == '/-':
 939                     find_arrow(point)
 940                 elif char in blank_chars:
 941                     pass
 942                 else:
 943                     if char not in seen_dict:
 944                         size = find_size(point)
 945                         topLeft = point + offset
 946                         if char not in nameDict:
 947                             # Unnamed blobs
 948                             self.add_blob(blobs.Block(char,
 949                                 nameDict.get(char, '_'),
 950                                 topLeft, size = size))
 951                         else:
 952                             # Named blobs, set visual info.
 953                             blob = nameDict[char]
 954                             blob.size = size
 955                             blob.topLeft = topLeft
 956                             self.add_blob(blob)
 957                     seen_dict[char] = True
 958                 point = skip_same_char(point, Point(1,0))
 959             point = point_return(point)
 960
 961     def load_picture(self, filename):
 962         """Load a picture file into the model"""
 963         def parse_blob_description(char, unit, macros, pairsList):
 964             # Parse the name value pairs in a blob-describing line
 965             def expand_macros(pairs, newPairs):
 966                 # Recursively expand macros
 967                 for name, value in newPairs:
 968                     if name in macros:
 969                         expand_macros(pairs, macros[name])
 970                     else:
 971                         pairs[name] = value
 972                 return pairs
 973
 974             pairs = expand_macros({}, pairsList)
 975
 976             ret = None
 977
 978             typ = pairs.get('type', 'block')
 979             colour = colours.name_to_colour(pairs.get('colour', 'black'))
 980
 981             if typ == 'key':
 982                 ret = blobs.Key(char, unit, Point(0,0), colour)
 983             elif typ == 'block':
 984                 ret = blobs.Block(char, unit, Point(0,0), colour)
 985             else:
 986                 print "Bad picture blog type:", typ
 987
 988             if 'hideId' in pairs:
 989                 hide = pairs['hideId']
 990                 ret.dataSelect.ids -= set(hide)
 991
 992             if typ == 'block':
 993                 ret.displayName = pairs.get('name', unit)
 994                 ret.nameLoc = pairs.get('nameLoc', 'top')
 995                 ret.shape = pairs.get('shape', 'box')
 996                 ret.stripDir = pairs.get('stripDir', 'horiz')
 997                 ret.stripOrd = pairs.get('stripOrd', 'LR')
 998                 ret.blankStrips = int(pairs.get('blankStrips', '0'))
 999                 ret.shorten = int(pairs.get('shorten', '0'))
1000
1001                 if 'decoder' in pairs:
1002                     decoderName = pairs['decoder']
1003                     dataElement = pairs.get('dataElement', decoderName)
1004
1005                     decoder = find_colour_decoder(ret.blankStrips,
1006                         decoderName, dataElement, pairs)
1007                     if decoder is not None:
1008                         ret.visualDecoder = decoder
1009                     else:
1010                         print 'Bad visualDecoder requested:', decoderName
1011
1012                 if 'border' in pairs:
1013                     border = pairs['border']
1014                     if border == 'thin':
1015                         ret.border = 0.2
1016                     elif border == 'mid':
1017                         ret.border = 0.5
1018                     else:
1019                         ret.border = 1.0
1020             elif typ == 'key':
1021                 ret.colours = pairs.get('colours', ret.colours)
1022
1023             return ret
1024
1025         def line_is_comment(line):
1026             """Returns true if a line starts with #, returns False
1027             for lines which are None"""
1028             return line is not None \
1029                 and re.match('^\s*#', line) is not None
1030
1031         def get_line(f):
1032             """Get a line from file f extending that line if it ends in
1033             '\' and dropping lines that start with '#'s"""
1034             ret = f.readline()
1035
1036             # Discard comment lines
1037             while line_is_comment(ret):
1038                 ret = f.readline()
1039
1040             if ret is not None:
1041                 extend_match = re.match('^(.*)\\\\$', ret)
1042
1043                 while extend_match is not None:
1044                     new_line = f.readline()
1045
1046                     if new_line is not None and not line_is_comment(new_line):
1047                         line_wo_backslash, = extend_match.groups()
1048                         ret = line_wo_backslash + new_line
1049                         extend_match = re.match('^(.*)\\\\$', ret)
1050                     else:
1051                         extend_match = None
1052
1053             return ret
1054
1055         # Macros are recursively expanded into name=value pairs
1056         macros = {}
1057
1058         if not os.access(filename, os.R_OK):
1059             print 'Can\'t open file', filename
1060             exit(1)
1061         else:
1062             print 'Opening file', filename
1063
1064         f = open(filename)
1065         l = get_line(f)
1066         picture = []
1067         blob_char_dict = {}
1068
1069         self.unitEvents = {}
1070         self.clear_events()
1071
1072         # Actually parse the file
1073         in_picture = False
1074         while l:
1075             l = parse.remove_trailing_ws(l)
1076             l = re.sub('#.*', '', l)
1077
1078             if re.match("^\s*$", l) is not None:
1079                 pass
1080             elif l == '<<<':
1081                 in_picture = True
1082             elif l == '>>>':
1083                 in_picture = False
1084             elif in_picture:
1085                 picture.append(re.sub('\s*$', '', l))
1086             else:
1087                 line_match = re.match(
1088                     '^([a-zA-Z0-9][a-zA-Z0-9]):\s+([\w.]+)\s*(.*)', l)
1089                 macro_match = re.match('macro\s+(\w+):(.*)', l)
1090
1091                 if macro_match is not None:
1092                     name, defn = macro_match.groups()
1093                     macros[name] = parse.parse_pairs_list(defn)
1094                 elif line_match is not None:
1095                     char, unit, pairs = line_match.groups()
1096                     blob = parse_blob_description(char, unit, macros,
1097                         parse.parse_pairs_list(pairs))
1098                     blob_char_dict[char] = blob
1099                     # Setup the events structure
1100                     self.unitEvents[unit] = []
1101                 else:
1102                     print 'Problem with Blob line:', l
1103
1104             l = get_line(f)
1105
1106         self.blobs = []
1107         self.add_blob_picture(Point(0,1), picture, blob_char_dict)