src/python/m5/stats/__init__.py

   1 # Copyright (c) 2017-2020 ARM Limited
   2 # All rights reserved.
   3 #
   4 # The license below extends only to copyright in the software and shall
   5 # not be construed as granting a license to any other intellectual
   6 # property including but not limited to intellectual property relating
   7 # to a hardware implementation of the functionality of the software
   8 # licensed hereunder.  You may use the software subject to the license
   9 # terms below provided that you ensure that this notice is replicated
  10 # unmodified and in its entirety in all distributions of the software,
  11 # modified or unmodified, in source code or in binary form.
  12 #
  13 # Copyright (c) 2007 The Regents of The University of Michigan
  14 # Copyright (c) 2010 The Hewlett-Packard Development Company
  15 # All rights reserved.
  16 #
  17 # Redistribution and use in source and binary forms, with or without
  18 # modification, are permitted provided that the following conditions are
  19 # met: redistributions of source code must retain the above copyright
  20 # notice, this list of conditions and the following disclaimer;
  21 # redistributions in binary form must reproduce the above copyright
  22 # notice, this list of conditions and the following disclaimer in the
  23 # documentation and/or other materials provided with the distribution;
  24 # neither the name of the copyright holders nor the names of its
  25 # contributors may be used to endorse or promote products derived from
  26 # this software without specific prior written permission.
  27 #
  28 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  29 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  30 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  31 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  32 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  33 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  34 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  35 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  36 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  37 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  38 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  39
  40 from __future__ import print_function
  41 from __future__ import absolute_import
  42
  43 import m5
  44
  45 import _m5.stats
  46 from m5.objects import Root
  47 from m5.params import isNullPointer
  48 from m5.util import attrdict, fatal
  49
  50 # Stat exports
  51 from _m5.stats import schedStatEvent as schedEvent
  52 from _m5.stats import periodicStatDump
  53
  54 outputList = []
  55
  56 # Dictionary of stat visitor factories populated by the _url_factory
  57 # visitor.
  58 factories = { }
  59
  60 # List of all factories. Contains tuples of (factory, schemes,
  61 # enabled).
  62 all_factories = []
  63
  64 def _url_factory(schemes, enable=True):
  65     """Wrap a plain Python function with URL parsing helpers
  66
  67     Wrap a plain Python function f(fn, **kwargs) to expect a URL that
  68     has been split using urlparse.urlsplit. First positional argument
  69     is assumed to be a filename, this is created as the concatenation
  70     of the netloc (~hostname) and path in the parsed URL. Keyword
  71     arguments are derived from the query values in the URL.
  72
  73     Arguments:
  74         schemes: A list of URL schemes to use for this function.
  75
  76     Keyword arguments:
  77         enable: Enable/disable this factory. Typically used when the
  78                 presence of a function depends on some runtime property.
  79
  80     For example:
  81         wrapped_f(urlparse.urlsplit("text://stats.txt?desc=False")) ->
  82         f("stats.txt", desc=False)
  83
  84     """
  85
  86     from functools import wraps
  87
  88     def decorator(func):
  89         @wraps(func)
  90         def wrapper(url):
  91             try:
  92                 from urllib.parse import parse_qs
  93             except ImportError:
  94                 # Python 2 fallback
  95                 from urlparse import parse_qs
  96             from ast import literal_eval
  97
  98             qs = parse_qs(url.query, keep_blank_values=True)
  99
 100             # parse_qs returns a list of values for each parameter. Only
 101             # use the last value since kwargs don't allow multiple values
 102             # per parameter. Use literal_eval to transform string param
 103             # values into proper Python types.
 104             def parse_value(key, values):
 105                 if len(values) == 0 or (len(values) == 1 and not values[0]):
 106                     fatal("%s: '%s' doesn't have a value." % (
 107                         url.geturl(), key))
 108                 elif len(values) > 1:
 109                     fatal("%s: '%s' has multiple values." % (
 110                         url.geturl(), key))
 111                 else:
 112                     try:
 113                         return key, literal_eval(values[0])
 114                     except ValueError:
 115                         fatal("%s: %s isn't a valid Python literal" \
 116                               % (url.geturl(), values[0]))
 117
 118             kwargs = dict([ parse_value(k, v) for k, v in qs.items() ])
 119
 120             try:
 121                 return func("%s%s" % (url.netloc, url.path), **kwargs)
 122             except TypeError:
 123                 fatal("Illegal stat visitor parameter specified")
 124
 125         all_factories.append((wrapper, schemes, enable))
 126         for scheme in schemes:
 127             assert scheme not in factories
 128             factories[scheme] = wrapper if enable else None
 129         return wrapper
 130
 131     return decorator
 132
 133 @_url_factory([ None, "", "text", "file", ])
 134 def _textFactory(fn, desc=True, spaces=True):
 135     """Output stats in text format.
 136
 137     Text stat files contain one stat per line with an optional
 138     description. The description is enabled by default, but can be
 139     disabled by setting the desc parameter to False.
 140
 141     Parameters:
 142       * desc (bool): Output stat descriptions (default: True)
 143       * spaces (bool): Output alignment spaces (default: True)
 144
 145     Example:
 146       text://stats.txt?desc=False;spaces=False
 147
 148     """
 149
 150     return _m5.stats.initText(fn, desc, spaces)
 151
 152 @_url_factory([ "h5", ], enable=hasattr(_m5.stats, "initHDF5"))
 153 def _hdf5Factory(fn, chunking=10, desc=True, formulas=True):
 154     """Output stats in HDF5 format.
 155
 156     The HDF5 file format is a structured binary file format. It has
 157     the multiple benefits over traditional text stat files:
 158
 159       * Efficient storage of time series (multiple stat dumps)
 160       * Fast lookup of stats
 161       * Plenty of existing tooling (e.g., Python libraries and graphical
 162         viewers)
 163       * File format can be used to store frame buffers together with
 164         normal stats.
 165
 166     There are some drawbacks compared to the default text format:
 167       * Large startup cost (single stat dump larger than text equivalent)
 168       * Stat dumps are slower than text
 169
 170
 171     Known limitations:
 172       * Distributions and histograms currently unsupported.
 173       * No support for forking.
 174
 175
 176     Parameters:
 177       * chunking (unsigned): Number of time steps to pre-allocate (default: 10)
 178       * desc (bool): Output stat descriptions (default: True)
 179       * formulas (bool): Output derived stats (default: True)
 180
 181     Example:
 182       h5://stats.h5?desc=False;chunking=100;formulas=False
 183
 184     """
 185
 186     return _m5.stats.initHDF5(fn, chunking, desc, formulas)
 187
 188 def addStatVisitor(url):
 189     """Add a stat visitor specified using a URL string
 190
 191     Stat visitors are specified using URLs on the following format:
 192     format://path[?param=value[;param=value]]
 193
 194     The available formats are listed in the factories list. Factories
 195     are called with the path as the first positional parameter and the
 196     parameters are keyword arguments. Parameter values must be valid
 197     Python literals.
 198
 199     """
 200
 201     try:
 202         from urllib.parse import urlsplit
 203     except ImportError:
 204         # Python 2 fallback
 205         from urlparse import urlsplit
 206
 207     parsed = urlsplit(url)
 208
 209     try:
 210         factory = factories[parsed.scheme]
 211     except KeyError:
 212         fatal("Illegal stat file type '%s' specified." % parsed.scheme)
 213
 214     if factory is None:
 215         fatal("Stat type '%s' disabled at compile time" % parsed.scheme)
 216
 217     outputList.append(factory(parsed))
 218
 219 def printStatVisitorTypes():
 220     """List available stat visitors and their documentation"""
 221
 222     import inspect
 223
 224     def print_doc(doc):
 225         for line in doc.splitlines():
 226             print("| %s" % line)
 227         print()
 228
 229     enabled_visitors = [ x for x in all_factories if x[2] ]
 230     for factory, schemes, _ in enabled_visitors:
 231         print("%s:" % ", ".join(filter(lambda x: x is not None, schemes)))
 232
 233         # Try to extract the factory doc string
 234         print_doc(inspect.getdoc(factory))
 235
 236 def initSimStats():
 237     _m5.stats.initSimStats()
 238     _m5.stats.registerPythonStatsHandlers()
 239
 240 def _visit_groups(visitor, root=None):
 241     if root is None:
 242         root = Root.getInstance()
 243     for group in root.getStatGroups().values():
 244         visitor(group)
 245         _visit_groups(visitor, root=group)
 246
 247 def _visit_stats(visitor, root=None):
 248     def for_each_stat(g):
 249         for stat in g.getStats():
 250             visitor(g, stat)
 251     _visit_groups(for_each_stat, root=root)
 252
 253 def _bindStatHierarchy(root):
 254     def _bind_obj(name, obj):
 255         if isNullPointer(obj):
 256             return
 257         if m5.SimObject.isSimObjectVector(obj):
 258             if len(obj) == 1:
 259                 _bind_obj(name, obj[0])
 260             else:
 261                 for idx, obj in enumerate(obj):
 262                     _bind_obj("{}{}".format(name, idx), obj)
 263         else:
 264             # We need this check because not all obj.getCCObject() is an
 265             # instance of Stat::Group. For example, sc_core::sc_module, the C++
 266             # class of SystemC_ScModule, is not a subclass of Stat::Group. So
 267             # it will cause a type error if obj is a SystemC_ScModule when
 268             # calling addStatGroup().
 269             if isinstance(obj.getCCObject(), _m5.stats.Group):
 270                 parent = root
 271                 while parent:
 272                     if hasattr(parent, 'addStatGroup'):
 273                         parent.addStatGroup(name, obj.getCCObject())
 274                         break
 275                     parent = parent.get_parent();
 276
 277             _bindStatHierarchy(obj)
 278
 279     for name, obj in root._children.items():
 280         _bind_obj(name, obj)
 281
 282 names = []
 283 stats_dict = {}
 284 stats_list = []
 285 def enable():
 286     '''Enable the statistics package.  Before the statistics package is
 287     enabled, all statistics must be created and initialized and once
 288     the package is enabled, no more statistics can be created.'''
 289
 290     def check_stat(group, stat):
 291         if not stat.check() or not stat.baseCheck():
 292             fatal("statistic '%s' (%d) was not properly initialized " \
 293                   "by a regStats() function\n", stat.name, stat.id)
 294
 295         if not (stat.flags & flags.display):
 296             stat.name = "__Stat%06d" % stat.id
 297
 298
 299     # Legacy stat
 300     global stats_list
 301     stats_list = list(_m5.stats.statsList())
 302
 303     for stat in stats_list:
 304         check_stat(None, stat)
 305
 306     stats_list.sort(key=lambda s: s.name.split('.'))
 307     for stat in stats_list:
 308         stats_dict[stat.name] = stat
 309         stat.enable()
 310
 311
 312     # New stats
 313     _visit_stats(check_stat)
 314     _visit_stats(lambda g, s: s.enable())
 315
 316     _m5.stats.enable();
 317
 318 def prepare():
 319     '''Prepare all stats for data access.  This must be done before
 320     dumping and serialization.'''
 321
 322     # Legacy stats
 323     for stat in stats_list:
 324         stat.prepare()
 325
 326     # New stats
 327     _visit_stats(lambda g, s: s.prepare())
 328
 329 def _dump_to_visitor(visitor, roots=None):
 330     # New stats
 331     def dump_group(group):
 332         for stat in group.getStats():
 333             stat.visit(visitor)
 334         for n, g in group.getStatGroups().items():
 335             visitor.beginGroup(n)
 336             dump_group(g)
 337             visitor.endGroup()
 338
 339     if roots:
 340         # New stats from selected subroots.
 341         for root in roots:
 342             for p in root.path_list():
 343                 visitor.beginGroup(p)
 344             dump_group(root)
 345             for p in reversed(root.path_list()):
 346                 visitor.endGroup()
 347     else:
 348         # Legacy stats
 349         for stat in stats_list:
 350             stat.visit(visitor)
 351
 352         # New stats starting from root.
 353         dump_group(Root.getInstance())
 354
 355 lastDump = 0
 356 # List[SimObject].
 357 global_dump_roots = []
 358
 359 def dump(roots=None):
 360     '''Dump all statistics data to the registered outputs'''
 361
 362     all_roots = []
 363     if roots is not None:
 364         all_roots.extend(roots)
 365     global global_dump_roots
 366     all_roots.extend(global_dump_roots)
 367
 368     now = m5.curTick()
 369     global lastDump
 370     assert lastDump <= now
 371     new_dump = lastDump != now
 372     lastDump = now
 373
 374     # Don't allow multiple global stat dumps in the same tick. It's
 375     # still possible to dump a multiple sub-trees.
 376     if not new_dump and not all_roots:
 377         return
 378
 379     # Only prepare stats the first time we dump them in the same tick.
 380     if new_dump:
 381         _m5.stats.processDumpQueue()
 382         # Notify new-style stats group that we are about to dump stats.
 383         sim_root = Root.getInstance()
 384         if sim_root:
 385             sim_root.preDumpStats();
 386         prepare()
 387
 388     for output in outputList:
 389         if output.valid():
 390             output.begin()
 391             _dump_to_visitor(output, roots=all_roots)
 392             output.end()
 393
 394 def reset():
 395     '''Reset all statistics to the base state'''
 396
 397     # call reset stats on all SimObjects
 398     root = Root.getInstance()
 399     if root:
 400         root.resetStats()
 401
 402     # call any other registered legacy stats reset callbacks
 403     for stat in stats_list:
 404         stat.reset()
 405
 406     _m5.stats.processResetQueue()
 407
 408 flags = attrdict({
 409     'none'    : 0x0000,
 410     'init'    : 0x0001,
 411     'display' : 0x0002,
 412     'total'   : 0x0010,
 413     'pdf'     : 0x0020,
 414     'cdf'     : 0x0040,
 415     'dist'    : 0x0080,
 416     'nozero'  : 0x0100,
 417     'nonan'   : 0x0200,
 418 })