# -*- mode:python -*-
-# Copyright (c) 2013, 2015-2017, 2019 ARM Limited
+# Copyright (c) 2013, 2015-2019 ARM Limited
# All rights reserved.
#
# The license below extends only to copyright in the software and shall
# cygwin has some header file issues...
main.Append(CCFLAGS=["-Wno-uninitialized"])
+
+have_pkg_config = readCommand(['pkg-config', '--version'], exception='')
+
# Check for the protobuf compiler
protoc_version = readCommand([main['PROTOC'], '--version'],
exception='').split()
# protobuf without the involvement of pkg-config. Later on we
# check go a library config check and at that point the test
# will fail if libprotobuf cannot be found.
- if readCommand(['pkg-config', '--version'], exception=''):
+ if have_pkg_config:
try:
# Attempt to establish what linking flags to add for protobuf
# using pkg-config
main['HAVE_PERF_ATTR_EXCLUDE_HOST'] = conf.CheckMember(
'linux/perf_event.h', 'struct perf_event_attr', 'exclude_host')
+def check_hdf5():
+ return \
+ conf.CheckLibWithHeader('hdf5', 'hdf5.h', 'C',
+ 'H5Fcreate("", 0, 0, 0);') and \
+ conf.CheckLibWithHeader('hdf5_cpp', 'H5Cpp.h', 'C++',
+ 'H5::H5File("", 0);')
+
+def check_hdf5_pkg(name):
+ print("Checking for %s using pkg-config..." % name, end="")
+ if not have_pkg_config:
+ print(" pkg-config not found")
+ return False
+
+ try:
+ main.ParseConfig('pkg-config --cflags-only-I --libs-only-L %s' % name)
+ print(" yes")
+ return True
+ except:
+ print(" no")
+ return False
+
+# Check if there is a pkg-config configuration for hdf5. If we find
+# it, setup the environment to enable linking and header inclusion. We
+# don't actually try to include any headers or link with hdf5 at this
+# stage.
+if not check_hdf5_pkg('hdf5-serial'):
+ check_hdf5_pkg('hdf5')
+
+# Check if the HDF5 libraries can be found. This check respects the
+# include path and library path provided by pkg-config. We perform
+# this check even if there isn't a pkg-config configuration for hdf5
+# since some installations don't use pkg-config.
+have_hdf5 = check_hdf5()
+if not have_hdf5:
+ print("Warning: Couldn't find any HDF5 C++ libraries. Disabling")
+ print(" HDF5 support.")
######################################################################
#
backtrace_impls[-1], backtrace_impls),
('NUMBER_BITS_PER_SET', 'Max elements in set (default 64)',
64),
+ BoolVariable('USE_HDF5', 'Enable the HDF5 support', have_hdf5),
)
# These variables get exported to #defines in config/*.hh (see src/SConscript).
'CP_ANNOTATE', 'USE_POSIX_CLOCK', 'USE_KVM', 'USE_TUNTAP',
'PROTOCOL', 'HAVE_PROTOBUF', 'HAVE_VALGRIND',
'HAVE_PERF_ATTR_EXCLUDE_HOST', 'USE_PNG',
- 'NUMBER_BITS_PER_SET']
+ 'NUMBER_BITS_PER_SET', 'USE_HDF5']
###################################################
#
Source('stats/group.cc')
Source('stats/text.cc')
+if env['USE_HDF5']:
+ Source('stats/hdf5.cc')
GTest('addr_range.test', 'addr_range.test.cc')
GTest('addr_range_map.test', 'addr_range_map.test.cc')
--- /dev/null
+/*
+ * Copyright (c) 2016-2019 Arm Limited
+ * All rights reserved
+ *
+ * The license below extends only to copyright in the software and shall
+ * not be construed as granting a license to any other intellectual
+ * property including but not limited to intellectual property relating
+ * to a hardware implementation of the functionality of the software
+ * licensed hereunder. You may use the software subject to the license
+ * terms below provided that you ensure that this notice is replicated
+ * unmodified and in its entirety in all distributions of the software,
+ * modified or unmodified, in source code or in binary form.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met: redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer;
+ * redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution;
+ * neither the name of the copyright holders nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Authors: Andreas Sandberg
+ */
+
+#include "base/stats/hdf5.hh"
+
+#include "base/logging.hh"
+#include "base/stats/info.hh"
+
+/**
+ * Check if all strings in a container are empty.
+ */
+template<typename T>
+bool emptyStrings(const T &labels)
+{
+ for (const auto &s : labels) {
+ if (!s.empty())
+ return false;
+ }
+ return true;
+}
+
+
+namespace Stats {
+
+Hdf5::Hdf5(const std::string &file, unsigned chunking,
+ bool desc, bool formulas)
+ : fname(file), timeChunk(chunking),
+ enableDescriptions(desc), enableFormula(formulas),
+ dumpCount(0)
+{
+ // Tell the library not to print exceptions by default. There are
+ // cases where we rely on exceptions to determine if we need to
+ // create a node or if we can just open it.
+ H5::Exception::dontPrint();
+}
+
+Hdf5::~Hdf5()
+{
+}
+
+
+void
+Hdf5::begin()
+{
+ h5File = H5::H5File(fname,
+ // Truncate the file if this is the first dump
+ dumpCount > 0 ? H5F_ACC_RDWR : H5F_ACC_TRUNC);
+ path.push(h5File.openGroup("/"));
+}
+
+void
+Hdf5::end()
+{
+ assert(valid());
+
+ dumpCount++;
+}
+
+bool
+Hdf5::valid() const
+{
+ return true;
+}
+
+
+void
+Hdf5::beginGroup(const char *name)
+{
+ auto base = path.top();
+
+ // Try to open an existing stat group corresponding to the
+ // name. Create it if it doesn't exist.
+ H5::Group group;
+ try {
+ group = base.openGroup(name);
+ } catch (H5::FileIException e) {
+ group = base.createGroup(name);
+ } catch (H5::GroupIException e) {
+ group = base.createGroup(name);
+ }
+
+ path.push(group);
+}
+
+void
+Hdf5::endGroup()
+{
+ assert(!path.empty());
+ path.pop();
+}
+
+void
+Hdf5::visit(const ScalarInfo &info)
+{
+ // Since this stat is a scalar, we need 1-dimensional value in the
+ // stat file. The Hdf5::appendStat helper will populate the size
+ // of the first dimension (time).
+ hsize_t fdims[1] = { 0, };
+ double data[1] = { info.result(), };
+
+ appendStat(info, 1, fdims, data);
+}
+
+void
+Hdf5::visit(const VectorInfo &info)
+{
+ appendVectorInfo(info);
+}
+
+void
+Hdf5::visit(const DistInfo &info)
+{
+ warn_once("HDF5 stat files don't support distributions.\n");
+}
+
+void
+Hdf5::visit(const VectorDistInfo &info)
+{
+ warn_once("HDF5 stat files don't support vector distributions.\n");
+}
+
+void
+Hdf5::visit(const Vector2dInfo &info)
+{
+ // Request a 3-dimensional stat, the first dimension will be
+ // populated by the Hdf5::appendStat() helper. The remaining two
+ // dimensions correspond to the stat instance.
+ hsize_t fdims[3] = { 0, info.x, info.y };
+ H5::DataSet data_set = appendStat(info, 3, fdims, info.cvec.data());
+
+ if (dumpCount == 0) {
+ if (!info.subnames.empty() && !emptyStrings(info.subnames))
+ addMetaData(data_set, "subnames", info.subnames);
+
+ if (!info.y_subnames.empty() && !emptyStrings(info.y_subnames))
+ addMetaData(data_set, "y_subnames", info.y_subnames);
+
+ if (!info.subdescs.empty() && !emptyStrings(info.subdescs))
+ addMetaData(data_set, "subdescs", info.subdescs);
+ }
+}
+
+void
+Hdf5::visit(const FormulaInfo &info)
+{
+ if (!enableFormula)
+ return;
+
+ H5::DataSet data_set = appendVectorInfo(info);
+
+ if (dumpCount == 0)
+ addMetaData(data_set, "equation", info.str());
+}
+
+void
+Hdf5::visit(const SparseHistInfo &info)
+{
+ warn_once("HDF5 stat files don't support sparse histograms.\n");
+}
+
+H5::DataSet
+Hdf5::appendVectorInfo(const VectorInfo &info)
+{
+ const VResult &vr(info.result());
+ // Request a 2-dimensional stat, the first dimension will be
+ // populated by the Hdf5::appendStat() helper. The remaining
+ // dimension correspond to the stat instance.
+ hsize_t fdims[2] = { 0, vr.size() };
+ H5::DataSet data_set = appendStat(info, 2, fdims, vr.data());
+
+ if (dumpCount == 0) {
+ if (!info.subnames.empty() && !emptyStrings(info.subnames))
+ addMetaData(data_set, "subnames", info.subnames);
+
+ if (!info.subdescs.empty() && !emptyStrings(info.subdescs))
+ addMetaData(data_set, "subdescs", info.subdescs);
+ }
+
+ return data_set;
+}
+
+H5::DataSet
+Hdf5::appendStat(const Info &info, int rank, hsize_t *dims, const double *data)
+{
+ H5::Group group = path.top();
+ H5::DataSet data_set;
+ H5::DataSpace fspace;
+
+ dims[0] = dumpCount + 1;
+
+ if (dumpCount > 0) {
+ // Get the existing stat if we have already dumped this stat
+ // before.
+ data_set = group.openDataSet(info.name);
+ data_set.extend(dims);
+ fspace = data_set.getSpace();
+ } else {
+ // We don't have the stat already, create it.
+
+ H5::DSetCreatPropList props;
+
+ // Setup max dimensions based on the requested file dimensions
+ std::vector<hsize_t> max_dims(rank);
+ std::copy(dims, dims + rank, max_dims.begin());
+ max_dims[0] = H5S_UNLIMITED;
+
+ // Setup chunking
+ std::vector<hsize_t> chunk_dims(rank);
+ std::copy(dims, dims + rank, chunk_dims.begin());
+ chunk_dims[0] = timeChunk;
+ props.setChunk(rank, chunk_dims.data());
+
+ // Enable compression
+ props.setDeflate(1);
+
+ fspace = H5::DataSpace(rank, dims, max_dims.data());
+ data_set = group.createDataSet(info.name, H5::PredType::NATIVE_DOUBLE,
+ fspace, props);
+
+ if (enableDescriptions && !info.desc.empty()) {
+ addMetaData(data_set, "description", info.desc);
+ }
+ }
+
+ // The first dimension is time which isn't included in data.
+ dims[0] = 1;
+ H5::DataSpace mspace(rank, dims);
+ std::vector<hsize_t> foffset(rank, 0);
+ foffset[0] = dumpCount;
+
+ fspace.selectHyperslab(H5S_SELECT_SET, dims, foffset.data());
+ data_set.write(data, H5::PredType::NATIVE_DOUBLE, mspace, fspace);
+
+ return data_set;
+}
+
+void
+Hdf5::addMetaData(H5::DataSet &loc, const char *name,
+ const std::vector<const char *> &values)
+{
+ H5::StrType type(H5::PredType::C_S1, H5T_VARIABLE);
+ hsize_t dims[1] = { values.size(), };
+ H5::DataSpace space(1, dims);
+ H5::Attribute attribute = loc.createAttribute(name, type, space);
+ attribute.write(type, values.data());
+}
+
+void
+Hdf5::addMetaData(H5::DataSet &loc, const char *name,
+ const std::vector<std::string> &values)
+{
+ std::vector<const char *> cstrs(values.size());
+ for (int i = 0; i < values.size(); ++i)
+ cstrs[i] = values[i].c_str();
+
+ addMetaData(loc, name, cstrs);
+}
+
+void
+Hdf5::addMetaData(H5::DataSet &loc, const char *name,
+ const std::string &value)
+{
+ H5::StrType type(H5::PredType::C_S1, value.length() + 1);
+ hsize_t dims[1] = { 1, };
+ H5::DataSpace space(1, dims);
+ H5::Attribute attribute = loc.createAttribute(name, type, space);
+ attribute.write(type, value.c_str());
+}
+
+void
+Hdf5::addMetaData(H5::DataSet &loc, const char *name, double value)
+{
+ hsize_t dims[1] = { 1, };
+ H5::DataSpace space(1, dims);
+ H5::Attribute attribute = loc.createAttribute(
+ name, H5::PredType::NATIVE_DOUBLE, space);
+ attribute.write(H5::PredType::NATIVE_DOUBLE, &value);
+}
+
+
+std::unique_ptr<Output>
+initHDF5(const std::string &filename, unsigned chunking,
+ bool desc, bool formulas)
+{
+ return std::unique_ptr<Output>(
+ new Hdf5(filename, chunking, desc, formulas));
+}
+
+}; // namespace Stats
--- /dev/null
+/*
+ * Copyright (c) 2016-2019 Arm Limited
+ * All rights reserved
+ *
+ * The license below extends only to copyright in the software and shall
+ * not be construed as granting a license to any other intellectual
+ * property including but not limited to intellectual property relating
+ * to a hardware implementation of the functionality of the software
+ * licensed hereunder. You may use the software subject to the license
+ * terms below provided that you ensure that this notice is replicated
+ * unmodified and in its entirety in all distributions of the software,
+ * modified or unmodified, in source code or in binary form.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met: redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer;
+ * redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution;
+ * neither the name of the copyright holders nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Authors: Andreas Sandberg
+ */
+
+#ifndef __BASE_STATS_HDF5_HH__
+#define __BASE_STATS_HDF5_HH__
+
+#include <H5Cpp.h>
+
+#include <memory>
+#include <stack>
+#include <string>
+#include <vector>
+
+#include "base/output.hh"
+#include "base/stats/output.hh"
+#include "base/stats/types.hh"
+
+namespace Stats {
+
+class Hdf5 : public Output
+{
+ public:
+ Hdf5(const std::string &file, unsigned chunking, bool desc, bool formulas);
+
+ ~Hdf5();
+
+ Hdf5() = delete;
+ Hdf5(const Hdf5 &other) = delete;
+
+ public: // Output interface
+ void begin() override;
+ void end() override;
+ bool valid() const override;
+
+ void beginGroup(const char *name) override;
+ void endGroup() override;
+
+ void visit(const ScalarInfo &info) override;
+ void visit(const VectorInfo &info) override;
+ void visit(const DistInfo &info) override;
+ void visit(const VectorDistInfo &info) override;
+ void visit(const Vector2dInfo &info) override;
+ void visit(const FormulaInfo &info) override;
+ void visit(const SparseHistInfo &info) override;
+
+ protected:
+ /**
+ * Helper function to append vector stats and set their metadata.
+ */
+ H5::DataSet appendVectorInfo(const VectorInfo &info);
+
+ /**
+ * Helper function to append an n-dimensional double stat to the
+ * file.
+ *
+ * This helper function assumes that all stats include a time
+ * component. I.e., a Stat::Scalar is a 1-dimensional stat.
+ *
+ * @param info Stat info structure.
+ * @param rank Stat dimensionality (including time).
+ * @param dims Size of each of the dimensions.
+ */
+ H5::DataSet appendStat(const Info &info, int rank, hsize_t *dims,
+ const double *data);
+
+ /**
+ * Helper function to add a string vector attribute to a stat.
+ *
+ * @param loc Parent location in the file.
+ * @param name Attribute name.
+ * @param values Attribute value.
+ */
+ void addMetaData(H5::DataSet &loc, const char *name,
+ const std::vector<const char *> &values);
+
+ /**
+ * Helper function to add a string vector attribute to a stat.
+ *
+ * @param loc Parent location in the file.
+ * @param name Attribute name.
+ * @param values Attribute value.
+ */
+ void addMetaData(H5::DataSet &loc, const char *name,
+ const std::vector<std::string> &values);
+
+ /**
+ * Helper function to add a string attribute to a stat.
+ *
+ * @param loc Parent location in the file.
+ * @param name Attribute name.
+ * @param value Attribute value.
+ */
+ void addMetaData(H5::DataSet &loc, const char *name,
+ const std::string &value);
+
+ /**
+ * Helper function to add a double attribute to a stat.
+ *
+ * @param loc Parent location in the file.
+ * @param name Attribute name.
+ * @param value Attribute value.
+ */
+ void addMetaData(H5::DataSet &loc, const char *name, double value);
+
+ protected:
+ const std::string fname;
+ const hsize_t timeChunk;
+ const bool enableDescriptions;
+ const bool enableFormula;
+
+ std::stack<H5::Group> path;
+
+ unsigned dumpCount;
+ H5::H5File h5File;
+};
+
+std::unique_ptr<Output> initHDF5(
+ const std::string &filename,unsigned chunking = 10,
+ bool desc = true, bool formulas = true);
+
+} // namespace Stats
+
+#endif // __BASE_STATS_HDF5_HH__
-# Copyright (c) 2017, 2019 Arm Limited
+# Copyright (c) 2017-2019 ARM Limited
# All rights reserved.
#
# The license below extends only to copyright in the software and shall
return _m5.stats.initText(fn, desc)
+@_url_factory
+def _hdf5Factory(fn, chunking=10, desc=True, formulas=True):
+ """Output stats in HDF5 format.
+
+ The HDF5 file format is a structured binary file format. It has
+ the multiple benefits over traditional text stat files:
+
+ * Efficient storage of time series (multiple stat dumps)
+ * Fast lookup of stats
+ * Plenty of existing tooling (e.g., Python libraries and graphical
+ viewers)
+ * File format can be used to store frame buffers together with
+ normal stats.
+
+ There are some drawbacks compared to the default text format:
+ * Large startup cost (single stat dump larger than text equivalent)
+ * Stat dumps are slower than text
+
+
+ Known limitations:
+ * Distributions and histograms currently unsupported.
+ * No support for forking.
+
+
+ Parameters:
+ * chunking (unsigned): Number of time steps to pre-allocate (default: 10)
+ * desc (bool): Output stat descriptions (default: True)
+ * formulas (bool): Output derived stats (default: True)
+
+ Example:
+ h5://stats.h5?desc=False;chunking=100;formulas=False
+
+ """
+
+ if hasattr(_m5.stats, "initHDF5"):
+ return _m5.stats.initHDF5(fn, chunking, desc, formulas)
+ else:
+ fatal("HDF5 support not enabled at compile time")
+
factories = {
# Default to the text factory if we're given a naked path
"" : _textFactory,
"file" : _textFactory,
"text" : _textFactory,
+ "h5" : _hdf5Factory,
}
+
def addStatVisitor(url):
"""Add a stat visitor specified using a URL string
/*
- * Copyright (c) 2017, 2019 ARM Limited
+ * Copyright (c) 2017-2019 ARM Limited
* All rights reserved
*
* The license below extends only to copyright in the software and shall
* Andreas Sandberg
*/
+#include "config/use_hdf5.hh"
+
#include "pybind11/pybind11.h"
#include "pybind11/stl.h"
#include "base/statistics.hh"
#include "base/stats/text.hh"
+#if USE_HDF5
+#include "base/stats/hdf5.hh"
+#endif
#include "sim/stat_control.hh"
#include "sim/stat_register.hh"
+
namespace py = pybind11;
namespace Stats {
m
.def("initSimStats", &Stats::initSimStats)
.def("initText", &Stats::initText, py::return_value_policy::reference)
+#if USE_HDF5
+ .def("initHDF5", &Stats::initHDF5)
+#endif
.def("registerPythonStatsHandlers",
&Stats::registerPythonStatsHandlers)
.def("schedStatEvent", &Stats::schedStatEvent)
projects / gem5.git / commitdiff