scons: Add support for google protobuf building

This patch enables the use of protobuf input files in the build
process, thus allowing .proto files to be added to input. Each .proto
file is compiled using the protoc tool and the newly created C++
source is added to the list of sources.

The first location where the protobufs will be used is in the
capturing and replay of memory traces, involving the communication
monitor and the trace-generator state of the traffic generator. This
will follow in the next patch.

This patch does add a dependency on the availability of the BSD
licensed protobuf library (and headers), and the protobuf compiler,
protoc. These dependencies are checked in the SConstruct, similar to
e.g. swig. The user can override the use of protoc from the PATH by
specifying the PROTOC environment variable.

Although the dependency on libprotobuf and protoc might seem like a
big step, they add significant value to the project going
forward. Execution traces and other types of traces could easily be
added and parsers for C++ and Python are automatically generated. We
could also envision using protobufs for the checkpoints, description
of the traffic-generator behaviour etc. The sky is the limit. We could
also use the GzipOutputStream from the protobuf library instead of the
current GPL gzstream.

Currently, only the C++ source and header is generated. Going forward
we might want to add the Python output to support simple command-line
tools for displaying and editing the traces.

diff --git a/SConstruct b/SConstruct
index 842c4fef59fb67d7f8cdf4f70313fe26be6087e6..a51acb15961d69ac847b8981fc94b0c07ecc88bd 100755 (executable)
--- a/SConstruct
+++ b/SConstruct
@@ -180,7 +180,8 @@ termcap = get_termcap(GetOption('use_colors'))
 #
 ########################################################################
 use_vars = set([ 'AS', 'AR', 'CC', 'CXX', 'HOME', 'LD_LIBRARY_PATH',
-                 'LIBRARY_PATH', 'PATH', 'PYTHONPATH', 'RANLIB', 'SWIG' ])
+                 'LIBRARY_PATH', 'PATH', 'PKG_CONFIG_PATH', 'PYTHONPATH',
+                 'RANLIB', 'SWIG' ])
 
 use_env = {}
 for key,val in os.environ.iteritems():
@@ -363,6 +364,7 @@ global_vars.AddVariables(
     ('CC', 'C compiler', environ.get('CC', main['CC'])),
     ('CXX', 'C++ compiler', environ.get('CXX', main['CXX'])),
     ('SWIG', 'SWIG tool', environ.get('SWIG', main['SWIG'])),
+    ('PROTOC', 'protoc tool', environ.get('PROTOC', 'protoc')),
     ('BATCH', 'Use batch pool for build and tests', False),
     ('BATCH_CMD', 'Batch pool submission command name', 'qdo'),
     ('M5_BUILD_CACHE', 'Cache built objects in this directory', False),
@@ -613,6 +615,34 @@ if sys.platform == 'cygwin':
     # cygwin has some header file issues...
     main.Append(CCFLAGS=["-Wno-uninitialized"])
 
+# Check for the protobuf compiler
+protoc_version = readCommand([main['PROTOC'], '--version'],
+                             exception='').split()
+
+# First two words should be "libprotoc x.y.z"
+if len(protoc_version) < 2 or protoc_version[0] != 'libprotoc':
+    print termcap.Yellow + termcap.Bold + \
+        'Warning: Protocol buffer compiler (protoc) not found.\n' + \
+        '         Please install protobuf-compiler for tracing support.' + \
+        termcap.Normal
+    main['PROTOC'] = False
+else:
+    # Determine the appropriate include path and library path using
+    # pkg-config, that means we also need to check for pkg-config
+    if not readCommand(['pkg-config', '--version'], exception=''):
+        print 'Error: pkg-config not found. Please install and retry.'
+        Exit(1)
+
+    main.ParseConfig('pkg-config --cflags --libs-only-L protobuf')
+
+    # Based on the availability of the compress stream wrappers,
+    # require 2.1.0
+    min_protoc_version = '2.1.0'
+    if compareVersions(protoc_version[1], min_protoc_version) < 0:
+        print 'Error: protoc version', min_protoc_version, 'or newer required.'
+        print '       Installed version:', protoc_version[1]
+        Exit(1)
+
 # Check for SWIG
 if not main.has_key('SWIG'):
     print 'Error: SWIG utility not found.'
@@ -814,6 +844,21 @@ if not conf.CheckLibWithHeader('z', 'zlib.h', 'C++','zlibVersion();'):
     print '       Please install zlib and try again.'
     Exit(1)
 
+# If we have the protobuf compiler, also make sure we have the
+# development libraries. If the check passes, libprotobuf will be
+# automatically added to the LIBS environment variable. After
+# this, we can use the HAVE_PROTOBUF flag to determine if we have
+# got both protoc and libprotobuf available.
+main['HAVE_PROTOBUF'] = main['PROTOC'] and \
+    conf.CheckLibWithHeader('protobuf', 'google/protobuf/message.h',
+                            'C++', 'GOOGLE_PROTOBUF_VERIFY_VERSION;')
+
+# If we have the compiler but not the library, treat it as an error.
+if main['PROTOC'] and not main['HAVE_PROTOBUF']:
+    print 'Error: did not find protocol buffer library and/or headers.'
+    print '       Please install libprotobuf-dev and try again.'
+    Exit(1)
+
 # Check for librt.
 have_posix_clock = \
     conf.CheckLibWithHeader(None, 'time.h', 'C',
@@ -940,7 +985,7 @@ sticky_vars.AddVariables(
 # These variables get exported to #defines in config/*.hh (see src/SConscript).
 export_vars += ['USE_FENV', 'SS_COMPATIBLE_FP',
                 'TARGET_ISA', 'CP_ANNOTATE', 'USE_POSIX_CLOCK', 'PROTOCOL',
-                'HAVE_STATIC_ASSERT']
+                'HAVE_STATIC_ASSERT', 'HAVE_PROTOBUF']
 
 ###################################################
 #

diff --git a/src/SConscript b/src/SConscript
index 9af71c9a6dd96632020796a0eacb991b118b17fb..845f514c57c9b589de5c08ae36b8e061f2e4e3b8 100755 (executable)
--- a/src/SConscript
+++ b/src/SConscript
@@ -233,6 +233,22 @@ class SwigSource(SourceFile):
         self.cc_source = Source(cc_file, swig=True, parent=self)
         self.py_source = PySource(package, py_file, parent=self)
 
+class ProtoBuf(SourceFile):
+    '''Add a Protocol Buffer to build'''
+
+    def __init__(self, source, **guards):
+        '''Specify the source file, and any guards'''
+        super(ProtoBuf, self).__init__(source, **guards)
+
+        # Get the file name and the extension
+        modname,ext = self.extname
+        assert ext == 'proto'
+
+        # Currently, we stick to generating the C++ headers, so we
+        # only need to track the source and header.
+        self.cc_file = File(joinpath(self.dirname, modname + '.pb.cc'))
+        self.hh_file = File(joinpath(self.dirname, modname + '.pb.h'))
+
 class UnitTest(object):
     '''Create a UnitTest'''
 
@@ -260,6 +276,7 @@ Export('Source')
 Export('PySource')
 Export('SimObject')
 Export('SwigSource')
+Export('ProtoBuf')
 Export('UnitTest')
 
 ########################################################################
@@ -676,6 +693,24 @@ for swig in SwigSource.all:
                 MakeAction(makeEmbeddedSwigInit, Transform("EMBED SW")))
     Source(init_file, **swig.guards)
 
+# Build all protocol buffers if we have got protoc and protobuf available
+if env['HAVE_PROTOBUF']:
+    for proto in ProtoBuf.all:
+        # Use both the source and header as the target, and the .proto
+        # file as the source. When executing the protoc compiler, also
+        # specify the proto_path to avoid having the generated files
+        # include the path.
+        env.Command([proto.cc_file, proto.hh_file], proto.tnode,
+                    MakeAction('$PROTOC --cpp_out ${TARGET.dir} '
+                               '--proto_path ${SOURCE.dir} $SOURCE',
+                               Transform("PROTOC")))
+
+        # Add the C++ source file
+        Source(proto.cc_file, **proto.guards)
+elif ProtoBuf.all:
+    print 'Got protobuf to build, but lacks support!'
+    Exit(1)
+
 #
 # Handle debug flags
 #