--- /dev/null
+syntax: glob
+
+*.pyc
--- /dev/null
+'construct' is the actual library (version 2, downloaded on 15.09.2010)
+Taken from http://construct.wikispaces.com/
+The code for version 2.00 has been released by the author into the public domain
+
+
--- /dev/null
+"""\r
+Construct 2.00 -- parsing made even more fun (and faster)\r
+\r
+Homepage:\r
+http://construct.wikispaces.com\r
+\r
+Typical usage:\r
+>>> from construct import *\r
+\r
+Example:\r
+>>> from construct import *\r
+>>>\r
+>>> s = Struct("foo",\r
+... UBInt8("a"),\r
+... UBInt16("b"),\r
+... )\r
+>>>\r
+>>> s.parse("\x01\x02\x03")\r
+Container(a = 1, b = 515)\r
+>>> print s.parse("\x01\x02\x03")\r
+Container:\r
+ a = 1\r
+ b = 515\r
+>>> s.build(Container(a = 1, b = 0x0203))\r
+"\x01\x02\x03"\r
+"""\r
+from core import *\r
+from adapters import *\r
+from macros import *\r
+from debug import Probe, Debugger\r
+\r
+\r
+#===============================================================================\r
+# meta data\r
+#===============================================================================\r
+__author__ = "tomer filiba (tomerfiliba [at] gmail.com)"\r
+__version__ = "2.00"\r
+\r
+#===============================================================================\r
+# shorthands\r
+#===============================================================================\r
+Bits = BitField\r
+Byte = UBInt8\r
+Bytes = Field\r
+Const = ConstAdapter\r
+Tunnel = TunnelAdapter\r
+Embed = Embedded\r
+\r
+#===============================================================================\r
+# backward compatibility with RC1\r
+#===============================================================================\r
+MetaField = Field\r
+MetaBytes = Field\r
+GreedyRepeater = GreedyRange\r
+OptionalGreedyRepeater = OptionalGreedyRange\r
+Repeater = Array\r
+StrictRepeater = Array\r
+MetaRepeater = Array\r
+OneOfValidator = OneOf\r
+NoneOfValidator = NoneOf\r
+\r
+#===============================================================================\r
+# don't want to leek these out...\r
+#===============================================================================\r
+del encode_bin, decode_bin, int_to_bin, bin_to_int, swap_bytes\r
+del Packer, StringIO\r
+del HexString, LazyContainer, AttrDict\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
--- /dev/null
+from core import Adapter, AdaptationError, Pass\r
+from lib import int_to_bin, bin_to_int, swap_bytes, StringIO\r
+from lib import FlagsContainer, HexString\r
+\r
+\r
+#===============================================================================\r
+# exceptions\r
+#===============================================================================\r
+class BitIntegerError(AdaptationError):\r
+ __slots__ = []\r
+class MappingError(AdaptationError):\r
+ __slots__ = []\r
+class ConstError(AdaptationError):\r
+ __slots__ = []\r
+class ValidationError(AdaptationError):\r
+ __slots__ = []\r
+class PaddingError(AdaptationError):\r
+ __slots__ = []\r
+\r
+#===============================================================================\r
+# adapters\r
+#===============================================================================\r
+class BitIntegerAdapter(Adapter):\r
+ """\r
+ Adapter for bit-integers (converts bitstrings to integers, and vice versa).\r
+ See BitField.\r
+ \r
+ Parameters:\r
+ * subcon - the subcon to adapt\r
+ * width - the size of the subcon, in bits\r
+ * swapped - whether to swap byte order (little endian/big endian). \r
+ default is False (big endian)\r
+ * signed - whether the value is signed (two's complement). the default\r
+ is False (unsigned)\r
+ * bytesize - number of bits per byte, used for byte-swapping (if swapped).\r
+ default is 8.\r
+ """\r
+ __slots__ = ["width", "swapped", "signed", "bytesize"]\r
+ def __init__(self, subcon, width, swapped = False, signed = False, \r
+ bytesize = 8):\r
+ Adapter.__init__(self, subcon)\r
+ self.width = width\r
+ self.swapped = swapped\r
+ self.signed = signed\r
+ self.bytesize = bytesize\r
+ def _encode(self, obj, context):\r
+ if obj < 0 and not self.signed:\r
+ raise BitIntegerError("object is negative, but field is not signed",\r
+ obj)\r
+ obj2 = int_to_bin(obj, width = self.width)\r
+ if self.swapped:\r
+ obj2 = swap_bytes(obj2, bytesize = self.bytesize)\r
+ return obj2\r
+ def _decode(self, obj, context):\r
+ if self.swapped:\r
+ obj = swap_bytes(obj, bytesize = self.bytesize)\r
+ return bin_to_int(obj, signed = self.signed)\r
+\r
+class MappingAdapter(Adapter):\r
+ """\r
+ Adapter that maps objects to other objects.\r
+ See SymmetricMapping and Enum.\r
+ \r
+ Parameters:\r
+ * subcon - the subcon to map\r
+ * decoding - the decoding (parsing) mapping (a dict)\r
+ * encoding - the encoding (building) mapping (a dict)\r
+ * decdefault - the default return value when the object is not found\r
+ in the decoding mapping. if no object is given, an exception is raised.\r
+ if `Pass` is used, the unmapped object will be passed as-is\r
+ * encdefault - the default return value when the object is not found\r
+ in the encoding mapping. if no object is given, an exception is raised.\r
+ if `Pass` is used, the unmapped object will be passed as-is\r
+ """\r
+ __slots__ = ["encoding", "decoding", "encdefault", "decdefault"]\r
+ def __init__(self, subcon, decoding, encoding, \r
+ decdefault = NotImplemented, encdefault = NotImplemented):\r
+ Adapter.__init__(self, subcon)\r
+ self.decoding = decoding\r
+ self.encoding = encoding\r
+ self.decdefault = decdefault\r
+ self.encdefault = encdefault\r
+ def _encode(self, obj, context):\r
+ try:\r
+ return self.encoding[obj]\r
+ except (KeyError, TypeError):\r
+ if self.encdefault is NotImplemented:\r
+ raise MappingError("no encoding mapping for %r" % (obj,))\r
+ if self.encdefault is Pass:\r
+ return obj\r
+ return self.encdefault\r
+ def _decode(self, obj, context):\r
+ try:\r
+ return self.decoding[obj]\r
+ except (KeyError, TypeError):\r
+ if self.decdefault is NotImplemented:\r
+ raise MappingError("no decoding mapping for %r" % (obj,))\r
+ if self.decdefault is Pass:\r
+ return obj\r
+ return self.decdefault\r
+\r
+class FlagsAdapter(Adapter):\r
+ """\r
+ Adapter for flag fields. Each flag is extracted from the number, resulting\r
+ in a FlagsContainer object. Not intended for direct usage.\r
+ See FlagsEnum.\r
+ \r
+ Parameters\r
+ * subcon - the subcon to extract\r
+ * flags - a dictionary mapping flag-names to their value\r
+ """\r
+ __slots__ = ["flags"]\r
+ def __init__(self, subcon, flags):\r
+ Adapter.__init__(self, subcon)\r
+ self.flags = flags\r
+ def _encode(self, obj, context):\r
+ flags = 0\r
+ for name, value in self.flags.iteritems():\r
+ if getattr(obj, name, False):\r
+ flags |= value\r
+ return flags\r
+ def _decode(self, obj, context):\r
+ obj2 = FlagsContainer()\r
+ for name, value in self.flags.iteritems():\r
+ setattr(obj2, name, bool(obj & value))\r
+ return obj2\r
+\r
+class StringAdapter(Adapter):\r
+ """\r
+ Adapter for strings. Converts a sequence of characters into a python \r
+ string, and optionally handles character encoding.\r
+ See String.\r
+ \r
+ Parameters:\r
+ * subcon - the subcon to convert\r
+ * encoding - the character encoding name (e.g., "utf8"), or None to \r
+ return raw bytes (usually 8-bit ASCII).\r
+ """\r
+ __slots__ = ["encoding"]\r
+ def __init__(self, subcon, encoding = None):\r
+ Adapter.__init__(self, subcon)\r
+ self.encoding = encoding\r
+ def _encode(self, obj, context):\r
+ if self.encoding:\r
+ obj = obj.encode(self.encoding)\r
+ return obj\r
+ def _decode(self, obj, context):\r
+ obj = "".join(obj)\r
+ if self.encoding:\r
+ obj = obj.decode(self.encoding)\r
+ return obj\r
+\r
+class PaddedStringAdapter(Adapter):\r
+ r"""\r
+ Adapter for padded strings.\r
+ See String.\r
+ \r
+ Parameters:\r
+ * subcon - the subcon to adapt\r
+ * padchar - the padding character. default is "\x00".\r
+ * paddir - the direction where padding is placed ("right", "left", or \r
+ "center"). the default is "right". \r
+ * trimdir - the direction where trimming will take place ("right" or \r
+ "left"). the default is "right". trimming is only meaningful for\r
+ building, when the given string is too long. \r
+ """\r
+ __slots__ = ["padchar", "paddir", "trimdir"]\r
+ def __init__(self, subcon, padchar = "\x00", paddir = "right", \r
+ trimdir = "right"):\r
+ if paddir not in ("right", "left", "center"):\r
+ raise ValueError("paddir must be 'right', 'left' or 'center'", \r
+ paddir)\r
+ if trimdir not in ("right", "left"):\r
+ raise ValueError("trimdir must be 'right' or 'left'", trimdir)\r
+ Adapter.__init__(self, subcon)\r
+ self.padchar = padchar\r
+ self.paddir = paddir\r
+ self.trimdir = trimdir\r
+ def _decode(self, obj, context):\r
+ if self.paddir == "right":\r
+ obj = obj.rstrip(self.padchar)\r
+ elif self.paddir == "left":\r
+ obj = obj.lstrip(self.padchar)\r
+ else:\r
+ obj = obj.strip(self.padchar)\r
+ return obj\r
+ def _encode(self, obj, context):\r
+ size = self._sizeof(context)\r
+ if self.paddir == "right":\r
+ obj = obj.ljust(size, self.padchar)\r
+ elif self.paddir == "left":\r
+ obj = obj.rjust(size, self.padchar)\r
+ else:\r
+ obj = obj.center(size, self.padchar)\r
+ if len(obj) > size:\r
+ if self.trimdir == "right":\r
+ obj = obj[:size]\r
+ else:\r
+ obj = obj[-size:]\r
+ return obj\r
+\r
+class LengthValueAdapter(Adapter):\r
+ """\r
+ Adapter for length-value pairs. It extracts only the value from the \r
+ pair, and calculates the length based on the value.\r
+ See PrefixedArray and PascalString.\r
+ \r
+ Parameters:\r
+ * subcon - the subcon returning a length-value pair\r
+ """\r
+ __slots__ = []\r
+ def _encode(self, obj, context):\r
+ return (len(obj), obj)\r
+ def _decode(self, obj, context):\r
+ return obj[1]\r
+\r
+class CStringAdapter(StringAdapter):\r
+ r"""\r
+ Adapter for C-style strings (strings terminated by a terminator char).\r
+ \r
+ Parameters:\r
+ * subcon - the subcon to convert\r
+ * terminators - a sequence of terminator chars. default is "\x00".\r
+ * encoding - the character encoding to use (e.g., "utf8"), or None to\r
+ return raw-bytes. the terminator characters are not affected by the \r
+ encoding.\r
+ """\r
+ __slots__ = ["terminators"]\r
+ def __init__(self, subcon, terminators = "\x00", encoding = None):\r
+ StringAdapter.__init__(self, subcon, encoding = encoding)\r
+ self.terminators = terminators\r
+ def _encode(self, obj, context):\r
+ return StringAdapter._encode(self, obj, context) + self.terminators[0]\r
+ def _decode(self, obj, context):\r
+ return StringAdapter._decode(self, obj[:-1], context)\r
+\r
+class TunnelAdapter(Adapter):\r
+ """\r
+ Adapter for tunneling (as in protocol tunneling). A tunnel is construct\r
+ nested upon another (layering). For parsing, the lower layer first parses\r
+ the data (note: it must return a string!), then the upper layer is called\r
+ to parse that data (bottom-up). For building it works in a top-down manner;\r
+ first the upper layer builds the data, then the lower layer takes it and\r
+ writes it to the stream.\r
+ \r
+ Parameters:\r
+ * subcon - the lower layer subcon\r
+ * inner_subcon - the upper layer (tunneled/nested) subcon\r
+ \r
+ Example:\r
+ # a pascal string containing compressed data (zlib encoding), so first\r
+ # the string is read, decompressed, and finally re-parsed as an array\r
+ # of UBInt16\r
+ TunnelAdapter(\r
+ PascalString("data", encoding = "zlib"),\r
+ GreedyRange(UBInt16("elements"))\r
+ )\r
+ """\r
+ __slots__ = ["inner_subcon"]\r
+ def __init__(self, subcon, inner_subcon):\r
+ Adapter.__init__(self, subcon)\r
+ self.inner_subcon = inner_subcon\r
+ def _decode(self, obj, context):\r
+ return self.inner_subcon._parse(StringIO(obj), context)\r
+ def _encode(self, obj, context):\r
+ stream = StringIO()\r
+ self.inner_subcon._build(obj, stream, context)\r
+ return stream.getvalue()\r
+\r
+class ExprAdapter(Adapter):\r
+ """\r
+ A generic adapter that accepts 'encoder' and 'decoder' as parameters. You\r
+ can use ExprAdapter instead of writing a full-blown class when only a \r
+ simple expression is needed.\r
+ \r
+ Parameters:\r
+ * subcon - the subcon to adapt\r
+ * encoder - a function that takes (obj, context) and returns an encoded \r
+ version of obj\r
+ * decoder - a function that takes (obj, context) and returns an decoded \r
+ version of obj\r
+ \r
+ Example:\r
+ ExprAdapter(UBInt8("foo"), \r
+ encoder = lambda obj, ctx: obj / 4,\r
+ decoder = lambda obj, ctx: obj * 4,\r
+ )\r
+ """\r
+ __slots__ = ["_encode", "_decode"]\r
+ def __init__(self, subcon, encoder, decoder):\r
+ Adapter.__init__(self, subcon)\r
+ self._encode = encoder\r
+ self._decode = decoder\r
+\r
+class HexDumpAdapter(Adapter):\r
+ """\r
+ Adapter for hex-dumping strings. It returns a HexString, which is a string\r
+ """\r
+ __slots__ = ["linesize"]\r
+ def __init__(self, subcon, linesize = 16):\r
+ Adapter.__init__(self, subcon)\r
+ self.linesize = linesize\r
+ def _encode(self, obj, context):\r
+ return obj\r
+ def _decode(self, obj, context):\r
+ return HexString(obj, linesize = self.linesize)\r
+\r
+class ConstAdapter(Adapter):\r
+ """\r
+ Adapter for enforcing a constant value ("magic numbers"). When decoding,\r
+ the return value is checked; when building, the value is substituted in.\r
+ \r
+ Parameters:\r
+ * subcon - the subcon to validate\r
+ * value - the expected value\r
+ \r
+ Example:\r
+ Const(Field("signature", 2), "MZ")\r
+ """\r
+ __slots__ = ["value"]\r
+ def __init__(self, subcon, value):\r
+ Adapter.__init__(self, subcon)\r
+ self.value = value\r
+ def _encode(self, obj, context):\r
+ if obj is None or obj == self.value:\r
+ return self.value\r
+ else:\r
+ raise ConstError("expected %r, found %r" % (self.value, obj))\r
+ def _decode(self, obj, context):\r
+ if obj != self.value:\r
+ raise ConstError("expected %r, found %r" % (self.value, obj))\r
+ return obj\r
+\r
+class SlicingAdapter(Adapter):\r
+ """\r
+ Adapter for slicing a list (getting a slice from that list)\r
+ \r
+ Parameters:\r
+ * subcon - the subcon to slice\r
+ * start - start index\r
+ * stop - stop index (or None for up-to-end)\r
+ * step - step (or None for every element)\r
+ """\r
+ __slots__ = ["start", "stop", "step"]\r
+ def __init__(self, subcon, start, stop = None):\r
+ Adapter.__init__(self, subcon)\r
+ self.start = start\r
+ self.stop = stop\r
+ def _encode(self, obj, context):\r
+ if self.start is None:\r
+ return obj\r
+ return [None] * self.start + obj\r
+ def _decode(self, obj, context):\r
+ return obj[self.start:self.stop]\r
+\r
+class IndexingAdapter(Adapter):\r
+ """\r
+ Adapter for indexing a list (getting a single item from that list)\r
+ \r
+ Parameters:\r
+ * subcon - the subcon to index\r
+ * index - the index of the list to get\r
+ """\r
+ __slots__ = ["index"]\r
+ def __init__(self, subcon, index):\r
+ Adapter.__init__(self, subcon)\r
+ if type(index) is not int:\r
+ raise TypeError("index must be an integer", type(index))\r
+ self.index = index\r
+ def _encode(self, obj, context):\r
+ return [None] * self.index + [obj]\r
+ def _decode(self, obj, context):\r
+ return obj[self.index]\r
+\r
+class PaddingAdapter(Adapter):\r
+ r"""\r
+ Adapter for padding.\r
+ \r
+ Parameters:\r
+ * subcon - the subcon to pad\r
+ * pattern - the padding pattern (character). default is "\x00")\r
+ * strict - whether or not to verify, during parsing, that the given \r
+ padding matches the padding pattern. default is False (unstrict)\r
+ """\r
+ __slots__ = ["pattern", "strict"]\r
+ def __init__(self, subcon, pattern = "\x00", strict = False):\r
+ Adapter.__init__(self, subcon)\r
+ self.pattern = pattern\r
+ self.strict = strict\r
+ def _encode(self, obj, context):\r
+ return self._sizeof(context) * self.pattern\r
+ def _decode(self, obj, context):\r
+ if self.strict:\r
+ expected = self._sizeof(context) * self.pattern\r
+ if obj != expected:\r
+ raise PaddingError("expected %r, found %r" % (expected, obj))\r
+ return obj\r
+\r
+\r
+#===============================================================================\r
+# validators\r
+#===============================================================================\r
+class Validator(Adapter):\r
+ """\r
+ Abstract class: validates a condition on the encoded/decoded object. \r
+ Override _validate(obj, context) in deriving classes.\r
+ \r
+ Parameters:\r
+ * subcon - the subcon to validate\r
+ """\r
+ __slots__ = []\r
+ def _decode(self, obj, context):\r
+ if not self._validate(obj, context):\r
+ raise ValidationError("invalid object", obj)\r
+ return obj\r
+ def _encode(self, obj, context):\r
+ return self._decode(obj, context)\r
+ def _validate(self, obj, context):\r
+ raise NotImplementedError()\r
+\r
+class OneOf(Validator):\r
+ """\r
+ Validates that the value is one of the listed values\r
+ \r
+ Parameters:\r
+ * subcon - the subcon to validate\r
+ * valids - a set of valid values\r
+ """\r
+ __slots__ = ["valids"]\r
+ def __init__(self, subcon, valids):\r
+ Validator.__init__(self, subcon)\r
+ self.valids = valids\r
+ def _validate(self, obj, context):\r
+ return obj in self.valids\r
+\r
+class NoneOf(Validator):\r
+ """\r
+ Validates that the value is none of the listed values\r
+ \r
+ Parameters:\r
+ * subcon - the subcon to validate\r
+ * invalids - a set of invalid values\r
+ """\r
+ __slots__ = ["invalids"]\r
+ def __init__(self, subcon, invalids):\r
+ Validator.__init__(self, subcon)\r
+ self.invalids = invalids\r
+ def _validate(self, obj, context):\r
+ return obj not in self.invalids\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
--- /dev/null
+from lib import StringIO, Packer\r
+from lib import Container, ListContainer, AttrDict, LazyContainer\r
+\r
+\r
+#===============================================================================\r
+# exceptions\r
+#===============================================================================\r
+class ConstructError(Exception):\r
+ __slots__ = []\r
+class FieldError(ConstructError):\r
+ __slots__ = []\r
+class SizeofError(ConstructError):\r
+ __slots__ = []\r
+class AdaptationError(ConstructError):\r
+ __slots__ = []\r
+class ArrayError(ConstructError):\r
+ __slots__ = []\r
+class RangeError(ConstructError):\r
+ __slots__ = []\r
+class SwitchError(ConstructError):\r
+ __slots__ = []\r
+class SelectError(ConstructError):\r
+ __slots__ = []\r
+class TerminatorError(ConstructError):\r
+ __slots__ = []\r
+\r
+#===============================================================================\r
+# abstract constructs\r
+#===============================================================================\r
+class Construct(object):\r
+ """\r
+ The mother of all constructs!\r
+ \r
+ User API:\r
+ * parse(buf) - parses an in-memory buffer (usually a string)\r
+ * parse_stream(stream) - parses a stream (in-memory, file, pipe, ...)\r
+ * build(obj) - builds the object into an in-memory buffer (a string)\r
+ * build_stream(obj, stream) - builds the object into the given stream\r
+ * sizeof(context) - calculates the size of the construct, if possible,\r
+ based on the context\r
+ \r
+ Overriable methods for subclassing:\r
+ * _parse(stream, context) - low-level parse from stream\r
+ * _build(obj, stream, context) - low-level build to stream\r
+ * _sizeof(context) - low-level compute size\r
+ \r
+ Flags API:\r
+ * _set_flag(flag) - sets the given flag/flags\r
+ * _clear_flag(flag) - clears the given flag/flags\r
+ * _inherit_flags(*subcons) - inherits the flag of subcons\r
+ * _is_flag(flag) - is the flag set? (predicate)\r
+ \r
+ Overridable methods for the copy-API:\r
+ * __getstate__() - returns a dict of the attributes of self\r
+ * __setstate__(attrs) - sets the attrs to self\r
+ \r
+ Attributes:\r
+ All constructs have a name and flags. The name is used for naming \r
+ struct-members and context dicts. Note that the name must be a string or\r
+ None (if the name is not needed). A single underscore ("_") is a reserved\r
+ name, and so are names starting with a less-than character ("<"). The name\r
+ should be descriptive, short, and valid as a python identifier (although\r
+ these rules are not enforced). \r
+ \r
+ The flags specify additional behavioral information about this construct.\r
+ The flags are used by enclosing constructs to determine a proper course \r
+ of action. Usually, flags are "inherited", i.e., an enclosing construct\r
+ inherits the flags of its subconstruct. The enclosing construct may\r
+ set new flags or clear existing ones, as necessary.\r
+ \r
+ For example, if FLAG_COPY_CONTEXT is set, repeaters will pass a copy of \r
+ the context for each iteration, which is necessary for OnDemand parsing.\r
+ """\r
+ FLAG_COPY_CONTEXT = 0x0001\r
+ FLAG_DYNAMIC = 0x0002\r
+ FLAG_EMBED = 0x0004\r
+ FLAG_NESTING = 0x0008\r
+ \r
+ __slots__ = ["name", "conflags"]\r
+ def __init__(self, name, flags = 0):\r
+ if name is not None:\r
+ if type(name) is not str:\r
+ raise TypeError("name must be a string or None", name)\r
+ if name == "_" or name.startswith("<"):\r
+ raise ValueError("reserved name", name)\r
+ self.name = name\r
+ self.conflags = flags\r
+ def __repr__(self):\r
+ return "%s(%r)" % (self.__class__.__name__, self.name)\r
+ \r
+ def _set_flag(self, flag):\r
+ self.conflags |= flag\r
+ def _clear_flag(self, flag):\r
+ self.conflags &= ~flag\r
+ def _inherit_flags(self, *subcons):\r
+ for sc in subcons:\r
+ self._set_flag(sc.conflags)\r
+ def _is_flag(self, flag):\r
+ return bool(self.conflags & flag)\r
+ \r
+ def __getstate__(self):\r
+ attrs = {}\r
+ if hasattr(self, "__dict__"):\r
+ attrs.update(self.__dict__)\r
+ slots = []\r
+ c = self.__class__\r
+ while c is not None:\r
+ if hasattr(c, "__slots__"):\r
+ slots.extend(c.__slots__)\r
+ c = c.__base__\r
+ for name in slots:\r
+ if hasattr(self, name):\r
+ attrs[name] = getattr(self, name)\r
+ return attrs\r
+ def __setstate__(self, attrs):\r
+ for name, value in attrs.iteritems():\r
+ setattr(self, name, value)\r
+ def __copy__(self):\r
+ """returns a copy of this construct"""\r
+ self2 = object.__new__(self.__class__)\r
+ self2.__setstate__(self.__getstate__())\r
+ return self2\r
+ \r
+ def parse(self, data):\r
+ """parses data given as a buffer or a string (in-memory)"""\r
+ return self.parse_stream(StringIO(data))\r
+ def parse_stream(self, stream):\r
+ """parses data read directly from a stream"""\r
+ return self._parse(stream, AttrDict())\r
+ def _parse(self, stream, context):\r
+ raise NotImplementedError()\r
+ \r
+ def build(self, obj):\r
+ """builds an object in a string (in memory)"""\r
+ stream = StringIO()\r
+ self.build_stream(obj, stream)\r
+ return stream.getvalue()\r
+ def build_stream(self, obj, stream):\r
+ """builds an object into a stream"""\r
+ self._build(obj, stream, AttrDict())\r
+ def _build(self, obj, stream, context):\r
+ raise NotImplementedError()\r
+ \r
+ def sizeof(self, context = None):\r
+ """calculates the size of the construct (if possible) using the \r
+ given context"""\r
+ if context is None:\r
+ context = AttrDict()\r
+ return self._sizeof(context)\r
+ def _sizeof(self, context):\r
+ raise SizeofError("can't calculate size")\r
+\r
+class Subconstruct(Construct):\r
+ """\r
+ Abstract subconstruct (wraps an inner construct, inheriting it's \r
+ name and flags). \r
+ \r
+ Parameters:\r
+ * subcon - the construct to wrap\r
+ """\r
+ __slots__ = ["subcon"]\r
+ def __init__(self, subcon):\r
+ Construct.__init__(self, subcon.name, subcon.conflags)\r
+ self.subcon = subcon\r
+ def _parse(self, stream, context):\r
+ return self.subcon._parse(stream, context)\r
+ def _build(self, obj, stream, context):\r
+ self.subcon._build(obj, stream, context)\r
+ def _sizeof(self, context):\r
+ return self.subcon._sizeof(context)\r
+\r
+class Adapter(Subconstruct):\r
+ """\r
+ Abstract adapter: calls _decode for parsing and _encode for building.\r
+ \r
+ Parameters:\r
+ * subcon - the construct to wrap\r
+ """\r
+ __slots__ = []\r
+ def _parse(self, stream, context):\r
+ return self._decode(self.subcon._parse(stream, context), context)\r
+ def _build(self, obj, stream, context):\r
+ self.subcon._build(self._encode(obj, context), stream, context)\r
+ def _decode(self, obj, context):\r
+ raise NotImplementedError()\r
+ def _encode(self, obj, context):\r
+ raise NotImplementedError()\r
+\r
+\r
+#===============================================================================\r
+# primitives\r
+#===============================================================================\r
+def _read_stream(stream, length):\r
+ if length < 0:\r
+ raise ValueError("length must be >= 0", length)\r
+ data = stream.read(length)\r
+ if len(data) != length:\r
+ raise FieldError("expected %d, found %d" % (length, len(data)))\r
+ return data\r
+\r
+def _write_stream(stream, length, data):\r
+ if length < 0:\r
+ raise ValueError("length must be >= 0", length)\r
+ if len(data) != length:\r
+ raise FieldError("expected %d, found %d" % (length, len(data)))\r
+ stream.write(data)\r
+\r
+class StaticField(Construct):\r
+ """\r
+ A field of a fixed size\r
+ \r
+ Parameters:\r
+ * name - the name of the field\r
+ * length - the length (an integer)\r
+ \r
+ Example:\r
+ StaticField("foo", 5)\r
+ """\r
+ __slots__ = ["length"]\r
+ def __init__(self, name, length):\r
+ Construct.__init__(self, name)\r
+ self.length = length\r
+ def _parse(self, stream, context):\r
+ return _read_stream(stream, self.length)\r
+ def _build(self, obj, stream, context):\r
+ _write_stream(stream, self.length, obj)\r
+ def _sizeof(self, context):\r
+ return self.length\r
+\r
+class FormatField(StaticField):\r
+ """\r
+ A field that uses python's built-in struct module to pack/unpack data\r
+ according to a format string.\r
+ Note: this field has been originally implemented as an Adapter, but it \r
+ was made a construct for performance reasons.\r
+ \r
+ Parameters:\r
+ * name - the name\r
+ * endianity - "<" for little endian, ">" for big endian, or "=" for native\r
+ * format - a single format character\r
+ \r
+ Example:\r
+ FormatField("foo", ">", "L")\r
+ """\r
+ __slots__ = ["packer"]\r
+ def __init__(self, name, endianity, format):\r
+ if endianity not in (">", "<", "="):\r
+ raise ValueError("endianity must be be '=', '<', or '>'", \r
+ endianity)\r
+ if len(format) != 1:\r
+ raise ValueError("must specify one and only one format char")\r
+ self.packer = Packer(endianity + format)\r
+ StaticField.__init__(self, name, self.packer.size)\r
+ def __getstate__(self):\r
+ attrs = StaticField.__getstate__(self)\r
+ attrs["packer"] = attrs["packer"].format\r
+ return attrs\r
+ def __setstate__(self, attrs):\r
+ attrs["packer"] = Packer(attrs["packer"])\r
+ return StaticField.__setstate__(attrs)\r
+ def _parse(self, stream, context):\r
+ try:\r
+ return self.packer.unpack(_read_stream(stream, self.length))[0]\r
+ except Exception, ex:\r
+ raise FieldError(ex)\r
+ def _build(self, obj, stream, context):\r
+ try:\r
+ _write_stream(stream, self.length, self.packer.pack(obj))\r
+ except Exception, ex:\r
+ raise FieldError(ex)\r
+\r
+class MetaField(Construct):\r
+ """\r
+ A field of a meta-length. The length is computed at runtime based on\r
+ the context.\r
+ \r
+ Parameters:\r
+ * name - the name of the field\r
+ * lengthfunc - a function that takes the context as a parameter and return\r
+ the length of the field\r
+ \r
+ Example:\r
+ MetaField("foo", lambda ctx: 5)\r
+ """\r
+ __slots__ = ["lengthfunc"]\r
+ def __init__(self, name, lengthfunc):\r
+ Construct.__init__(self, name)\r
+ self.lengthfunc = lengthfunc\r
+ self._set_flag(self.FLAG_DYNAMIC)\r
+ def _parse(self, stream, context):\r
+ return _read_stream(stream, self.lengthfunc(context))\r
+ def _build(self, obj, stream, context):\r
+ _write_stream(stream, self.lengthfunc(context), obj)\r
+ def _sizeof(self, context):\r
+ return self.lengthfunc(context)\r
+\r
+\r
+#===============================================================================\r
+# arrays and repeaters\r
+#===============================================================================\r
+class MetaArray(Subconstruct):\r
+ """\r
+ An array (repeater) of a meta-count. The array will iterate exactly \r
+ `countfunc()` times. Will raise ArrayError if less elements are found.\r
+ See also Array, Range and RepeatUntil.\r
+ \r
+ Parameters:\r
+ * countfunc - a function that takes the context as a parameter and returns\r
+ the number of elements of the array (count)\r
+ * subcon - the subcon to repeat `countfunc()` times\r
+ \r
+ Example:\r
+ MetaArray(lambda ctx: 5, UBInt8("foo"))\r
+ """\r
+ __slots__ = ["countfunc"]\r
+ def __init__(self, countfunc, subcon):\r
+ Subconstruct.__init__(self, subcon)\r
+ self.countfunc = countfunc\r
+ self._clear_flag(self.FLAG_COPY_CONTEXT)\r
+ self._set_flag(self.FLAG_DYNAMIC)\r
+ def _parse(self, stream, context):\r
+ obj = ListContainer()\r
+ c = 0\r
+ count = self.countfunc(context)\r
+ try:\r
+ if self.subcon.conflags & self.FLAG_COPY_CONTEXT:\r
+ while c < count:\r
+ obj.append(self.subcon._parse(stream, context.__copy__()))\r
+ c += 1\r
+ else:\r
+ while c < count:\r
+ obj.append(self.subcon._parse(stream, context))\r
+ c += 1\r
+ except ConstructError, ex:\r
+ raise ArrayError("expected %d, found %d" % (count, c), ex)\r
+ return obj\r
+ def _build(self, obj, stream, context):\r
+ count = self.countfunc(context)\r
+ if len(obj) != count:\r
+ raise ArrayError("expected %d, found %d" % (count, len(obj)))\r
+ if self.subcon.conflags & self.FLAG_COPY_CONTEXT:\r
+ for subobj in obj:\r
+ self.subcon._build(subobj, stream, context.__copy__())\r
+ else:\r
+ for subobj in obj:\r
+ self.subcon._build(subobj, stream, context)\r
+ def _sizeof(self, context):\r
+ return self.subcon._sizeof(context) * self.countfunc(context)\r
+\r
+class Range(Subconstruct):\r
+ """\r
+ A range-array. The subcon will iterate between `mincount` to `maxcount`\r
+ times. If less than `mincount` elements are found, raises RangeError.\r
+ See also GreedyRange and OptionalGreedyRange.\r
+ \r
+ Notes:\r
+ * requires a seekable stream.\r
+ \r
+ Parameters:\r
+ * mincount - the minimal count (an integer)\r
+ * maxcount - the maximal count (an integer)\r
+ * subcon - the subcon to repeat\r
+ \r
+ Example:\r
+ Range(5, 8, UBInt8("foo"))\r
+ """\r
+ __slots__ = ["mincount", "maxcout"]\r
+ def __init__(self, mincount, maxcout, subcon):\r
+ Subconstruct.__init__(self, subcon)\r
+ self.mincount = mincount\r
+ self.maxcout = maxcout\r
+ self._clear_flag(self.FLAG_COPY_CONTEXT)\r
+ self._set_flag(self.FLAG_DYNAMIC)\r
+ def _parse(self, stream, context):\r
+ obj = ListContainer()\r
+ c = 0\r
+ try:\r
+ if self.subcon.conflags & self.FLAG_COPY_CONTEXT:\r
+ while c < self.maxcout:\r
+ pos = stream.tell()\r
+ obj.append(self.subcon._parse(stream, context.__copy__()))\r
+ c += 1\r
+ else:\r
+ while c < self.maxcout:\r
+ pos = stream.tell()\r
+ obj.append(self.subcon._parse(stream, context))\r
+ c += 1\r
+ except ConstructError:\r
+ if c < self.mincount:\r
+ raise RangeError("expected %d to %d, found %d" % \r
+ (self.mincount, self.maxcout, c))\r
+ stream.seek(pos)\r
+ return obj\r
+ def _build(self, obj, stream, context):\r
+ if len(obj) < self.mincount or len(obj) > self.maxcout:\r
+ raise RangeError("expected %d to %d, found %d" % \r
+ (self.mincount, self.maxcout, len(obj)))\r
+ cnt = 0\r
+ try:\r
+ if self.subcon.conflags & self.FLAG_COPY_CONTEXT:\r
+ for subobj in obj:\r
+ self.subcon._build(subobj, stream, context.__copy__())\r
+ cnt += 1\r
+ else:\r
+ for subobj in obj:\r
+ self.subcon._build(subobj, stream, context)\r
+ cnt += 1\r
+ except ConstructError:\r
+ if cnt < self.mincount:\r
+ raise RangeError("expected %d to %d, found %d" % \r
+ (self.mincount, self.maxcout, len(obj)))\r
+ def _sizeof(self, context):\r
+ raise SizeofError("can't calculate size")\r
+\r
+class RepeatUntil(Subconstruct):\r
+ """\r
+ An array that repeat until the predicate indicates it to stop. Note that\r
+ the last element (which caused the repeat to exit) is included in the \r
+ return value.\r
+ \r
+ Parameters:\r
+ * predicate - a predicate function that takes (obj, context) and returns\r
+ True if the stop-condition is met, or False to continue.\r
+ * subcon - the subcon to repeat.\r
+ \r
+ Example:\r
+ # will read chars until \x00 (inclusive)\r
+ RepeatUntil(lambda obj, ctx: obj == "\x00",\r
+ Field("chars", 1)\r
+ )\r
+ """\r
+ __slots__ = ["predicate"]\r
+ def __init__(self, predicate, subcon):\r
+ Subconstruct.__init__(self, subcon)\r
+ self.predicate = predicate\r
+ self._clear_flag(self.FLAG_COPY_CONTEXT)\r
+ self._set_flag(self.FLAG_DYNAMIC)\r
+ def _parse(self, stream, context):\r
+ obj = []\r
+ try:\r
+ if self.subcon.conflags & self.FLAG_COPY_CONTEXT:\r
+ while True:\r
+ subobj = self.subcon._parse(stream, context.__copy__())\r
+ obj.append(subobj)\r
+ if self.predicate(subobj, context):\r
+ break\r
+ else:\r
+ while True:\r
+ subobj = self.subcon._parse(stream, context)\r
+ obj.append(subobj)\r
+ if self.predicate(subobj, context):\r
+ break\r
+ except ConstructError, ex:\r
+ raise ArrayError("missing terminator", ex)\r
+ return obj\r
+ def _build(self, obj, stream, context):\r
+ terminated = False\r
+ if self.subcon.conflags & self.FLAG_COPY_CONTEXT:\r
+ for subobj in obj:\r
+ self.subcon._build(subobj, stream, context.__copy__())\r
+ if self.predicate(subobj, context):\r
+ terminated = True\r
+ break\r
+ else:\r
+ for subobj in obj:\r
+ self.subcon._build(subobj, stream, context.__copy__())\r
+ if self.predicate(subobj, context):\r
+ terminated = True\r
+ break\r
+ if not terminated:\r
+ raise ArrayError("missing terminator")\r
+ def _sizeof(self, context):\r
+ raise SizeofError("can't calculate size")\r
+\r
+\r
+#===============================================================================\r
+# structures and sequences\r
+#===============================================================================\r
+class Struct(Construct):\r
+ """\r
+ A sequence of named constructs, similar to structs in C. The elements are\r
+ parsed and built in the order they are defined.\r
+ See also Embedded.\r
+ \r
+ Parameters:\r
+ * name - the name of the structure\r
+ * subcons - a sequence of subconstructs that make up this structure.\r
+ * nested - a keyword-only argument that indicates whether this struct \r
+ creates a nested context. The default is True. This parameter is \r
+ considered "advanced usage", and may be removed in the future.\r
+ \r
+ Example:\r
+ Struct("foo",\r
+ UBInt8("first_element"),\r
+ UBInt16("second_element"),\r
+ Padding(2),\r
+ UBInt8("third_element"),\r
+ )\r
+ """\r
+ __slots__ = ["subcons", "nested"]\r
+ def __init__(self, name, *subcons, **kw):\r
+ self.nested = kw.pop("nested", True)\r
+ if kw:\r
+ raise TypeError("the only keyword argument accepted is 'nested'", kw)\r
+ Construct.__init__(self, name)\r
+ self.subcons = subcons\r
+ self._inherit_flags(*subcons)\r
+ self._clear_flag(self.FLAG_EMBED)\r
+ def _parse(self, stream, context):\r
+ if "<obj>" in context:\r
+ obj = context["<obj>"]\r
+ del context["<obj>"]\r
+ else:\r
+ obj = Container()\r
+ if self.nested:\r
+ context = AttrDict(_ = context)\r
+ for sc in self.subcons:\r
+ if sc.conflags & self.FLAG_EMBED:\r
+ context["<obj>"] = obj\r
+ sc._parse(stream, context)\r
+ else:\r
+ subobj = sc._parse(stream, context)\r
+ if sc.name is not None:\r
+ obj[sc.name] = subobj\r
+ context[sc.name] = subobj\r
+ return obj\r
+ def _build(self, obj, stream, context):\r
+ if "<unnested>" in context:\r
+ del context["<unnested>"]\r
+ elif self.nested:\r
+ context = AttrDict(_ = context)\r
+ for sc in self.subcons:\r
+ if sc.conflags & self.FLAG_EMBED:\r
+ context["<unnested>"] = True\r
+ subobj = obj\r
+ elif sc.name is None:\r
+ subobj = None\r
+ else:\r
+ subobj = getattr(obj, sc.name)\r
+ context[sc.name] = subobj\r
+ sc._build(subobj, stream, context)\r
+ def _sizeof(self, context):\r
+ if self.nested:\r
+ context = AttrDict(_ = context)\r
+ return sum(sc._sizeof(context) for sc in self.subcons)\r
+\r
+class Sequence(Struct):\r
+ """\r
+ A sequence of unnamed constructs. The elements are parsed and built in the\r
+ order they are defined.\r
+ See also Embedded.\r
+ \r
+ Parameters:\r
+ * name - the name of the structure\r
+ * subcons - a sequence of subconstructs that make up this structure.\r
+ * nested - a keyword-only argument that indicates whether this struct \r
+ creates a nested context. The default is True. This parameter is \r
+ considered "advanced usage", and may be removed in the future.\r
+ \r
+ Example:\r
+ Sequence("foo",\r
+ UBInt8("first_element"),\r
+ UBInt16("second_element"),\r
+ Padding(2),\r
+ UBInt8("third_element"),\r
+ )\r
+ """\r
+ __slots__ = []\r
+ def _parse(self, stream, context):\r
+ if "<obj>" in context:\r
+ obj = context["<obj>"]\r
+ del context["<obj>"]\r
+ else:\r
+ obj = ListContainer()\r
+ if self.nested:\r
+ context = AttrDict(_ = context)\r
+ for sc in self.subcons:\r
+ if sc.conflags & self.FLAG_EMBED:\r
+ context["<obj>"] = obj\r
+ sc._parse(stream, context)\r
+ else:\r
+ subobj = sc._parse(stream, context)\r
+ if sc.name is not None:\r
+ obj.append(subobj)\r
+ context[sc.name] = subobj\r
+ return obj\r
+ def _build(self, obj, stream, context):\r
+ if "<unnested>" in context:\r
+ del context["<unnested>"]\r
+ elif self.nested:\r
+ context = AttrDict(_ = context)\r
+ objiter = iter(obj)\r
+ for sc in self.subcons:\r
+ if sc.conflags & self.FLAG_EMBED:\r
+ context["<unnested>"] = True\r
+ subobj = objiter\r
+ elif sc.name is None:\r
+ subobj = None\r
+ else:\r
+ subobj = objiter.next()\r
+ context[sc.name] = subobj\r
+ sc._build(subobj, stream, context)\r
+\r
+class Union(Construct):\r
+ """\r
+ a set of overlapping fields (like unions in C). when parsing, \r
+ all fields read the same data; when building, only the first subcon\r
+ (called "master") is used. \r
+ \r
+ Parameters:\r
+ * name - the name of the union\r
+ * master - the master subcon, i.e., the subcon used for building and \r
+ calculating the total size\r
+ * subcons - additional subcons\r
+ \r
+ Example:\r
+ Union("what_are_four_bytes",\r
+ UBInt32("one_dword"),\r
+ Struct("two_words", UBInt16("first"), UBInt16("second")),\r
+ Struct("four_bytes", \r
+ UBInt8("a"), \r
+ UBInt8("b"), \r
+ UBInt8("c"), \r
+ UBInt8("d")\r
+ ),\r
+ )\r
+ """\r
+ __slots__ = ["parser", "builder"]\r
+ def __init__(self, name, master, *subcons, **kw):\r
+ Construct.__init__(self, name)\r
+ args = [Peek(sc) for sc in subcons]\r
+ args.append(MetaField(None, lambda ctx: master._sizeof(ctx)))\r
+ self.parser = Struct(name, Peek(master, perform_build = True), *args)\r
+ self.builder = Struct(name, master)\r
+ def _parse(self, stream, context):\r
+ return self.parser._parse(stream, context)\r
+ def _build(self, obj, stream, context):\r
+ return self.builder._build(obj, stream, context)\r
+ def _sizeof(self, context):\r
+ return self.builder._sizeof(context)\r
+\r
+#===============================================================================\r
+# conditional\r
+#===============================================================================\r
+class Switch(Construct):\r
+ """\r
+ A conditional branch. Switch will choose the case to follow based on\r
+ the return value of keyfunc. If no case is matched, and no default value \r
+ is given, SwitchError will be raised.\r
+ See also Pass.\r
+ \r
+ Parameters:\r
+ * name - the name of the construct\r
+ * keyfunc - a function that takes the context and returns a key, which \r
+ will ne used to choose the relevant case.\r
+ * cases - a dictionary mapping keys to constructs. the keys can be any \r
+ values that may be returned by keyfunc.\r
+ * default - a default value to use when the key is not found in the cases.\r
+ if not supplied, an exception will be raised when the key is not found.\r
+ You can use the builtin construct Pass for 'do-nothing'.\r
+ * include_key - whether or not to include the key in the return value\r
+ of parsing. defualt is False.\r
+ \r
+ Example:\r
+ Struct("foo",\r
+ UBInt8("type"),\r
+ Switch("value", lambda ctx: ctx.type, {\r
+ 1 : UBInt8("spam"),\r
+ 2 : UBInt16("spam"),\r
+ 3 : UBInt32("spam"),\r
+ 4 : UBInt64("spam"),\r
+ }\r
+ ),\r
+ )\r
+ """\r
+ \r
+ class NoDefault(Construct):\r
+ def _parse(self, stream, context):\r
+ raise SwitchError("no default case defined")\r
+ def _build(self, obj, stream, context):\r
+ raise SwitchError("no default case defined")\r
+ def _sizeof(self, context):\r
+ raise SwitchError("no default case defined")\r
+ NoDefault = NoDefault("NoDefault")\r
+ \r
+ __slots__ = ["subcons", "keyfunc", "cases", "default", "include_key"]\r
+ \r
+ def __init__(self, name, keyfunc, cases, default = NoDefault, \r
+ include_key = False):\r
+ Construct.__init__(self, name)\r
+ self._inherit_flags(*cases.values())\r
+ self.keyfunc = keyfunc\r
+ self.cases = cases\r
+ self.default = default\r
+ self.include_key = include_key\r
+ self._inherit_flags(*cases.values())\r
+ self._set_flag(self.FLAG_DYNAMIC)\r
+ def _parse(self, stream, context):\r
+ key = self.keyfunc(context)\r
+ obj = self.cases.get(key, self.default)._parse(stream, context)\r
+ if self.include_key:\r
+ return key, obj\r
+ else:\r
+ return obj\r
+ def _build(self, obj, stream, context):\r
+ if self.include_key:\r
+ key, obj = obj\r
+ else:\r
+ key = self.keyfunc(context)\r
+ case = self.cases.get(key, self.default)\r
+ case._build(obj, stream, context)\r
+ def _sizeof(self, context):\r
+ case = self.cases.get(self.keyfunc(context), self.default)\r
+ return case._sizeof(context)\r
+\r
+class Select(Construct):\r
+ """\r
+ Selects the first matching subconstruct. It will literally try each of\r
+ the subconstructs, until one matches.\r
+ \r
+ Notes:\r
+ * requires a seekable stream.\r
+ \r
+ Parameters:\r
+ * name - the name of the construct\r
+ * subcons - the subcons to try (order-sensitive)\r
+ * include_name - a keyword only argument, indicating whether to include \r
+ the name of the selected subcon in the return value of parsing. default\r
+ is false.\r
+ \r
+ Example:\r
+ Select("foo",\r
+ UBInt64("large"),\r
+ UBInt32("medium"),\r
+ UBInt16("small"),\r
+ UBInt8("tiny"),\r
+ )\r
+ """\r
+ __slots__ = ["subcons", "include_name"]\r
+ def __init__(self, name, *subcons, **kw):\r
+ include_name = kw.pop("include_name", False)\r
+ if kw:\r
+ raise TypeError("the only keyword argument accepted "\r
+ "is 'include_name'", kw)\r
+ Construct.__init__(self, name)\r
+ self.subcons = subcons\r
+ self.include_name = include_name\r
+ self._inherit_flags(*subcons)\r
+ self._set_flag(self.FLAG_DYNAMIC)\r
+ def _parse(self, stream, context):\r
+ for sc in self.subcons:\r
+ pos = stream.tell()\r
+ context2 = context.__copy__()\r
+ try:\r
+ obj = sc._parse(stream, context2)\r
+ except ConstructError:\r
+ stream.seek(pos)\r
+ else:\r
+ context.__update__(context2)\r
+ if self.include_name:\r
+ return sc.name, obj\r
+ else:\r
+ return obj\r
+ raise SelectError("no subconstruct matched")\r
+ def _build(self, obj, stream, context):\r
+ if self.include_name:\r
+ name, obj = obj\r
+ for sc in self.subcons:\r
+ if sc.name == name:\r
+ sc._build(obj, stream, context)\r
+ return\r
+ else: \r
+ for sc in self.subcons:\r
+ stream2 = StringIO()\r
+ context2 = context.__copy__()\r
+ try:\r
+ sc._build(obj, stream2, context2)\r
+ except Exception:\r
+ pass\r
+ else:\r
+ context.__update__(context2)\r
+ stream.write(stream2.getvalue())\r
+ return\r
+ raise SelectError("no subconstruct matched", obj)\r
+ def _sizeof(self, context):\r
+ raise SizeofError("can't calculate size")\r
+\r
+\r
+#===============================================================================\r
+# stream manipulation\r
+#===============================================================================\r
+class Pointer(Subconstruct):\r
+ """\r
+ Changes the stream position to a given offset, where the construction\r
+ should take place, and restores the stream position when finished.\r
+ See also Anchor, OnDemand and OnDemandPointer.\r
+ \r
+ Notes:\r
+ * requires a seekable stream.\r
+ \r
+ Parameters:\r
+ * offsetfunc: a function that takes the context and returns an absolute \r
+ stream position, where the construction would take place\r
+ * subcon - the subcon to use at `offsetfunc()`\r
+ \r
+ Example:\r
+ Struct("foo",\r
+ UBInt32("spam_pointer"),\r
+ Pointer(lambda ctx: ctx.spam_pointer,\r
+ Array(5, UBInt8("spam"))\r
+ )\r
+ )\r
+ """\r
+ __slots__ = ["offsetfunc"]\r
+ def __init__(self, offsetfunc, subcon):\r
+ Subconstruct.__init__(self, subcon)\r
+ self.offsetfunc = offsetfunc\r
+ def _parse(self, stream, context):\r
+ newpos = self.offsetfunc(context)\r
+ origpos = stream.tell()\r
+ stream.seek(newpos)\r
+ obj = self.subcon._parse(stream, context)\r
+ stream.seek(origpos)\r
+ return obj\r
+ def _build(self, obj, stream, context):\r
+ newpos = self.offsetfunc(context)\r
+ origpos = stream.tell()\r
+ stream.seek(newpos)\r
+ self.subcon._build(obj, stream, context)\r
+ stream.seek(origpos)\r
+ def _sizeof(self, context):\r
+ return 0\r
+\r
+class Peek(Subconstruct):\r
+ """\r
+ Peeks at the stream: parses without changing the stream position.\r
+ See also Union. If the end of the stream is reached when peeking,\r
+ returns None.\r
+ \r
+ Notes:\r
+ * requires a seekable stream.\r
+ \r
+ Parameters:\r
+ * subcon - the subcon to peek at\r
+ * perform_build - whether or not to perform building. by default this \r
+ parameter is set to False, meaning building is a no-op.\r
+ \r
+ Example:\r
+ Peek(UBInt8("foo"))\r
+ """\r
+ __slots__ = ["perform_build"]\r
+ def __init__(self, subcon, perform_build = False):\r
+ Subconstruct.__init__(self, subcon)\r
+ self.perform_build = perform_build\r
+ def _parse(self, stream, context):\r
+ pos = stream.tell()\r
+ try:\r
+ try:\r
+ return self.subcon._parse(stream, context)\r
+ except FieldError:\r
+ pass\r
+ finally:\r
+ stream.seek(pos)\r
+ def _build(self, obj, stream, context):\r
+ if self.perform_build:\r
+ self.subcon._build(obj, stream, context)\r
+ def _sizeof(self, context):\r
+ return 0\r
+\r
+class OnDemand(Subconstruct):\r
+ """\r
+ Allows for on-demand (lazy) parsing. When parsing, it will return a \r
+ LazyContainer that represents a pointer to the data, but does not actually\r
+ parses it from stream until it's "demanded".\r
+ By accessing the 'value' property of LazyContainers, you will demand the \r
+ data from the stream. The data will be parsed and cached for later use.\r
+ You can use the 'has_value' property to know whether the data has already \r
+ been demanded.\r
+ See also OnDemandPointer.\r
+ \r
+ Notes:\r
+ * requires a seekable stream.\r
+ \r
+ Parameters:\r
+ * subcon - \r
+ * advance_stream - whether or not to advance the stream position. by \r
+ default this is True, but if subcon is a pointer, this should be False.\r
+ * force_build - whether or not to force build. If set to False, and the\r
+ LazyContainer has not been demaned, building is a no-op.\r
+ \r
+ Example:\r
+ OnDemand(Array(10000, UBInt8("foo"))\r
+ """\r
+ __slots__ = ["advance_stream", "force_build"]\r
+ def __init__(self, subcon, advance_stream = True, force_build = True):\r
+ Subconstruct.__init__(self, subcon)\r
+ self.advance_stream = advance_stream\r
+ self.force_build = force_build\r
+ def _parse(self, stream, context):\r
+ obj = LazyContainer(self.subcon, stream, stream.tell(), context)\r
+ if self.advance_stream:\r
+ stream.seek(self.subcon._sizeof(context), 1)\r
+ return obj\r
+ def _build(self, obj, stream, context):\r
+ if not isinstance(obj, LazyContainer):\r
+ self.subcon._build(obj, stream, context)\r
+ elif self.force_build or obj.has_value:\r
+ self.subcon._build(obj.value, stream, context)\r
+ elif self.advance_stream:\r
+ stream.seek(self.subcon._sizeof(context), 1)\r
+\r
+class Buffered(Subconstruct):\r
+ """\r
+ Creates an in-memory buffered stream, which can undergo encoding and \r
+ decoding prior to being passed on to the subconstruct.\r
+ See also Bitwise.\r
+ \r
+ Note:\r
+ * Do not use pointers inside Buffered\r
+ \r
+ Parameters:\r
+ * subcon - the subcon which will operate on the buffer\r
+ * encoder - a function that takes a string and returns an encoded\r
+ string (used after building)\r
+ * decoder - a function that takes a string and returns a decoded\r
+ string (used before parsing)\r
+ * resizer - a function that takes the size of the subcon and "adjusts"\r
+ or "resizes" it according to the encoding/decoding process.\r
+ \r
+ Example:\r
+ Buffered(BitField("foo", 16),\r
+ encoder = decode_bin,\r
+ decoder = encode_bin,\r
+ resizer = lambda size: size / 8,\r
+ )\r
+ """\r
+ __slots__ = ["encoder", "decoder", "resizer"]\r
+ def __init__(self, subcon, decoder, encoder, resizer):\r
+ Subconstruct.__init__(self, subcon)\r
+ self.encoder = encoder\r
+ self.decoder = decoder\r
+ self.resizer = resizer\r
+ def _parse(self, stream, context):\r
+ data = _read_stream(stream, self._sizeof(context))\r
+ stream2 = StringIO(self.decoder(data))\r
+ return self.subcon._parse(stream2, context)\r
+ def _build(self, obj, stream, context):\r
+ size = self._sizeof(context)\r
+ stream2 = StringIO()\r
+ self.subcon._build(obj, stream2, context)\r
+ data = self.encoder(stream2.getvalue())\r
+ assert len(data) == size\r
+ _write_stream(stream, self._sizeof(context), data)\r
+ def _sizeof(self, context):\r
+ return self.resizer(self.subcon._sizeof(context))\r
+\r
+class Restream(Subconstruct):\r
+ """\r
+ Wraps the stream with a read-wrapper (for parsing) or a \r
+ write-wrapper (for building). The stream wrapper can buffer the data\r
+ internally, reading it from- or writing it to the underlying stream \r
+ as needed. For example, BitStreamReader reads whole bytes from the \r
+ underlying stream, but returns them as individual bits. \r
+ See also Bitwise.\r
+ \r
+ When the parsing or building is done, the stream's close method \r
+ will be invoked. It can perform any finalization needed for the stream\r
+ wrapper, but it must not close the underlying stream.\r
+ \r
+ Note:\r
+ * Do not use pointers inside Restream\r
+ \r
+ Parameters:\r
+ * subcon - the subcon\r
+ * stream_reader - the read-wrapper\r
+ * stream_writer - the write wrapper\r
+ * resizer - a function that takes the size of the subcon and "adjusts"\r
+ or "resizes" it according to the encoding/decoding process.\r
+ \r
+ Example:\r
+ Restream(BitField("foo", 16),\r
+ stream_reader = BitStreamReader,\r
+ stream_writer = BitStreamWriter,\r
+ resizer = lambda size: size / 8,\r
+ )\r
+ """\r
+ __slots__ = ["stream_reader", "stream_writer", "resizer"]\r
+ def __init__(self, subcon, stream_reader, stream_writer, resizer):\r
+ Subconstruct.__init__(self, subcon)\r
+ self.stream_reader = stream_reader\r
+ self.stream_writer = stream_writer\r
+ self.resizer = resizer\r
+ def _parse(self, stream, context):\r
+ stream2 = self.stream_reader(stream)\r
+ obj = self.subcon._parse(stream2, context)\r
+ stream2.close()\r
+ return obj\r
+ def _build(self, obj, stream, context):\r
+ stream2 = self.stream_writer(stream)\r
+ self.subcon._build(obj, stream2, context)\r
+ stream2.close()\r
+ def _sizeof(self, context):\r
+ return self.resizer(self.subcon._sizeof(context))\r
+\r
+\r
+#===============================================================================\r
+# miscellaneous\r
+#===============================================================================\r
+class Reconfig(Subconstruct):\r
+ """\r
+ Reconfigures a subconstruct. Reconfig can be used to change the name and\r
+ set and clear flags of the inner subcon.\r
+ \r
+ Parameters:\r
+ * name - the new name\r
+ * subcon - the subcon to reconfigure\r
+ * setflags - the flags to set (default is 0)\r
+ * clearflags - the flags to clear (default is 0)\r
+ \r
+ Example:\r
+ Reconfig("foo", UBInt8("bar"))\r
+ """\r
+ __slots__ = []\r
+ def __init__(self, name, subcon, setflags = 0, clearflags = 0):\r
+ Construct.__init__(self, name, subcon.conflags)\r
+ self.subcon = subcon\r
+ self._set_flag(setflags)\r
+ self._clear_flag(clearflags)\r
+\r
+class Anchor(Construct):\r
+ """\r
+ Returns the "anchor" (stream position) at the point where it's inserted.\r
+ Useful for adjusting relative offsets to absolute positions, or to measure\r
+ sizes of constructs.\r
+ absolute pointer = anchor + relative offset\r
+ size = anchor_after - anchor_before\r
+ See also Pointer.\r
+ \r
+ Notes:\r
+ * requires a seekable stream.\r
+ \r
+ Parameters:\r
+ * name - the name of the anchor\r
+ \r
+ Example:\r
+ Struct("foo",\r
+ Anchor("base"),\r
+ UBInt8("relative_offset"),\r
+ Pointer(lambda ctx: ctx.relative_offset + ctx.base,\r
+ UBInt8("data")\r
+ )\r
+ )\r
+ """\r
+ __slots__ = []\r
+ def _parse(self, stream, context):\r
+ return stream.tell()\r
+ def _build(self, obj, stream, context):\r
+ context[self.name] = stream.tell()\r
+ def _sizeof(self, context):\r
+ return 0\r
+\r
+class Value(Construct):\r
+ """\r
+ A computed value.\r
+ \r
+ Parameters:\r
+ * name - the name of the value\r
+ * func - a function that takes the context and return the computed value\r
+ \r
+ Example:\r
+ Struct("foo",\r
+ UBInt8("width"),\r
+ UBInt8("height"),\r
+ Value("total_pixels", lambda ctx: ctx.width * ctx.height),\r
+ )\r
+ """\r
+ __slots__ = ["func"]\r
+ def __init__(self, name, func):\r
+ Construct.__init__(self, name)\r
+ self.func = func\r
+ self._set_flag(self.FLAG_DYNAMIC)\r
+ def _parse(self, stream, context):\r
+ return self.func(context)\r
+ def _build(self, obj, stream, context):\r
+ context[self.name] = self.func(context)\r
+ def _sizeof(self, context):\r
+ return 0\r
+\r
+#class Dynamic(Construct):\r
+# """\r
+# Dynamically creates a construct and uses it for parsing and building.\r
+# This allows you to create change the construction tree on the fly.\r
+# Deprecated.\r
+# \r
+# Parameters:\r
+# * name - the name of the construct\r
+# * factoryfunc - a function that takes the context and returns a new \r
+# construct object which will be used for parsing and building.\r
+# \r
+# Example:\r
+# def factory(ctx):\r
+# if ctx.bar == 8:\r
+# return UBInt8("spam")\r
+# if ctx.bar == 9:\r
+# return String("spam", 9)\r
+# \r
+# Struct("foo",\r
+# UBInt8("bar"),\r
+# Dynamic("spam", factory),\r
+# )\r
+# """\r
+# __slots__ = ["factoryfunc"]\r
+# def __init__(self, name, factoryfunc):\r
+# Construct.__init__(self, name, self.FLAG_COPY_CONTEXT)\r
+# self.factoryfunc = factoryfunc\r
+# self._set_flag(self.FLAG_DYNAMIC)\r
+# def _parse(self, stream, context):\r
+# return self.factoryfunc(context)._parse(stream, context)\r
+# def _build(self, obj, stream, context):\r
+# return self.factoryfunc(context)._build(obj, stream, context)\r
+# def _sizeof(self, context):\r
+# return self.factoryfunc(context)._sizeof(context)\r
+\r
+class LazyBound(Construct):\r
+ """\r
+ Lazily bound construct, useful for constructs that need to make cyclic \r
+ references (linked-lists, expression trees, etc.).\r
+ \r
+ Parameters:\r
+ \r
+ \r
+ Example:\r
+ foo = Struct("foo",\r
+ UBInt8("bar"),\r
+ LazyBound("next", lambda: foo),\r
+ )\r
+ """\r
+ __slots__ = ["bindfunc", "bound"]\r
+ def __init__(self, name, bindfunc):\r
+ Construct.__init__(self, name)\r
+ self.bound = None\r
+ self.bindfunc = bindfunc\r
+ def _parse(self, stream, context):\r
+ if self.bound is None:\r
+ self.bound = self.bindfunc()\r
+ return self.bound._parse(stream, context)\r
+ def _build(self, obj, stream, context):\r
+ if self.bound is None:\r
+ self.bound = self.bindfunc()\r
+ self.bound._build(obj, stream, context)\r
+ def _sizeof(self, context):\r
+ if self.bound is None:\r
+ self.bound = self.bindfunc()\r
+ return self.bound._sizeof(context)\r
+\r
+class Pass(Construct):\r
+ """\r
+ A do-nothing construct, useful as the default case for Switch, or\r
+ to indicate Enums.\r
+ See also Switch and Enum.\r
+ \r
+ Notes:\r
+ * this construct is a singleton. do not try to instatiate it, as it \r
+ will not work :)\r
+ \r
+ Example:\r
+ Pass\r
+ """\r
+ __slots__ = []\r
+ def _parse(self, stream, context):\r
+ pass\r
+ def _build(self, obj, stream, context):\r
+ assert obj is None\r
+ def _sizeof(self, context):\r
+ return 0\r
+Pass = Pass(None)\r
+\r
+class Terminator(Construct):\r
+ """\r
+ Asserts the end of the stream has been reached at the point it's placed.\r
+ You can use this to ensure no more unparsed data follows.\r
+ \r
+ Notes:\r
+ * this construct is a singleton. do not try to instatiate it, as it \r
+ will not work :)\r
+ \r
+ Example:\r
+ Terminator\r
+ """\r
+ __slots__ = []\r
+ def _parse(self, stream, context):\r
+ if stream.read(1):\r
+ raise TerminatorError("expected end of stream")\r
+ def _build(self, obj, stream, context):\r
+ assert obj is None\r
+ def _sizeof(self, context):\r
+ return 0\r
+Terminator = Terminator(None)\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
--- /dev/null
+"""\r
+Debugging utilities for constructs\r
+"""\r
+import sys\r
+import traceback\r
+import pdb\r
+import inspect\r
+from core import Construct, Subconstruct\r
+from lib import HexString, Container, ListContainer, AttrDict\r
+\r
+\r
+class Probe(Construct):\r
+ """\r
+ A probe: dumps the context, stack frames, and stream content to the screen\r
+ to aid the debugging process.\r
+ See also Debugger.\r
+ \r
+ Parameters:\r
+ * name - the display name\r
+ * show_stream - whether or not to show stream contents. default is True. \r
+ the stream must be seekable.\r
+ * show_context - whether or not to show the context. default is True.\r
+ * show_stack - whether or not to show the upper stack frames. default \r
+ is True.\r
+ * stream_lookahead - the number of bytes to dump when show_stack is set.\r
+ default is 100.\r
+ \r
+ Example:\r
+ Struct("foo",\r
+ UBInt8("a"),\r
+ Probe("between a and b"),\r
+ UBInt8("b"),\r
+ )\r
+ """\r
+ __slots__ = [\r
+ "printname", "show_stream", "show_context", "show_stack", \r
+ "stream_lookahead"\r
+ ]\r
+ counter = 0\r
+ \r
+ def __init__(self, name = None, show_stream = True, \r
+ show_context = True, show_stack = True, \r
+ stream_lookahead = 100):\r
+ Construct.__init__(self, None)\r
+ if name is None:\r
+ Probe.counter += 1\r
+ name = "<unnamed %d>" % (Probe.counter,)\r
+ self.printname = name\r
+ self.show_stream = show_stream\r
+ self.show_context = show_context\r
+ self.show_stack = show_stack\r
+ self.stream_lookahead = stream_lookahead\r
+ def __repr__(self):\r
+ return "%s(%r)" % (self.__class__.__name__, self.printname)\r
+ def _parse(self, stream, context):\r
+ self.printout(stream, context)\r
+ def _build(self, obj, stream, context):\r
+ self.printout(stream, context)\r
+ def _sizeof(self, context):\r
+ return 0\r
+ \r
+ def printout(self, stream, context):\r
+ obj = Container()\r
+ if self.show_stream:\r
+ obj.stream_position = stream.tell()\r
+ follows = stream.read(self.stream_lookahead)\r
+ if not follows:\r
+ obj.following_stream_data = "EOF reached"\r
+ else:\r
+ stream.seek(-len(follows), 1)\r
+ obj.following_stream_data = HexString(follows)\r
+ print\r
+ \r
+ if self.show_context:\r
+ obj.context = context\r
+ \r
+ if self.show_stack:\r
+ obj.stack = ListContainer()\r
+ frames = [s[0] for s in inspect.stack()][1:-1]\r
+ frames.reverse()\r
+ for f in frames:\r
+ a = AttrDict()\r
+ a.__update__(f.f_locals)\r
+ obj.stack.append(a)\r
+ \r
+ print "=" * 80\r
+ print "Probe", self.printname\r
+ print obj\r
+ print "=" * 80\r
+\r
+class Debugger(Subconstruct):\r
+ """\r
+ A pdb-based debugger. When an exception occurs in the subcon, a debugger\r
+ will appear and allow you to debug the error (and even fix on-the-fly).\r
+ \r
+ Parameters:\r
+ * subcon - the subcon to debug\r
+ \r
+ Example:\r
+ Debugger(\r
+ Enum(UBInt8("foo"),\r
+ a = 1,\r
+ b = 2,\r
+ c = 3\r
+ )\r
+ )\r
+ """\r
+ __slots__ = ["retval"]\r
+ def _parse(self, stream, context):\r
+ try:\r
+ return self.subcon._parse(stream, context)\r
+ except Exception:\r
+ self.retval = NotImplemented\r
+ self.handle_exc("(you can set the value of 'self.retval', "\r
+ "which will be returned)")\r
+ if self.retval is NotImplemented:\r
+ raise\r
+ else:\r
+ return self.retval\r
+ def _build(self, obj, stream, context):\r
+ try:\r
+ self.subcon._build(obj, stream, context)\r
+ except Exception:\r
+ self.handle_exc()\r
+ def handle_exc(self, msg = None):\r
+ print "=" * 80\r
+ print "Debugging exception of %s:" % (self.subcon,)\r
+ print "".join(traceback.format_exception(*sys.exc_info())[1:])\r
+ if msg:\r
+ print msg\r
+ pdb.post_mortem(sys.exc_info()[2])\r
+ print "=" * 80\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
--- /dev/null
+from binary import int_to_bin, bin_to_int, swap_bytes, encode_bin, decode_bin\r
+from bitstream import BitStreamReader, BitStreamWriter\r
+from container import (Container, AttrDict, FlagsContainer, \r
+ ListContainer, LazyContainer)\r
+from hex import HexString, hexdump\r
+from utils import Packer, StringIO\r
+\r
+\r
+\r
+\r
--- /dev/null
+def int_to_bin(number, width = 32):\r
+ if number < 0:\r
+ number += 1 << width\r
+ i = width - 1\r
+ bits = ["\x00"] * width\r
+ while number and i >= 0:\r
+ bits[i] = "\x00\x01"[number & 1]\r
+ number >>= 1\r
+ i -= 1\r
+ return "".join(bits)\r
+\r
+_bit_values = {"\x00" : 0, "\x01" : 1, "0" : 0, "1" : 1}\r
+def bin_to_int(bits, signed = False):\r
+ number = 0\r
+ bias = 0\r
+ if signed and _bit_values[bits[0]] == 1:\r
+ bits = bits[1:]\r
+ bias = 1 << len(bits)\r
+ for b in bits:\r
+ number <<= 1\r
+ number |= _bit_values[b]\r
+ return number - bias\r
+\r
+def swap_bytes(bits, bytesize = 8):\r
+ i = 0\r
+ l = len(bits)\r
+ output = [""] * ((l // bytesize) + 1)\r
+ j = len(output) - 1\r
+ while i < l:\r
+ output[j] = bits[i : i + bytesize]\r
+ i += bytesize\r
+ j -= 1\r
+ return "".join(output)\r
+\r
+_char_to_bin = {}\r
+_bin_to_char = {}\r
+for i in range(256):\r
+ ch = chr(i)\r
+ bin = int_to_bin(i, 8)\r
+ _char_to_bin[ch] = bin\r
+ _bin_to_char[bin] = ch\r
+ _bin_to_char[bin] = ch\r
+\r
+def encode_bin(data):\r
+ return "".join(_char_to_bin[ch] for ch in data)\r
+\r
+def decode_bin(data):\r
+ assert len(data) & 7 == 0, "data length must be a multiple of 8"\r
+ i = 0\r
+ j = 0\r
+ l = len(data) // 8\r
+ chars = [""] * l\r
+ while j < l:\r
+ chars[j] = _bin_to_char[data[i:i+8]]\r
+ i += 8\r
+ j += 1\r
+ return "".join(chars)\r
+\r
+\r
+\r
+\r
--- /dev/null
+from binary import encode_bin, decode_bin\r
+\r
+\r
+class BitStreamReader(object):\r
+ __slots__ = ["substream", "buffer", "total_size"]\r
+ def __init__(self, substream):\r
+ self.substream = substream\r
+ self.total_size = 0\r
+ self.buffer = ""\r
+ def close(self):\r
+ if self.total_size % 8 != 0:\r
+ raise ValueError("total size of read data must be a multiple of 8",\r
+ self.total_size)\r
+ def tell(self):\r
+ return self.substream.tell()\r
+ def seek(self, pos, whence = 0):\r
+ self.buffer = ""\r
+ self.total_size = 0\r
+ self.substream.seek(pos, whence)\r
+ def read(self, count):\r
+ assert count >= 0\r
+ l = len(self.buffer)\r
+ if count == 0:\r
+ data = ""\r
+ elif count <= l:\r
+ data = self.buffer[:count]\r
+ self.buffer = self.buffer[count:]\r
+ else:\r
+ data = self.buffer\r
+ count -= l\r
+ bytes = count // 8\r
+ if count & 7: \r
+ bytes += 1\r
+ buf = encode_bin(self.substream.read(bytes))\r
+ data += buf[:count]\r
+ self.buffer = buf[count:]\r
+ self.total_size += len(data)\r
+ return data\r
+\r
+\r
+class BitStreamWriter(object):\r
+ __slots__ = ["substream", "buffer", "pos"]\r
+ def __init__(self, substream):\r
+ self.substream = substream\r
+ self.buffer = []\r
+ self.pos = 0\r
+ def close(self):\r
+ self.flush()\r
+ def flush(self):\r
+ bytes = decode_bin("".join(self.buffer))\r
+ self.substream.write(bytes)\r
+ self.buffer = []\r
+ self.pos = 0\r
+ def tell(self):\r
+ return self.substream.tell() + self.pos // 8\r
+ def seek(self, pos, whence = 0):\r
+ self.flush()\r
+ self.substream.seek(pos, whence)\r
+ def write(self, data):\r
+ if not data:\r
+ return\r
+ if type(data) is not str:\r
+ raise TypeError("data must be a string, not %r" % (type(data),))\r
+ self.buffer.append(data)\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
--- /dev/null
+def recursion_lock(retval, lock_name = "__recursion_lock__"):\r
+ def decorator(func):\r
+ def wrapper(self, *args, **kw):\r
+ if getattr(self, lock_name, False):\r
+ return retval\r
+ setattr(self, lock_name, True)\r
+ try:\r
+ return func(self, *args, **kw)\r
+ finally:\r
+ setattr(self, lock_name, False)\r
+ wrapper.__name__ = func.__name__\r
+ return wrapper\r
+ return decorator\r
+\r
+class Container(object):\r
+ """\r
+ A generic container of attributes\r
+ """\r
+ __slots__ = ["__dict__", "__attrs__"]\r
+ def __init__(self, **kw):\r
+ self.__dict__.update(kw)\r
+ object.__setattr__(self, "__attrs__", kw.keys())\r
+ \r
+ def __eq__(self, other):\r
+ try:\r
+ return self.__dict__ == other.__dict__\r
+ except AttributeError:\r
+ return False\r
+ def __ne__(self, other):\r
+ return not (self == other)\r
+ \r
+ def __delattr__(self, name):\r
+ object.__delattr__(self, name)\r
+ self.__attrs__.remove(name)\r
+ def __setattr__(self, name, value):\r
+ d = self.__dict__\r
+ if name not in d:\r
+ self.__attrs__.append(name)\r
+ d[name] = value\r
+ def __getitem__(self, name):\r
+ return self.__dict__[name]\r
+ def __delitem__(self, name):\r
+ self.__delattr__(name)\r
+ def __setitem__(self, name, value):\r
+ self.__setattr__(name, value)\r
+ def __update__(self, obj):\r
+ for name in obj.__attrs__:\r
+ self[name] = obj[name]\r
+ def __copy__(self):\r
+ new = self.__class__()\r
+ new.__attrs__ = self.__attrs__[:]\r
+ new.__dict__ = self.__dict__.copy()\r
+ return new\r
+ \r
+ @recursion_lock("<...>")\r
+ def __repr__(self):\r
+ attrs = sorted("%s = %r" % (k, v) \r
+ for k, v in self.__dict__.iteritems() \r
+ if not k.startswith("_"))\r
+ return "%s(%s)" % (self.__class__.__name__, ", ".join(attrs))\r
+ def __str__(self):\r
+ return self.__pretty_str__()\r
+ @recursion_lock("<...>")\r
+ def __pretty_str__(self, nesting = 1, indentation = " "):\r
+ attrs = []\r
+ ind = indentation * nesting\r
+ for k in self.__attrs__:\r
+ v = self.__dict__[k]\r
+ if not k.startswith("_"):\r
+ text = [ind, k, " = "]\r
+ if hasattr(v, "__pretty_str__"):\r
+ text.append(v.__pretty_str__(nesting + 1, indentation))\r
+ else:\r
+ text.append(repr(v))\r
+ attrs.append("".join(text))\r
+ if not attrs:\r
+ return "%s()" % (self.__class__.__name__,)\r
+ attrs.insert(0, self.__class__.__name__ + ":")\r
+ return "\n".join(attrs)\r
+\r
+class FlagsContainer(Container):\r
+ """\r
+ A container providing pretty-printing for flags. Only set flags are \r
+ displayed. \r
+ """\r
+ def __pretty_str__(self, nesting = 1, indentation = " "):\r
+ attrs = []\r
+ ind = indentation * nesting\r
+ for k in self.__attrs__:\r
+ v = self.__dict__[k]\r
+ if not k.startswith("_") and v:\r
+ attrs.append(ind + k)\r
+ if not attrs:\r
+ return "%s()" % (self.__class__.__name__,)\r
+ attrs.insert(0, self.__class__.__name__+ ":")\r
+ return "\n".join(attrs)\r
+\r
+class ListContainer(list):\r
+ """\r
+ A container for lists\r
+ """\r
+ __slots__ = ["__recursion_lock__"]\r
+ def __str__(self):\r
+ return self.__pretty_str__()\r
+ @recursion_lock("[...]")\r
+ def __pretty_str__(self, nesting = 1, indentation = " "):\r
+ if not self:\r
+ return "[]"\r
+ ind = indentation * nesting\r
+ lines = ["["]\r
+ for elem in self:\r
+ lines.append("\n")\r
+ lines.append(ind)\r
+ if hasattr(elem, "__pretty_str__"):\r
+ lines.append(elem.__pretty_str__(nesting + 1, indentation))\r
+ else:\r
+ lines.append(repr(elem))\r
+ lines.append("\n")\r
+ lines.append(indentation * (nesting - 1))\r
+ lines.append("]")\r
+ return "".join(lines)\r
+\r
+class AttrDict(object):\r
+ """\r
+ A dictionary that can be accessed both using indexing and attributes,\r
+ i.e., \r
+ x = AttrDict()\r
+ x.foo = 5\r
+ print x["foo"]\r
+ """\r
+ __slots__ = ["__dict__"]\r
+ def __init__(self, **kw):\r
+ self.__dict__ = kw\r
+ def __contains__(self, key):\r
+ return key in self.__dict__\r
+ def __nonzero__(self):\r
+ return bool(self.__dict__)\r
+ def __repr__(self):\r
+ return repr(self.__dict__)\r
+ def __str__(self):\r
+ return self.__pretty_str__()\r
+ def __pretty_str__(self, nesting = 1, indentation = " "):\r
+ if not self:\r
+ return "{}"\r
+ text = ["{\n"]\r
+ ind = nesting * indentation\r
+ for k in sorted(self.__dict__.keys()):\r
+ v = self.__dict__[k]\r
+ text.append(ind)\r
+ text.append(repr(k))\r
+ text.append(" : ")\r
+ if hasattr(v, "__pretty_str__"):\r
+ try:\r
+ text.append(v.__pretty_str__(nesting+1, indentation))\r
+ except Exception:\r
+ text.append(repr(v))\r
+ else:\r
+ text.append(repr(v))\r
+ text.append("\n")\r
+ text.append((nesting-1) * indentation)\r
+ text.append("}")\r
+ return "".join(text)\r
+ def __delitem__(self, key):\r
+ del self.__dict__[key]\r
+ def __getitem__(self, key):\r
+ return self.__dict__[key]\r
+ def __setitem__(self, key, value):\r
+ self.__dict__[key] = value\r
+ def __copy__(self):\r
+ new = self.__class__()\r
+ new.__dict__ = self.__dict__.copy()\r
+ return new\r
+ def __update__(self, other):\r
+ if isinstance(other, dict):\r
+ self.__dict__.update(other)\r
+ else:\r
+ self.__dict__.update(other.__dict__)\r
+\r
+class LazyContainer(object):\r
+ __slots__ = ["subcon", "stream", "pos", "context", "_value"]\r
+ def __init__(self, subcon, stream, pos, context):\r
+ self.subcon = subcon\r
+ self.stream = stream\r
+ self.pos = pos\r
+ self.context = context\r
+ self._value = NotImplemented\r
+ def __eq__(self, other):\r
+ try:\r
+ return self._value == other._value\r
+ except AttributeError:\r
+ return False\r
+ def __ne__(self, other):\r
+ return not (self == other)\r
+ def __str__(self):\r
+ return self.__pretty_str__()\r
+ def __pretty_str__(self, nesting = 1, indentation = " "):\r
+ if self._value is NotImplemented:\r
+ text = "<unread>"\r
+ elif hasattr(self._value, "__pretty_str__"):\r
+ text = self._value.__pretty_str__(nesting, indentation)\r
+ else:\r
+ text = repr(self._value)\r
+ return "%s: %s" % (self.__class__.__name__, text)\r
+ def read(self):\r
+ self.stream.seek(self.pos)\r
+ return self.subcon._parse(self.stream, self.context)\r
+ def dispose(self):\r
+ self.subcon = None\r
+ self.stream = None\r
+ self.context = None\r
+ self.pos = None\r
+ def _get_value(self):\r
+ if self._value is NotImplemented:\r
+ self._value = self.read()\r
+ return self._value\r
+ value = property(_get_value)\r
+ has_value = property(lambda self: self._value is not NotImplemented)\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
--- /dev/null
+_printable = dict((chr(i), ".") for i in range(256))\r
+_printable.update((chr(i), chr(i)) for i in range(32, 128))\r
+\r
+def hexdump(data, linesize = 16):\r
+ prettylines = []\r
+ if len(data) < 65536:\r
+ fmt = "%%04X %%-%ds %%s"\r
+ else:\r
+ fmt = "%%08X %%-%ds %%s"\r
+ fmt = fmt % (3 * linesize - 1,)\r
+ for i in xrange(0, len(data), linesize):\r
+ line = data[i : i + linesize]\r
+ hextext = " ".join(b.encode("hex") for b in line)\r
+ rawtext = "".join(_printable[b] for b in line)\r
+ prettylines.append(fmt % (i, hextext, rawtext))\r
+ return prettylines\r
+\r
+class HexString(str):\r
+ """\r
+ represents a string that will be hex-dumped (only via __pretty_str__).\r
+ this class derives of str, and behaves just like a normal string in all\r
+ other contexts.\r
+ """\r
+ def __init__(self, data, linesize = 16):\r
+ str.__init__(self, data)\r
+ self.linesize = linesize\r
+ def __new__(cls, data, *args, **kwargs):\r
+ return str.__new__(cls, data)\r
+ def __pretty_str__(self, nesting = 1, indentation = " "):\r
+ sep = "\n" + indentation * nesting\r
+ return sep + sep.join(hexdump(self))\r
+\r
+\r
+\r
--- /dev/null
+try:\r
+ from cStringIO import StringIO\r
+except ImportError:\r
+ from StringIO import StringIO\r
+\r
+\r
+try:\r
+ from struct import Struct as Packer\r
+except ImportError:\r
+ from struct import pack, unpack, calcsize\r
+ class Packer(object):\r
+ __slots__ = ["format", "size"]\r
+ def __init__(self, format):\r
+ self.format = format\r
+ self.size = calcsize(format)\r
+ def pack(self, *args):\r
+ return pack(self.format, *args)\r
+ def unpack(self, data):\r
+ return unpack(self.format, data)\r
+\r
+\r
+\r
--- /dev/null
+from lib import BitStreamReader, BitStreamWriter, encode_bin, decode_bin\r
+from core import *\r
+from adapters import *\r
+\r
+\r
+#===============================================================================\r
+# fields\r
+#===============================================================================\r
+def Field(name, length):\r
+ """a field\r
+ * name - the name of the field\r
+ * length - the length of the field. the length can be either an integer\r
+ (StaticField), or a function that takes the context as an argument and \r
+ returns the length (MetaField)\r
+ """\r
+ if callable(length):\r
+ return MetaField(name, length)\r
+ else:\r
+ return StaticField(name, length)\r
+\r
+def BitField(name, length, swapped = False, signed = False, bytesize = 8):\r
+ """a bit field; must be enclosed in a BitStruct\r
+ * name - the name of the field\r
+ * length - the length of the field in bits. the length can be either \r
+ an integer, or a function that takes the context as an argument and \r
+ returns the length\r
+ * swapped - whether the value is byte-swapped (little endian). the \r
+ default is False.\r
+ * signed - whether the value of the bitfield is a signed integer. the \r
+ default is False.\r
+ * bytesize - the number of bits in a byte (used for byte-swapping). the\r
+ default is 8.\r
+ """\r
+ return BitIntegerAdapter(Field(name, length), \r
+ length,\r
+ swapped = swapped, \r
+ signed = signed,\r
+ bytesize = bytesize\r
+ )\r
+\r
+def Padding(length, pattern = "\x00", strict = False):\r
+ r"""a padding field (value is discarded)\r
+ * length - the length of the field. the length can be either an integer,\r
+ or a function that takes the context as an argument and returns the \r
+ length\r
+ * pattern - the padding pattern (character) to use. default is "\x00"\r
+ * strict - whether or not to raise an exception is the actual padding \r
+ pattern mismatches the desired pattern. default is False.\r
+ """\r
+ return PaddingAdapter(Field(None, length), \r
+ pattern = pattern, \r
+ strict = strict,\r
+ )\r
+\r
+def Flag(name, truth = 1, falsehood = 0, default = False):\r
+ """a flag field (True or False)\r
+ * name - the name of the field\r
+ * truth - the numeric value of truth. the default is 1.\r
+ * falsehood - the numeric value of falsehood. the default is 0.\r
+ * default - the default value to assume, when the value is neither \r
+ `truth` nor `falsehood`. the default is False.\r
+ """\r
+ return SymmetricMapping(Field(name, 1), \r
+ {True : chr(truth), False : chr(falsehood)},\r
+ default = default,\r
+ )\r
+\r
+#===============================================================================\r
+# field shortcuts\r
+#===============================================================================\r
+def Bit(name):\r
+ """a 1-bit BitField; must be enclosed in a BitStruct"""\r
+ return BitField(name, 1)\r
+def Nibble(name):\r
+ """a 4-bit BitField; must be enclosed in a BitStruct"""\r
+ return BitField(name, 4)\r
+def Octet(name):\r
+ """an 8-bit BitField; must be enclosed in a BitStruct"""\r
+ return BitField(name, 8)\r
+\r
+def UBInt8(name):\r
+ """unsigned, big endian 8-bit integer"""\r
+ return FormatField(name, ">", "B")\r
+def UBInt16(name):\r
+ """unsigned, big endian 16-bit integer"""\r
+ return FormatField(name, ">", "H")\r
+def UBInt32(name):\r
+ """unsigned, big endian 32-bit integer"""\r
+ return FormatField(name, ">", "L")\r
+def UBInt64(name):\r
+ """unsigned, big endian 64-bit integer"""\r
+ return FormatField(name, ">", "Q")\r
+\r
+def SBInt8(name):\r
+ """signed, big endian 8-bit integer"""\r
+ return FormatField(name, ">", "b")\r
+def SBInt16(name):\r
+ """signed, big endian 16-bit integer"""\r
+ return FormatField(name, ">", "h")\r
+def SBInt32(name):\r
+ """signed, big endian 32-bit integer"""\r
+ return FormatField(name, ">", "l")\r
+def SBInt64(name):\r
+ """signed, big endian 64-bit integer"""\r
+ return FormatField(name, ">", "q")\r
+\r
+def ULInt8(name):\r
+ """unsigned, little endian 8-bit integer"""\r
+ return FormatField(name, "<", "B")\r
+def ULInt16(name):\r
+ """unsigned, little endian 16-bit integer"""\r
+ return FormatField(name, "<", "H")\r
+def ULInt32(name):\r
+ """unsigned, little endian 32-bit integer"""\r
+ return FormatField(name, "<", "L")\r
+def ULInt64(name):\r
+ """unsigned, little endian 64-bit integer"""\r
+ return FormatField(name, "<", "Q")\r
+\r
+def SLInt8(name):\r
+ """signed, little endian 8-bit integer"""\r
+ return FormatField(name, "<", "b")\r
+def SLInt16(name):\r
+ """signed, little endian 16-bit integer"""\r
+ return FormatField(name, "<", "h")\r
+def SLInt32(name):\r
+ """signed, little endian 32-bit integer"""\r
+ return FormatField(name, "<", "l")\r
+def SLInt64(name):\r
+ """signed, little endian 64-bit integer"""\r
+ return FormatField(name, "<", "q")\r
+\r
+def UNInt8(name):\r
+ """unsigned, native endianity 8-bit integer"""\r
+ return FormatField(name, "=", "B")\r
+def UNInt16(name):\r
+ """unsigned, native endianity 16-bit integer"""\r
+ return FormatField(name, "=", "H")\r
+def UNInt32(name):\r
+ """unsigned, native endianity 32-bit integer"""\r
+ return FormatField(name, "=", "L")\r
+def UNInt64(name):\r
+ """unsigned, native endianity 64-bit integer"""\r
+ return FormatField(name, "=", "Q")\r
+\r
+def SNInt8(name):\r
+ """signed, native endianity 8-bit integer"""\r
+ return FormatField(name, "=", "b")\r
+def SNInt16(name):\r
+ """signed, native endianity 16-bit integer"""\r
+ return FormatField(name, "=", "h")\r
+def SNInt32(name):\r
+ """signed, native endianity 32-bit integer"""\r
+ return FormatField(name, "=", "l")\r
+def SNInt64(name):\r
+ """signed, native endianity 64-bit integer"""\r
+ return FormatField(name, "=", "q")\r
+\r
+def BFloat32(name):\r
+ """big endian, 32-bit IEEE floating point number"""\r
+ return FormatField(name, ">", "f")\r
+def LFloat32(name):\r
+ """little endian, 32-bit IEEE floating point number"""\r
+ return FormatField(name, "<", "f")\r
+def NFloat32(name):\r
+ """native endianity, 32-bit IEEE floating point number"""\r
+ return FormatField(name, "=", "f")\r
+\r
+def BFloat64(name):\r
+ """big endian, 64-bit IEEE floating point number"""\r
+ return FormatField(name, ">", "d")\r
+def LFloat64(name):\r
+ """little endian, 64-bit IEEE floating point number"""\r
+ return FormatField(name, "<", "d")\r
+def NFloat64(name):\r
+ """native endianity, 64-bit IEEE floating point number"""\r
+ return FormatField(name, "=", "d")\r
+\r
+\r
+#===============================================================================\r
+# arrays\r
+#===============================================================================\r
+def Array(count, subcon):\r
+ """array of subcon repeated count times.\r
+ * subcon - the subcon.\r
+ * count - an integer, or a function taking the context as an argument, \r
+ returning the count\r
+ """\r
+ if callable(count):\r
+ con = MetaArray(count, subcon)\r
+ else:\r
+ con = MetaArray(lambda ctx: count, subcon)\r
+ con._clear_flag(con.FLAG_DYNAMIC)\r
+ return con\r
+\r
+def PrefixedArray(subcon, length_field = UBInt8("length")):\r
+ """an array prefixed by a length field.\r
+ * subcon - the subcon to be repeated\r
+ * length_field - an integer construct\r
+ """\r
+ return LengthValueAdapter(\r
+ Sequence(subcon.name, \r
+ length_field, \r
+ Array(lambda ctx: ctx[length_field.name], subcon),\r
+ nested = False\r
+ )\r
+ )\r
+\r
+def OpenRange(mincount, subcon):\r
+ from sys import maxint\r
+ return Range(mincount, maxint, subcon)\r
+\r
+def GreedyRange(subcon):\r
+ """an open range (1 or more times) of repeated subcon.\r
+ * subcon - the subcon to repeat"""\r
+ return OpenRange(1, subcon)\r
+\r
+def OptionalGreedyRange(subcon):\r
+ """an open range (0 or more times) of repeated subcon.\r
+ * subcon - the subcon to repeat"""\r
+ return OpenRange(0, subcon)\r
+\r
+\r
+#===============================================================================\r
+# subconstructs\r
+#===============================================================================\r
+def Optional(subcon):\r
+ """an optional construct. if parsing fails, returns None.\r
+ * subcon - the subcon to optionally parse or build\r
+ """\r
+ return Select(subcon.name, subcon, Pass)\r
+\r
+def Bitwise(subcon):\r
+ """converts the stream to bits, and passes the bitstream to subcon\r
+ * subcon - a bitwise construct (usually BitField)\r
+ """\r
+ # subcons larger than MAX_BUFFER will be wrapped by Restream instead \r
+ # of Buffered. implementation details, don't stick your nose :)\r
+ MAX_BUFFER = 1024 * 8\r
+ def resizer(length):\r
+ if length & 7:\r
+ raise SizeofError("size must be a multiple of 8", length)\r
+ return length >> 3\r
+ if not subcon._is_flag(subcon.FLAG_DYNAMIC) and subcon.sizeof() < MAX_BUFFER:\r
+ con = Buffered(subcon, \r
+ encoder = decode_bin, \r
+ decoder = encode_bin, \r
+ resizer = resizer\r
+ )\r
+ else:\r
+ con = Restream(subcon, \r
+ stream_reader = BitStreamReader, \r
+ stream_writer = BitStreamWriter, \r
+ resizer = resizer)\r
+ return con\r
+\r
+def Aligned(subcon, modulus = 4, pattern = "\x00"):\r
+ r"""aligns subcon to modulus boundary using padding pattern\r
+ * subcon - the subcon to align\r
+ * modulus - the modulus boundary (default is 4)\r
+ * pattern - the padding pattern (default is \x00)\r
+ """\r
+ if modulus < 2:\r
+ raise ValueError("modulus must be >= 2", modulus)\r
+ if modulus in (2, 4, 8, 16, 32, 64, 128, 256, 512, 1024):\r
+ def padlength(ctx):\r
+ m1 = modulus - 1\r
+ return (modulus - (subcon._sizeof(ctx) & m1)) & m1\r
+ else:\r
+ def padlength(ctx):\r
+ return (modulus - (subcon._sizeof(ctx) % modulus)) % modulus\r
+ return IndexingAdapter(\r
+ Sequence(subcon.name, \r
+ subcon, \r
+ Padding(padlength, pattern = pattern),\r
+ nested = False,\r
+ ),\r
+ 0\r
+ )\r
+\r
+def Embedded(subcon):\r
+ """embeds a struct into the enclosing struct.\r
+ * subcon - the struct to embed\r
+ """\r
+ return Reconfig(subcon.name, subcon, subcon.FLAG_EMBED)\r
+\r
+def Rename(newname, subcon):\r
+ """renames an existing construct\r
+ * newname - the new name\r
+ * subcon - the subcon to rename\r
+ """\r
+ return Reconfig(newname, subcon)\r
+\r
+def Alias(newname, oldname):\r
+ """creates an alias for an existing element in a struct\r
+ * newname - the new name\r
+ * oldname - the name of an existing element\r
+ """\r
+ return Value(newname, lambda ctx: ctx[oldname])\r
+\r
+\r
+#===============================================================================\r
+# mapping\r
+#===============================================================================\r
+def SymmetricMapping(subcon, mapping, default = NotImplemented):\r
+ """defines a symmetrical mapping: a->b, b->a.\r
+ * subcon - the subcon to map\r
+ * mapping - the encoding mapping (a dict); the decoding mapping is \r
+ achieved by reversing this mapping\r
+ * default - the default value to use when no mapping is found. if no \r
+ default value is given, and exception is raised. setting to Pass would\r
+ return the value "as is" (unmapped)\r
+ """\r
+ reversed_mapping = dict((v, k) for k, v in mapping.iteritems())\r
+ return MappingAdapter(subcon, \r
+ encoding = mapping, \r
+ decoding = reversed_mapping, \r
+ encdefault = default,\r
+ decdefault = default, \r
+ )\r
+\r
+def Enum(subcon, **kw):\r
+ """a set of named values mapping. \r
+ * subcon - the subcon to map\r
+ * kw - keyword arguments which serve as the encoding mapping\r
+ * _default_ - an optional, keyword-only argument that specifies the \r
+ default value to use when the mapping is undefined. if not given,\r
+ and exception is raised when the mapping is undefined. use `Pass` to\r
+ pass the unmapped value as-is\r
+ """\r
+ return SymmetricMapping(subcon, kw, kw.pop("_default_", NotImplemented))\r
+\r
+def FlagsEnum(subcon, **kw):\r
+ """a set of flag values mapping.\r
+ * subcon - the subcon to map\r
+ * kw - keyword arguments which serve as the encoding mapping\r
+ """\r
+ return FlagsAdapter(subcon, kw)\r
+\r
+\r
+#===============================================================================\r
+# structs\r
+#===============================================================================\r
+def AlignedStruct(name, *subcons, **kw):\r
+ """a struct of aligned fields\r
+ * name - the name of the struct\r
+ * subcons - the subcons that make up this structure\r
+ * kw - keyword arguments to pass to Aligned: 'modulus' and 'pattern'\r
+ """\r
+ return Struct(name, *(Aligned(sc, **kw) for sc in subcons))\r
+\r
+def BitStruct(name, *subcons):\r
+ """a struct of bitwise fields\r
+ * name - the name of the struct\r
+ * subcons - the subcons that make up this structure\r
+ """\r
+ return Bitwise(Struct(name, *subcons))\r
+\r
+def EmbeddedBitStruct(*subcons):\r
+ """an embedded BitStruct. no name is necessary.\r
+ * subcons - the subcons that make up this structure\r
+ """\r
+ return Bitwise(Embedded(Struct(None, *subcons)))\r
+\r
+#===============================================================================\r
+# strings\r
+#===============================================================================\r
+def String(name, length, encoding = None, padchar = None, \r
+ paddir = "right", trimdir = "right"):\r
+ """a fixed-length, optionally padded string of characters\r
+ * name - the name of the field\r
+ * length - the length (integer)\r
+ * encoding - the encoding to use (e.g., "utf8"), or None, for raw bytes.\r
+ default is None\r
+ * padchar - the padding character (commonly "\x00"), or None to \r
+ disable padding. default is None\r
+ * paddir - the direction where padding is placed ("right", "left", or \r
+ "center"). the default is "right". this argument is meaningless if \r
+ padchar is None.\r
+ * trimdir - the direction where trimming will take place ("right" or \r
+ "left"). the default is "right". trimming is only meaningful for\r
+ building, when the given string is too long. this argument is \r
+ meaningless if padchar is None.\r
+ """\r
+ con = StringAdapter(Field(name, length), encoding = encoding)\r
+ if padchar is not None:\r
+ con = PaddedStringAdapter(con, \r
+ padchar = padchar, \r
+ paddir = paddir, \r
+ trimdir = trimdir\r
+ )\r
+ return con\r
+\r
+def PascalString(name, length_field = UBInt8("length"), encoding = None):\r
+ """a string prefixed with a length field. the data must directly follow \r
+ the length field.\r
+ * name - the name of the \r
+ * length_field - a numeric construct (i.e., UBInt8) that holds the \r
+ length. default is an unsigned, 8-bit integer field. note that this\r
+ argument must pass an instance of a construct, not a class \r
+ (`UBInt8("length")` rather than `UBInt8`)\r
+ * encoding - the encoding to use (e.g., "utf8"), or None, for raw bytes.\r
+ default is None\r
+ """\r
+ return StringAdapter(\r
+ LengthValueAdapter(\r
+ Sequence(name,\r
+ length_field,\r
+ Field("data", lambda ctx: ctx[length_field.name]),\r
+ )\r
+ ),\r
+ encoding = encoding,\r
+ )\r
+\r
+def CString(name, terminators = "\x00", encoding = None, \r
+ char_field = Field(None, 1)):\r
+ r"""a c-style string (string terminated by a terminator char)\r
+ * name - the name fo the string\r
+ * terminators - a sequence of terminator chars. default is "\x00".\r
+ * encoding - the encoding to use (e.g., "utf8"), or None, for raw bytes.\r
+ default is None\r
+ * char_field - the construct that represents a single character. default\r
+ is a one-byte character. note that this argument must be an instance\r
+ of a construct, not a construct class (`Field("char", 1)` rather than\r
+ `Field`)\r
+ """\r
+ return Rename(name,\r
+ CStringAdapter(\r
+ RepeatUntil(lambda obj, ctx: obj in terminators, \r
+ char_field,\r
+ ),\r
+ terminators = terminators,\r
+ encoding = encoding,\r
+ )\r
+ )\r
+\r
+\r
+#===============================================================================\r
+# conditional\r
+#===============================================================================\r
+def IfThenElse(name, predicate, then_subcon, else_subcon):\r
+ """an if-then-else conditional construct: if the predicate indicates True,\r
+ `then_subcon` will be used; otherwise `else_subcon`\r
+ * name - the name of the construct\r
+ * predicate - a function taking the context as an argument and returning\r
+ True or False\r
+ * then_subcon - the subcon that will be used if the predicate returns True\r
+ * else_subcon - the subcon that will be used if the predicate returns False\r
+ """\r
+ return Switch(name, lambda ctx: bool(predicate(ctx)),\r
+ {\r
+ True : then_subcon,\r
+ False : else_subcon,\r
+ }\r
+ )\r
+\r
+def If(predicate, subcon, elsevalue = None):\r
+ """an if-then conditional construct: if the predicate indicates True,\r
+ subcon will be used; otherwise, `elsevalue` will be returned instead.\r
+ * predicate - a function taking the context as an argument and returning\r
+ True or False\r
+ * subcon - the subcon that will be used if the predicate returns True\r
+ * elsevalue - the value that will be used should the predicate return False.\r
+ by default this value is None.\r
+ """\r
+ return IfThenElse(subcon.name, \r
+ predicate, \r
+ subcon, \r
+ Value("elsevalue", lambda ctx: elsevalue)\r
+ )\r
+\r
+\r
+#===============================================================================\r
+# misc\r
+#===============================================================================\r
+def OnDemandPointer(offsetfunc, subcon, force_build = True):\r
+ """an on-demand pointer. \r
+ * offsetfunc - a function taking the context as an argument and returning \r
+ the absolute stream position\r
+ * subcon - the subcon that will be parsed from the `offsetfunc()` stream \r
+ position on demand\r
+ * force_build - see OnDemand. by default True.\r
+ """\r
+ return OnDemand(Pointer(offsetfunc, subcon), \r
+ advance_stream = False, \r
+ force_build = force_build\r
+ )\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
--- /dev/null
+from core import *\r
+from adapters import *\r
+from macros import *\r
+\r
+\r
+#===============================================================================\r
+# exceptions\r
+#===============================================================================\r
+class QuotedStringError(ConstructError):\r
+ __slots__ = []\r
+\r
+\r
+#===============================================================================\r
+# constructs\r
+#===============================================================================\r
+class QuotedString(Construct):\r
+ r"""\r
+ A quoted string (begins with an opening-quote, terminated by a \r
+ closing-quote, which may be escaped by an escape character)\r
+ \r
+ Parameters:\r
+ * name - the name of the field\r
+ * start_quote - the opening quote character. default is '"'\r
+ * end_quote - the closing quote character. default is '"'\r
+ * esc_char - the escape character, or None to disable escaping. defualt\r
+ is "\" (backslash)\r
+ * encoding - the character encoding (e.g., "utf8"), or None to return\r
+ raw bytes. defualt is None.\r
+ * allow_eof - whether to allow EOF before the closing quote is matched.\r
+ if False, an exception will be raised when EOF is reached by the closing\r
+ quote is missing. default is False.\r
+ \r
+ Example:\r
+ QuotedString("foo", start_quote = "{", end_quote = "}", esc_char = None)\r
+ """\r
+ __slots__ = [\r
+ "start_quote", "end_quote", "char", "esc_char", "encoding", \r
+ "allow_eof"\r
+ ]\r
+ def __init__(self, name, start_quote = '"', end_quote = None, \r
+ esc_char = '\\', encoding = None, allow_eof = False):\r
+ Construct.__init__(self, name)\r
+ if end_quote is None:\r
+ end_quote = start_quote\r
+ self.start_quote = Literal(start_quote)\r
+ self.char = Char("char")\r
+ self.end_quote = end_quote\r
+ self.esc_char = esc_char\r
+ self.encoding = encoding\r
+ self.allow_eof = allow_eof\r
+ \r
+ def _parse(self, stream, context):\r
+ self.start_quote._parse(stream, context)\r
+ text = []\r
+ escaped = False\r
+ try:\r
+ while True:\r
+ ch = self.char._parse(stream, context)\r
+ if ch == self.esc_char:\r
+ if escaped:\r
+ text.append(ch)\r
+ escaped = False\r
+ else:\r
+ escaped = True\r
+ elif ch == self.end_quote and not escaped:\r
+ break\r
+ else:\r
+ text.append(ch)\r
+ escaped = False\r
+ except FieldError:\r
+ if not self.allow_eof:\r
+ raise\r
+ text = "".join(text)\r
+ if self.encoding is not None:\r
+ text = text.decode(self.encoding)\r
+ return text\r
+ \r
+ def _build(self, obj, stream, context):\r
+ self.start_quote._build(None, stream, context)\r
+ if self.encoding:\r
+ obj = obj.encode(self.encoding)\r
+ for ch in obj:\r
+ if ch == self.esc_char:\r
+ self.char._build(self.esc_char, stream, context)\r
+ elif ch == self.end_quote:\r
+ if self.esc_char is None:\r
+ raise QuotedStringError("found ending quote in data, "\r
+ "but no escape char defined", ch)\r
+ else:\r
+ self.char._build(self.esc_char, stream, context)\r
+ self.char._build(ch, stream, context)\r
+ self.char._build(self.end_quote, stream, context)\r
+ \r
+ def _sizeof(self, context):\r
+ raise SizeofError("can't calculate size")\r
+\r
+\r
+#===============================================================================\r
+# macros\r
+#===============================================================================\r
+class WhitespaceAdapter(Adapter):\r
+ """\r
+ Adapter for whitespace sequences; do not use directly.\r
+ See Whitespace.\r
+ \r
+ Parameters:\r
+ * subcon - the subcon to adapt\r
+ * build_char - the character used for encoding (building)\r
+ """\r
+ __slots__ = ["build_char"]\r
+ def __init__(self, subcon, build_char):\r
+ Adapter.__init__(self, subcon)\r
+ self.build_char = build_char\r
+ def _encode(self, obj, context):\r
+ return self.build_char\r
+ def _decode(self, obj, context):\r
+ return None\r
+\r
+def Whitespace(charset = " \t", optional = True):\r
+ """whitespace (space that is ignored between tokens). when building, the\r
+ first character of the charset is used.\r
+ * charset - the set of characters that are considered whitespace. default\r
+ is space and tab.\r
+ * optional - whether or not whitespace is optional. default is True.\r
+ """\r
+ con = CharOf(None, charset)\r
+ if optional:\r
+ con = OptionalGreedyRange(con)\r
+ else:\r
+ con = GreedyRange(con)\r
+ return WhitespaceAdapter(con, build_char = charset[0])\r
+\r
+def Literal(text):\r
+ """matches a literal string in the text\r
+ * text - the text (string) to match\r
+ """\r
+ return ConstAdapter(Field(None, len(text)), text)\r
+\r
+def Char(name):\r
+ """a one-byte character"""\r
+ return Field(name, 1)\r
+\r
+def CharOf(name, charset):\r
+ """matches only characters of a given charset\r
+ * name - the name of the field\r
+ * charset - the set of valid characters\r
+ """\r
+ return OneOf(Char(name), charset)\r
+\r
+def CharNoneOf(name, charset):\r
+ """matches only characters that do not belong to a given charset\r
+ * name - the name of the field\r
+ * charset - the set of invalid characters\r
+ """\r
+ return NoneOf(Char(name), charset)\r
+\r
+def Alpha(name):\r
+ """a letter character (A-Z, a-z)"""\r
+ return CharOf(name, set('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'))\r
+\r
+def Digit(name):\r
+ """a digit character (0-9)"""\r
+ return CharOf(name, set('0123456789'))\r
+\r
+def AlphaDigit(name):\r
+ """an alphanumeric character (A-Z, a-z, 0-9)"""\r
+ return CharOf(name, set("0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"))\r
+\r
+def BinDigit(name):\r
+ """a binary digit (0-1)"""\r
+ return CharOf(name, set('01'))\r
+\r
+def HexDigit(name):\r
+ """a hexadecimal digit (0-9, A-F, a-f)"""\r
+ return CharOf(name, set('0123456789abcdefABCDEF'))\r
+\r
+def Word(name):\r
+ """a sequence of letters"""\r
+ return StringAdapter(GreedyRange(Alpha(name)))\r
+\r
+class TextualIntAdapter(Adapter):\r
+ """\r
+ Adapter for textual integers\r
+ \r
+ Parameters:\r
+ * subcon - the subcon to adapt\r
+ * radix - the base of the integer (decimal, hexadecimal, binary, ...)\r
+ * digits - the sequence of digits of that radix\r
+ """\r
+ __slots__ = ["radix", "digits"]\r
+ def __init__(self, subcon, radix = 10, digits = "0123456789abcdef"):\r
+ Adapter.__init__(self, subcon)\r
+ if radix > len(digits):\r
+ raise ValueError("not enough digits for radix %d" % (radix,))\r
+ self.radix = radix\r
+ self.digits = digits\r
+ def _encode(self, obj, context):\r
+ chars = []\r
+ if obj < 0:\r
+ chars.append("-")\r
+ n = -obj\r
+ else:\r
+ n = obj\r
+ r = self.radix\r
+ digs = self.digits\r
+ while n > 0:\r
+ n, d = divmod(n, r)\r
+ chars.append(digs[d])\r
+ # obj2 = "".join(reversed(chars))\r
+ # filler = digs[0] * (self._sizeof(context) - len(obj2))\r
+ # return filler + obj2\r
+ return "".join(reversed(chars))\r
+ def _decode(self, obj, context):\r
+ return int("".join(obj), self.radix)\r
+\r
+def DecNumber(name):\r
+ """decimal number"""\r
+ return TextualIntAdapter(GreedyRange(Digit(name)))\r
+\r
+def BinNumber(name):\r
+ """binary number"""\r
+ return TextualIntAdapter(GreedyRange(Digit(name)), 2)\r
+\r
+def HexNumber(name):\r
+ """hexadecimal number"""\r
+ return TextualIntAdapter(GreedyRange(Digit(name)), 16)\r
+\r
+def StringUpto(name, charset):\r
+ """a string that stretches up to a terminator, or EOF. unlike CString, \r
+ StringUpto will no consume the terminator char.\r
+ * name - the name of the field\r
+ * charset - the set of terminator characters"""\r
+ return StringAdapter(OptionalGreedyRange(CharNoneOf(name, charset)))\r
+\r
+def Line(name):\r
+ r"""a textual line (up to "\n")"""\r
+ return StringUpto(name, "\n")\r
+\r
+class IdentifierAdapter(Adapter):\r
+ """\r
+ Adapter for programmatic identifiers\r
+ \r
+ Parameters:\r
+ * subcon - the subcon to adapt\r
+ """\r
+ def _encode(self, obj, context):\r
+ return obj[0], obj[1:]\r
+ def _decode(self, obj, context):\r
+ return obj[0] + "".join(obj[1])\r
+\r
+def Identifier(name, \r
+ headset = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_"), \r
+ tailset = set("0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_")\r
+ ):\r
+ """a programmatic identifier (symbol). must start with a char of headset,\r
+ followed by a sequence of tailset characters\r
+ * name - the name of the field\r
+ * headset - charset for the first character. default is A-Z, a-z, and _\r
+ * tailset - charset for the tail. default is A-Z, a-z, 0-9 and _\r
+ """\r
+ return IdentifierAdapter(\r
+ Sequence(name,\r
+ CharOf("head", headset),\r
+ OptionalGreedyRange(CharOf("tail", tailset)),\r
+ )\r
+ )\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
+\r
--- /dev/null
+from ..construct import (\r
+ UBInt8, UBInt16, UBInt32, UBInt64,\r
+ ULInt8, ULInt16, ULInt32, ULInt64,\r
+ SBInt32, SLInt32, SBInt64, SLInt64,\r
+ Struct, Array,\r
+ )\r
+\r
+\r
+class ELFStructs(object):
+ def __init__(self, little_endian=True, elfclass=32):\r
+ assert elfclass == 32 or elfclass == 64\r
+ self.little_endian = little_endian\r
+ self.elfclass = elfclass \r
+ self._create_structs()\r
+ \r
+ def _create_structs(self):
+ if self.little_endian:\r
+ self.Elf_byte = ULInt8\r
+ self.Elf_half = ULInt16\r
+ self.Elf_word = ULInt32\r
+ self.Elf_addr = ULInt32 if self.elfclass == 32 else ULInt64\r
+ self.Elf_offset = self.Elf_addr\r
+ self.Elf_sword = SLInt32\r
+ self.Elf_xword = ULInt64\r
+ self.Elf_sxword = SLInt64\r
+ else:\r
+ self.Elf_byte = UBInt8\r
+ self.Elf_half = UBInt16\r
+ self.Elf_word = UBInt32\r
+ self.Elf_addr = UBInt32 if self.elfclass == 32 else UBInt64\r
+ self.Elf_offset = self.Elf_addr\r
+ self.Elf_sword = SBInt32\r
+ self.Elf_xword = UBInt64\r
+ self.Elf_sxword = SBInt64\r
+ \r
+ self.Elf_Ehdr = Struct('Elf_Ehdr',\r
+
\ No newline at end of file
--- /dev/null
+from elftools.elf.structs import ELFStructs\r
+\r
projects / pyelftools.git / commitdiff