binpac: Initial import of Bro's binpac subdirectory from SVN r7088.

tools/binpac/TODO

@@ -0,0 +1,34 @@
+Big features
+* Variable context (xid, call in RPC)? -- no variable context
+* Helpers
+* Connection states and actions
+* Case and analyzer redef
+* &also withinput
+* Explicit analyzer context (interface + instantiation) "withcontext"
++ Interface with C++ and Bro (events, extern, weird)
++ Incremental input
++ ASCII protocols
++ Reassembly
+- Dealing with exceptions
+- Dependency analysis to save parsing time on unused fields
+- Performance measurement
+
+Small features
+* Restructure the code: break up pac.{h,cc}
+* ref counting (to keep certain structures)
+* analyzer context as a parameter of class
+* &autolength
+* find a better name for "analyzer_context" ("analcxt", "context", "analyzer") $context
+* &if
+* &autolength (now &restofdata)
+* Use vector<> instead of array<>?
+* set end_of_data when &length = ...
+- make the `default' case mandatory?
+- &inline
+- &warn and &check? (follow &if)
+- typedef?
+
+Binpac 1
+- create a namespace for each .pac file
+- type equivalence
+- byteorder() for every type?

tools/binpac/lib/Makefile.am

@@ -0,0 +1,8 @@
+## Process this file with automake to produce Makefile.in
+
+noinst_LIBRARIES = libbinpac.a
+
+libbinpac_a_SOURCES = \
+	binpac_buffer.cc binpac_bytestring.cc \
+	binpac.h binpac_analyzer.h binpac_buffer.h \
+	binpac_bytestring.h binpac_exception.h binpac_regex.h

tools/binpac/lib/README

@@ -0,0 +1,3 @@
+This directory contains a library needed by generated C++ code from
+binpac. Note that the library is not needed by the binpac compiler
+itself.

tools/binpac/lib/binpac.h.in

@@ -0,0 +1,142 @@
+// $Id: binpac.h,v 1.1.4.2 2006/06/02 15:13:13 rpang Exp $
+// Do not edit binpac.h, edit binpac.h.in instead!
+
+#ifndef binpac_h
+#define binpac_h
+
+#include <sys/param.h>
+
+@HOST_BIGENDIAN@
+#ifdef HOST_BIGENDIAN
+#  define HOST_BYTEORDER	bigendian
+#else
+#  define HOST_BYTEORDER	littleendian
+#endif
+
+#include <assert.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <string>
+#include <memory>
+
+#define BINPAC_ASSERT(x)	assert(x)
+
+using namespace std;
+
+namespace binpac {
+
+const int bigendian = 0;
+const int littleendian = 1;
+const int unspecified_byteorder = -1;
+
+#ifndef pac_type_defs
+#define pac_type_defs
+
+typedef char 		int8;
+typedef short 		int16;
+typedef long 		int32;
+typedef unsigned char 	uint8;
+typedef unsigned short 	uint16;
+typedef unsigned int 	uint32;
+typedef void		*nullptr;
+typedef void		*voidptr;
+typedef uint8		*byteptr;
+typedef const uint8	*const_byteptr;
+typedef const char	*const_charptr;
+
+#if @SIZEOF_UNSIGNED_INT@ != 4
+#error "unexpected size of unsigned int"
+#endif
+
+#endif /* pac_type_defs */
+
+/* Handling byte order */
+
+namespace {
+
+inline int16 pac_swap(int16 x)
+	{
+	return (x >> 8) | ((x & 0xff) << 8);
+	}
+
+inline uint16 pac_swap(uint16 x)
+	{
+	return (x >> 8) | ((x & 0xff) << 8);
+	}
+
+inline int32 pac_swap(int32 x)
+	{
+	return 	(x >> 24) | 
+		((x & 0xff0000) >> 8) | 
+		((x & 0xff00) << 8) | 
+		((x & 0xff) << 24);
+	}
+
+inline uint32 pac_swap(uint32 x)
+	{
+	return 	(x >> 24) | 
+		((x & 0xff0000) >> 8) | 
+		((x & 0xff00) << 8) | 
+		((x & 0xff) << 24);
+	}
+
+#define FixByteOrder(byteorder, x)	(byteorder == HOST_BYTEORDER ? (x) : pac_swap(x))
+
+template <class T>
+inline T UnMarshall(const u_char *data, int byteorder)
+	{
+	T result = 0;
+	for ( int i = 0; i < (int) sizeof(T); ++i )
+		result = ( result << 8 ) | 
+			data[byteorder == bigendian ? i : sizeof(T) - 1 - i];
+	return result;
+	}
+
+inline const char* do_fmt(const char* format, va_list ap)
+	{
+	static char buf[1024];
+	vsnprintf(buf, sizeof(buf), format, ap);
+	return buf;
+	}
+
+inline string strfmt(const char* format, ...)
+	{
+	va_list ap;
+	va_start(ap, format);
+	const char* r = do_fmt(format, ap);
+	va_end(ap);
+	return string(r);
+	}
+
+} // anonymous namespace
+
+#define binpac_fmt(x...) strfmt(x).c_str()
+
+class RefCount
+{
+public:
+	RefCount() 	{ count = 1; }
+	void Ref() 	{ ++count; }
+	int Unref() 	{ BINPAC_ASSERT(count > 0); return --count; }
+
+private:
+	int count;
+};
+
+namespace {
+	inline void Unref(RefCount *x)
+		{
+		if ( x && x->Unref() <= 0 )
+			delete x;
+		}
+}  // anonymous namespace
+
+} // namespace binpac
+
+#include "binpac_analyzer.h"
+#include "binpac_buffer.h"
+#include "binpac_bytestring.h"
+#include "binpac_exception.h"
+#include "binpac_regex.h"
+
+#endif /* binpac_h */

tools/binpac/lib/binpac_analyzer.h

@@ -0,0 +1,27 @@
+#ifndef binpac_an_h
+#define binpac_an_h
+
+namespace binpac {
+
+// TODO: Add the Done() function
+
+// The interface for a connection analyzer
+class ConnectionAnalyzer {
+public:
+	virtual ~ConnectionAnalyzer() {}
+	virtual void NewData(bool is_orig,
+	                     const u_char *begin_of_data, 
+	                     const u_char *end_of_data) = 0;
+};
+
+// The interface for a flow analyzer
+class FlowAnalyzer {
+public:
+	virtual ~FlowAnalyzer() {}
+	virtual void NewData(const u_char *begin_of_data, 
+	                     const u_char *end_of_data) = 0;
+};
+
+}  // namespace binpac
+
+#endif  // binpac_an_h

tools/binpac/lib/binpac_buffer.cc

@@ -0,0 +1,465 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>	// for memcpy
+
+#define binpac_regex_h
+
+#include "binpac.h"
+#include "binpac_buffer.h"
+
+namespace binpac {
+
+extern double network_time();
+
+namespace {
+	const u_char CR = '\r';
+	const u_char LF = '\n';
+}
+
+FlowBuffer::FlowBuffer(LineBreakStyle linebreak_style)
+	{
+	buffer_length_ = 0;
+	buffer_ = 0;
+
+	orig_data_begin_ = 0;
+	orig_data_end_ = 0;
+
+	linebreak_style_ = linebreak_style;
+	ResetLineState();
+
+	mode_ = UNKNOWN_MODE;
+	frame_length_ = 0;
+	chunked_ = false;
+
+	data_seq_at_orig_data_end_ = 0;
+	eof_ = false;
+	have_pending_request_ = false;
+
+	buffer_n_ = 0;
+
+	NewMessage();
+	}
+
+FlowBuffer::~FlowBuffer()
+	{
+	if ( buffer_ )
+		free(buffer_);
+	}
+
+void FlowBuffer::NewMessage()
+	{
+	BINPAC_ASSERT(frame_length_ >= 0);
+
+	int bytes_to_advance = 0;
+	if ( buffer_n_ == 0 ) 
+		{
+		switch ( mode_ ) 
+			{
+			case LINE_MODE:
+				bytes_to_advance = (frame_length_ + 
+					(linebreak_style_ == STRICT_CRLF ? 
+						2 : 1));
+				break;
+			case FRAME_MODE:
+				bytes_to_advance = frame_length_;
+				break;
+			case UNKNOWN_MODE:
+				break;
+			}
+		}
+
+	orig_data_begin_ += bytes_to_advance;
+	BINPAC_ASSERT(orig_data_begin_ <= orig_data_end_);
+
+	buffer_n_ = 0;
+	message_complete_ = false;
+	}
+
+void FlowBuffer::ResetLineState()
+	{
+	switch ( linebreak_style_ )
+		{
+		case CR_OR_LF:
+			state_ = CR_OR_LF_0;
+			break;
+		case STRICT_CRLF:
+			state_ = STRICT_CRLF_0;
+			break;
+		default:
+			BINPAC_ASSERT(0);
+			break;
+		}
+	}
+
+void FlowBuffer::ExpandBuffer(int length)
+	{
+	if ( buffer_length_ >= length )
+		return;
+	// So length > 0
+	if ( length < 512 )
+		length = 512;
+	
+	if ( length < buffer_length_ * 2 )
+		length = buffer_length_ * 2;
+
+	// Allocate a new buffer and copy the existing contents
+	buffer_length_ = length;
+	u_char *new_buf = (u_char *) realloc(buffer_, buffer_length_);
+	BINPAC_ASSERT(new_buf);
+#if 0
+	u_char* new_buf = new u_char[buffer_length_];
+	if ( buffer_ && buffer_n_ > 0 )
+		memcpy(new_buf, buffer_, buffer_n_);
+	delete [] buffer_;
+#endif
+	buffer_ = new_buf;
+	}
+
+void FlowBuffer::NewLine()
+	{
+	FlowBuffer::NewMessage();
+	mode_ = LINE_MODE;
+	frame_length_ = 0;
+	chunked_ = false;
+	have_pending_request_ = true;
+	if ( state_ == FRAME_0 )
+		ResetLineState();
+	MarkOrCopyLine();
+	}
+
+void FlowBuffer::NewFrame(int frame_length, bool chunked)
+	{
+	FlowBuffer::NewMessage();
+	mode_ = FRAME_MODE;
+	frame_length_ = frame_length;
+	chunked_ = chunked;
+	have_pending_request_ = true;
+	MarkOrCopyFrame();
+	}
+
+void FlowBuffer::GrowFrame(int length)
+	{
+	BINPAC_ASSERT(frame_length_ >= 0);
+	if ( length <= frame_length_ )
+		return;
+	BINPAC_ASSERT(! chunked_ || frame_length_ == 0);
+	mode_ = FRAME_MODE;
+	frame_length_ = length;
+	MarkOrCopyFrame();
+	}
+
+void FlowBuffer::DiscardData()
+	{
+	mode_ = UNKNOWN_MODE;
+	message_complete_ = false;
+	have_pending_request_ = false;
+	orig_data_begin_ = orig_data_end_ = 0;
+
+	buffer_n_ = 0;
+	frame_length_ = 0;
+	}
+
+void FlowBuffer::set_eof()
+	{
+	// fprintf(stderr, "EOF\n");
+	eof_ = true;
+	if ( chunked_ )
+		frame_length_ = orig_data_end_ - orig_data_begin_;
+	if ( frame_length_ < 0 )
+		frame_length_ = 0;
+	}
+
+void FlowBuffer::NewData(const_byteptr begin, const_byteptr end)
+	{
+	BINPAC_ASSERT(begin <= end);
+
+	ClearPreviousData();
+
+	BINPAC_ASSERT((buffer_n_ == 0 && message_complete_) || 
+		orig_data_begin_ == orig_data_end_);
+
+	orig_data_begin_ = begin;
+	orig_data_end_ = end;
+	data_seq_at_orig_data_end_ += (end - begin);
+
+	MarkOrCopy();
+	}
+
+void FlowBuffer::MarkOrCopy()
+	{
+	if ( ! message_complete_ )
+		{
+		switch ( mode_ ) 
+			{
+			case LINE_MODE:
+				MarkOrCopyLine();
+				break;
+
+			case FRAME_MODE:
+				MarkOrCopyFrame();
+				break;
+		
+			default:
+				break;
+			}
+		}
+	}
+
+void FlowBuffer::ClearPreviousData()
+	{
+	// All previous data must have been processed or buffered already
+	if ( orig_data_begin_ < orig_data_end_ )
+		{
+		BINPAC_ASSERT(buffer_n_ == 0);
+		if ( chunked_ )
+			{
+			if ( frame_length_ > 0 )
+				{
+				frame_length_ -= 
+					(orig_data_end_ - orig_data_begin_);
+				}
+			orig_data_begin_ = orig_data_end_;
+			}
+		}
+	}
+
+void FlowBuffer::NewGap(int length)
+	{
+	ClearPreviousData();
+
+	if ( chunked_ && frame_length_ >= 0 )
+		{
+		frame_length_ -= length;
+		if ( frame_length_ < 0 )
+			frame_length_ = 0;
+		}
+
+	orig_data_begin_ = orig_data_end_ = 0;
+	MarkOrCopy();
+	}
+
+void FlowBuffer::MarkOrCopyLine()
+	{
+	switch ( linebreak_style_ )
+		{
+		case CR_OR_LF:
+			MarkOrCopyLine_CR_OR_LF();
+			break;
+		case STRICT_CRLF:
+			MarkOrCopyLine_STRICT_CRLF();
+			break;
+		default:
+			BINPAC_ASSERT(0);
+			break;
+		}
+	}
+
+/*
+Finite state automaton for CR_OR_LF:
+(!--line is complete, *--add to buffer)
+
+CR_OR_LF_0:
+	CR:	CR_OR_LF_1 !
+	LF:	CR_OR_LF_0 !
+	.:	CR_OR_LF_0 *
+
+CR_OR_LF_1:
+	CR:	CR_OR_LF_1 !
+	LF:	CR_OR_LF_0
+	.:	CR_OR_LF_0 *
+*/
+
+void FlowBuffer::MarkOrCopyLine_CR_OR_LF()
+	{
+	if ( state_ == CR_OR_LF_1 && 
+	     orig_data_begin_ < orig_data_end_ && *orig_data_begin_ == LF )
+		{
+		state_ = CR_OR_LF_0;
+		++orig_data_begin_;
+		}
+
+	const_byteptr data;
+	for ( data = orig_data_begin_; data < orig_data_end_; ++data )
+		{
+		switch ( *data )
+			{
+			case CR:
+				state_ = CR_OR_LF_1;
+				goto found_end_of_line;
+
+			case LF:
+				// state_ = CR_OR_LF_0;
+				goto found_end_of_line;
+
+			default:
+				// state_ = CR_OR_LF_0;
+				break;
+			}
+		}
+
+	AppendToBuffer(orig_data_begin_, orig_data_end_ - orig_data_begin_);
+	return;
+
+found_end_of_line:
+	if ( buffer_n_ == 0 )
+		{
+		frame_length_ = data - orig_data_begin_;
+		}
+	else
+		{
+		AppendToBuffer(orig_data_begin_, data + 1 - orig_data_begin_);
+		// But eliminate the last CR or LF
+		--buffer_n_;
+		}
+	message_complete_ = true;
+
+#if DEBUG_FLOW_BUFFER
+	fprintf(stderr, "%.6f Line complete: [%s]\n",
+		network_time(),
+		string((const char *) begin(), (const char *) end()).c_str());
+#endif
+	}
+
+/*
+Finite state automaton and STRICT_CRLF:
+(!--line is complete, *--add to buffer)
+
+STRICT_CRLF_0:
+	CR:	STRICT_CRLF_1 *
+	LF:	STRICT_CRLF_0 *
+	.:	STRICT_CRLF_0 *
+
+STRICT_CRLF_1:
+	CR:	STRICT_CRLF_1 *
+	LF:	STRICT_CRLF_0 ! (--buffer_n_)
+	.:	STRICT_CRLF_0 *
+*/
+
+void FlowBuffer::MarkOrCopyLine_STRICT_CRLF()
+	{
+	const_byteptr data;
+	for ( data = orig_data_begin_; data < orig_data_end_; ++data )
+		{
+		switch ( *data )
+			{
+			case CR:
+				state_ = STRICT_CRLF_1;
+				break;
+
+			case LF:
+				if ( state_ == STRICT_CRLF_1 )
+					{
+					state_ = STRICT_CRLF_0;
+					goto found_end_of_line;
+					}
+				break;
+
+			default:
+				state_ = STRICT_CRLF_0;
+				break;
+			}
+		}
+
+	AppendToBuffer(orig_data_begin_, orig_data_end_ - orig_data_begin_);
+	return;
+
+found_end_of_line:
+	if ( buffer_n_ == 0 )
+		{
+		frame_length_ = data - 1 - orig_data_begin_;
+		}
+	else
+		{
+		AppendToBuffer(orig_data_begin_, data + 1 - orig_data_begin_);
+		// Pop the preceding CR and LF from the buffer
+		buffer_n_ -= 2;
+		}
+
+	message_complete_ = true;
+
+#if DEBUG_FLOW_BUFFER
+	fprintf(stderr, "%.6f Line complete: [%s]\n",
+		network_time(),
+		string((const char *) begin(), (const char *) end()).c_str());
+#endif
+	}
+
+// Invariants:
+// 
+// When buffer_n_ == 0:
+// Frame = [orig_data_begin_..(orig_data_begin_ + frame_length_)]
+// 
+// When buffer_n_ > 0:
+// Frame = [0..buffer_n_][orig_data_begin_..]
+
+void FlowBuffer::MarkOrCopyFrame()
+	{
+	if ( mode_ == FRAME_MODE && state_ == CR_OR_LF_1 && 
+	     orig_data_begin_ < orig_data_end_ )
+		{
+		// Skip the lingering LF
+		if ( *orig_data_begin_ == LF )
+			{
+			++orig_data_begin_;
+			}
+		state_ = FRAME_0;
+		}
+
+	if ( buffer_n_ == 0 )
+		{
+		// If there is enough data
+		if ( frame_length_ >= 0 &&
+		     orig_data_end_ - orig_data_begin_ >= frame_length_ )
+			{
+			// Do nothing except setting the message complete flag
+			message_complete_ = true;
+			}
+		else 
+			{
+			if ( ! chunked_ )
+				{
+				AppendToBuffer(orig_data_begin_, 
+					orig_data_end_ - orig_data_begin_);
+				}
+			message_complete_ = false;
+			}
+		}
+	else
+		{
+		BINPAC_ASSERT(!chunked_);
+		int bytes_to_copy = orig_data_end_ - orig_data_begin_;
+		message_complete_ = false;
+		if ( frame_length_ >= 0 && buffer_n_ + bytes_to_copy >= frame_length_ ) 
+			{
+			bytes_to_copy = frame_length_ - buffer_n_;
+			message_complete_ = true;
+			}
+		AppendToBuffer(orig_data_begin_, bytes_to_copy);
+		}
+
+#if DEBUG_FLOW_BUFFER
+	if ( message_complete_ )
+		{
+		fprintf(stderr, "%.6f frame complete: [%s]\n",
+			network_time(),
+			string((const char *) begin(), 
+				(const char *) end()).c_str());
+		}
+#endif
+	}
+
+void FlowBuffer::AppendToBuffer(const_byteptr data, int len)
+	{
+	if ( len <= 0 )
+		return;
+
+	BINPAC_ASSERT(! chunked_);
+	ExpandBuffer(buffer_n_ + len);
+	memcpy(buffer_ + buffer_n_, data, len);
+	buffer_n_ += len;
+
+	orig_data_begin_ += len;
+	BINPAC_ASSERT(orig_data_begin_ <= orig_data_end_);
+	}
+
+}  // namespace binpac

tools/binpac/lib/binpac_buffer.h

@@ -0,0 +1,148 @@
+#ifndef binpac_buffer_h
+#define binpac_buffer_h
+
+#include <sys/types.h>
+#include "binpac.h"
+
+namespace binpac {
+
+class FlowBuffer {
+public:
+	enum LineBreakStyle { 
+		CR_OR_LF, 	// CR or LF or CRLF
+		STRICT_CRLF, 	// CR followed by LF
+		CR_LF_NUL,	// CR or LF or CR-LF or CR-NUL
+	};
+
+	FlowBuffer(LineBreakStyle linebreak_style = CR_OR_LF);
+	virtual ~FlowBuffer();
+
+	void NewData(const_byteptr begin, const_byteptr end);
+	void NewGap(int length);
+
+	// Discard unprocessed data
+	void DiscardData();
+
+	// Whether there is enough data for the frame
+	bool ready() const{ return message_complete_ || mode_ == UNKNOWN_MODE; }
+
+	inline const_byteptr begin() const
+		{
+		BINPAC_ASSERT(ready());
+		return ( buffer_n_ == 0 ) ? 
+			orig_data_begin_ : buffer_;
+		}
+
+	inline const_byteptr end() const
+		{
+		BINPAC_ASSERT(ready());
+		if ( buffer_n_ == 0 )
+			{
+			BINPAC_ASSERT(frame_length_ >= 0);
+			const_byteptr end = orig_data_begin_ + frame_length_;
+			BINPAC_ASSERT(end <= orig_data_end_);
+			return end;
+			}
+		else
+			return buffer_ + buffer_n_;
+		}
+
+	inline int data_length() const
+		{
+		if ( buffer_n_ > 0 ) 
+			return buffer_n_;
+
+		if ( frame_length_ < 0 ||
+		     orig_data_begin_ + frame_length_ > orig_data_end_ )
+			return orig_data_end_ - orig_data_begin_;
+		else
+			return frame_length_;
+		}
+
+	inline bool data_available() const
+	        {
+		return buffer_n_ > 0 || orig_data_end_ > orig_data_begin_;
+		}
+
+	void NewLine();
+	// A negative frame_length represents a frame till EOF
+	void NewFrame(int frame_length, bool chunked_);
+	void GrowFrame(int new_frame_length);
+
+	int data_seq() const
+		{ 
+		int data_seq_at_orig_data_begin = 
+			data_seq_at_orig_data_end_ - 
+			(orig_data_end_ - orig_data_begin_);
+		if ( buffer_n_ > 0 )
+			return data_seq_at_orig_data_begin;
+		else
+			return data_seq_at_orig_data_begin + data_length(); 
+		}
+	bool eof() const	{ return eof_; }
+	void set_eof();
+
+	bool have_pending_request() const { return have_pending_request_; }
+
+protected:
+	// Reset the buffer for a new message
+	void NewMessage();
+
+	void ClearPreviousData();
+
+	// Expand the buffer to at least <length> bytes. If there 
+	// are contents in the existing buffer, copy them to the new
+	// buffer.
+	void ExpandBuffer(int length);
+
+	// Reset line state when transit from frame mode to line mode.
+	void ResetLineState();
+
+	void AppendToBuffer(const_byteptr data, int len);
+
+	// MarkOrCopy{Line,Frame} sets message_complete_ and
+	// marks begin/end pointers if a line/frame is complete, 
+	// otherwise it clears message_complete_ and copies all 
+	// the original data to the buffer.
+	//
+	void MarkOrCopy();
+	void MarkOrCopyLine();
+	void MarkOrCopyFrame();
+
+	void MarkOrCopyLine_CR_OR_LF();
+	void MarkOrCopyLine_STRICT_CRLF();
+
+	int 	buffer_n_;		// number of bytes in the buffer
+	int 	buffer_length_;	// size of the buffer
+	u_char	*buffer_;
+	bool 	message_complete_;
+	int 	frame_length_;
+	bool	chunked_;
+	const_byteptr orig_data_begin_, orig_data_end_;
+
+	LineBreakStyle linebreak_style_;
+
+	enum {
+		UNKNOWN_MODE,
+		LINE_MODE,
+		FRAME_MODE,
+	} mode_;
+
+	enum {
+		CR_OR_LF_0,
+		CR_OR_LF_1,
+		STRICT_CRLF_0,
+		STRICT_CRLF_1,
+		FRAME_0,
+	} state_;
+
+	int 	data_seq_at_orig_data_end_;
+	bool 	eof_;
+	bool    have_pending_request_;
+};
+
+typedef FlowBuffer *flow_buffer_t;
+
+}  // namespace binpac
+
+#endif // binpac_buffer_h

tools/binpac/lib/binpac_bytestring.cc

@@ -0,0 +1,24 @@
+#define binpac_regex_h
+
+#include <stdlib.h>
+#include "binpac_bytestring.h"
+
+namespace binpac 
+{
+
+std::string std_string(bytestring const *s)
+	{
+	return std::string((const char *) s->begin(), (const char *) s->end());
+	}
+
+int bytestring_to_int(bytestring const *s)
+	{
+	return atoi((const char *) s->begin());
+	}
+
+double bytestring_to_double(bytestring const *s)
+	{
+	return atof((const char *) s->begin());
+	}
+
+}  // namespace binpac

tools/binpac/lib/binpac_bytestring.h

@@ -0,0 +1,199 @@
+#ifndef binpac_bytestring_h
+#define binpac_bytestring_h
+
+#include <string.h>
+#include <string>
+#include "binpac.h"
+
+namespace binpac
+{
+
+template<class T> class datastring;
+
+template <class T>
+class const_datastring
+{
+public:
+	const_datastring()
+		: begin_(0), end_(0)
+		{ 
+		}
+
+	const_datastring(T const *data, int length)
+		: begin_(data), end_(data + length)
+		{
+		}
+
+	const_datastring(const T *begin, const T *end)
+		: begin_(begin), end_(end)
+		{
+		}
+
+	const_datastring(datastring<T> const &s)
+		: begin_(s.begin()), end_(s.end())
+		{
+		}
+
+	void init(const T *data, int length)
+		{
+		begin_ = data;
+		end_ = data + length;
+		}
+
+	T const *begin() const	{ return begin_; }
+	T const *end() const	{ return end_; }
+	int length() const	{ return end_ - begin_; }
+
+	T const &operator[](int index) const
+		{
+		return begin()[index];
+		}
+
+	bool operator==(const_datastring<T> const &s)
+		{
+		if ( length() != s.length() )
+			return false;
+		return memcmp((const void *) begin(), (const void *) s.begin(),
+			sizeof(T) * length()) == 0;
+		}
+
+	void set_begin(T const *begin)	{ begin_ = begin; }
+	void set_end(T const *end)	{ end_ = end; }
+
+private:
+	T const *begin_; 
+	T const *end_;
+};
+
+typedef const_datastring<uint8>	const_bytestring;
+
+template<class T>
+class datastring
+{
+public:
+	datastring()
+		{ 
+		clear();
+		}
+
+	datastring(T *data, int len)
+		{
+		set(data, len);
+		}
+
+	datastring(T const *begin, T const *end)
+		{
+		set_const(begin, end - begin);
+		}
+
+	datastring(datastring<T> const &x)
+		: data_(x.data()), length_(x.length())
+		{
+		}
+
+	explicit datastring(const_datastring<T> const &x)
+		{
+		set_const(x.begin(), x.length());
+		}
+
+	datastring const &operator=(datastring<T> const &x)
+		{
+		BINPAC_ASSERT(!data_);
+		set(x.data(), x.length());
+		return *this;
+		}
+
+	void init(T const *begin, int length)
+		{
+		BINPAC_ASSERT(!data_);
+		set_const(begin, length);
+		}
+
+	void clear()
+		{
+		data_ = 0; length_ = 0;
+		}
+
+	void free()
+		{
+		if ( data_ )
+			delete [] data_;
+		clear();
+		}
+
+	void clone()
+		{
+		set_const(begin(), length());
+		}
+
+	datastring const &operator=(const_datastring<T> const &x)
+		{
+		BINPAC_ASSERT(!data_);
+		set_const(x.begin(), x.length());
+		return *this;
+		}
+
+	T const &operator[](int index) const
+		{
+		return begin()[index];
+		}
+
+	T *data() const		{ return data_; }
+	int length() const	{ return length_; }
+
+	T const *begin() const	{ return data_; }
+	T const *end() const	{ return data_ + length_; }
+
+private:
+	void set(T *data, int len)
+		{
+		data_ = data;
+		length_ = len;
+		}
+
+	void set_const(T const *data, int len)
+		{
+		length_ = len;
+		data_ = new T[len + 1];
+		memcpy(data_, data, sizeof(T) * len);
+		data_[len] = 0;
+		}
+
+	T * data_;
+	int length_;
+};
+
+typedef datastring<uint8> bytestring;
+
+inline const char *c_str(bytestring const &s)
+	{
+	return (const char *) s.begin();
+	}
+
+inline std::string std_str(const_bytestring const &s) 
+	{
+	return std::string((const char *) s.begin(), (const char *) s.end());
+	}
+
+inline bool operator==(bytestring const &s1, const char *s2)
+	{
+	return strcmp(c_str(s1), s2) == 0;
+	}
+
+inline void get_pointers(const_bytestring const &s, 
+		uint8 const **pbegin, uint8 const **pend)
+	{
+	*pbegin = s.begin();
+	*pend = s.end();
+	}
+
+inline void get_pointers(bytestring const *s, 
+		uint8 const **pbegin, uint8 const **pend)
+	{
+	*pbegin = s->begin();
+	*pend = s->end();
+	}
+
+} // namespace binpac
+
+#endif  // binpac_bytestring_h

tools/binpac/lib/binpac_exception.h

@@ -0,0 +1,112 @@
+#ifndef binpac_exception_h
+#define binpac_exception_h
+
+namespace binpac {
+
+class Exception
+{
+public:
+	Exception(const char* m = 0)
+		: msg_("binpac exception: ")
+		{
+		if ( m )
+			append(m);
+		// abort();
+		}
+
+	void append(string m) 		{ msg_ += m; }
+	string msg() const		{ return msg_; }
+	const char* c_msg() const 	{ return msg().c_str(); }
+
+protected:
+	string msg_;
+};
+
+class ExceptionOutOfBound : public Exception
+{
+public:
+	ExceptionOutOfBound(const char* where, int len_needed, int len_given)
+		{
+		append(binpac_fmt("out_of_bound: %s: %d > %d", 
+			where, len_needed, len_given));
+		}
+};
+
+class ExceptionInvalidCase : public Exception
+{
+public:
+	ExceptionInvalidCase(const char* location, 
+			int index,
+			const char *expected)
+		: location_(location), 
+		  index_(index), 
+		  expected_(expected)
+		{
+		append(binpac_fmt("invalid case: %s: %d (%s)",
+			location, index, expected));
+		}
+
+protected:
+	const char* location_;
+	int index_;
+	string expected_;
+};
+
+class ExceptionInvalidCaseIndex : public Exception
+{
+public:
+	ExceptionInvalidCaseIndex(const char* location, 
+			int index)
+		: location_(location), 
+		  index_(index)
+		{
+		append(binpac_fmt("invalid index for case: %s: %d",
+			location, index));
+		}
+
+protected:
+	const char* location_;
+	int index_;
+};
+
+class ExceptionInvalidOffset : public Exception
+{
+public:
+	ExceptionInvalidOffset(const char* location, 
+			int min_offset, int offset)
+		: location_(location), 
+		  min_offset_(min_offset), offset_(offset)
+		{
+		append(binpac_fmt("invalid offset: %s: min_offset = %d, offset = %d",
+			location, min_offset, offset));
+		}
+
+protected:
+	const char* location_;
+	int min_offset_, offset_;
+};
+
+class ExceptionStringMismatch : public Exception
+{
+public:
+	ExceptionStringMismatch(const char* location, 
+			const char *expected, const char *actual_data)
+		{
+		append(binpac_fmt("string mismatch at %s: \nexpected pattern: \"%s\"\nactual data: \"%s\"",
+			location, expected, actual_data));
+		}
+};
+
+class ExceptionInvalidStringLength : public Exception
+{
+public:
+	ExceptionInvalidStringLength(const char* location, int len)
+		{
+		append(binpac_fmt("invalid length string: %s: %d",
+			location, len));
+		}
+};
+
+}
+
+#endif  // binpac_exception_h

tools/binpac/lib/binpac_regex.h

@@ -0,0 +1,43 @@
+#ifndef binpac_regex_h
+#define binpac_regex_h
+
+#include "binpac.h"
+#include "RE.h"
+
+class RE_Matcher;
+
+namespace binpac
+{
+
+class RegExMatcher {
+public:
+	RegExMatcher(const char *pattern)
+		: pattern_(pattern)
+		{
+		re_matcher_ = 0;
+		}
+
+	~RegExMatcher()
+		{
+		delete re_matcher_;
+		}
+
+	// Returns the length of longest match, or -1 on mismatch.
+	int MatchPrefix(const_byteptr data, int len)
+		{
+		if ( ! re_matcher_ )
+			{
+			re_matcher_ = new RE_Matcher(pattern_.c_str());
+			re_matcher_->Compile();
+			}
+		return re_matcher_->MatchPrefix(data, len);
+		}
+
+private:
+	string pattern_;
+	RE_Matcher *re_matcher_;
+};
+
+}  // namespace binpac
+
+#endif  // binpac_regex_h

tools/binpac/patches/binpac-1.patch

@@ -0,0 +1,31 @@
+diff -urN bro-1.2.1-orig/src/binpac/pac_expr.cc bro-1.2.1-ssl-binpac/src/binpac/pac_expr.cc
+--- bro-1.2.1-orig/src/binpac/pac_expr.cc	2006-07-26 15:02:40.000000000 -0700
++++ bro-1.2.1-ssl-binpac/src/binpac/pac_expr.cc	2007-05-04 14:31:11.728494000 -0700
+@@ -776,6 +776,27 @@
+ 			}
+ 			break;
+ 
++		case EXPR_CALLARGS:
++		        {
++		        mhs = 0;
++			if ( args_ )
++			        for ( uint i = 0; i < args_->size(); ++i )
++				        mhs = mhs_max(mhs, args_->at(i)->MinimalHeaderSize(env));
++			}
++		        break;
++		case EXPR_CASE:
++		        {
++		        mhs = operand_[0]->MinimalHeaderSize(env);
++			for ( uint i = 0; i < cases_->size(); ++i )
++			        {
++				CaseExpr * ce = cases_->at(i);
++				if ( ce->index() )
++				        for ( uint j = 0; j < ce->index()->size(); ++j )
++						mhs = mhs_max(mhs, ce->index()->at(j)->MinimalHeaderSize(env));
++				mhs = mhs_max(mhs, ce->value()->MinimalHeaderSize(env));
++				}
++			}
++			break;
+ 		default:
+ 			// Evaluate every operand by default
+ 			mhs = 0;

tools/binpac/patches/binpac-2.patch

@@ -0,0 +1,97 @@
+diff -urN bro-1.2.1-orig/src/binpac/pac_expr.cc bro-1.2.1-ssl-binpac/src/binpac/pac_expr.cc
+--- bro-1.2.1-orig/src/binpac/pac_expr.cc	2006-07-26 15:02:40.000000000 -0700
++++ bro-1.2.1-ssl-binpac/src/binpac/pac_expr.cc	2007-05-04 14:31:11.728494000 -0700
+@@ -245,6 +245,12 @@
+ 	out_cc->println("%s %s;", 
+ 	                val_type->DataTypeStr().c_str(), 
+ 	                env->LValue(val_var));
++
++	// force evaluation of IDs appearing in case stmt
++       operand_[0]->ForceIDEval(out_cc, env);
++	foreach(i, CaseExprList, cases_)
++		(*i)->value()->ForceIDEval(out_cc, env);
++
+ 	out_cc->println("switch ( %s )", operand_[0]->EvalExpr(out_cc, env));
+ 
+ 	out_cc->inc_indent();
+@@ -386,6 +392,49 @@
+ 		}
+ 	}
+ 
++void Expr::ForceIDEval(Output* out_cc, Env* env)
++        {
++	switch ( expr_type_ )
++		{
++		case EXPR_NUM:
++		case EXPR_SIZEOF:
++		case EXPR_OFFSETOF:
++			break;
++
++		case EXPR_ID:
++			if ( ! env->Evaluated(id_) )
++				env->Evaluate(out_cc, id_);
++			break;
++
++		case EXPR_MEMBER:
++			operand_[0]->ForceIDEval(out_cc, env);
++			break;
++
++		case EXPR_CALLARGS:
++		        {
++		        foreach(i, ExprList, args_)
++			        (*i)->ForceIDEval(out_cc, env);
++			}
++			break;
++
++		case EXPR_CASE:
++		        {
++		        operand_[0]->ForceIDEval(out_cc, env);
++			foreach(i, CaseExprList, cases_)
++				(*i)->value()->ForceIDEval(out_cc, env);
++		        }
++			break;
++
++		default:
++			// Evaluate every operand by default
++			for ( int i = 0; i < 3; ++i )
++				if ( operand_[i] )
++					operand_[i]->ForceIDEval(out_cc, env);
++			break;
++		}
++	}
++
++
+ const char* Expr::EvalExpr(Output* out_cc, Env* env)
+ 	{
+ 	GenEval(out_cc, env);
+diff -urN bro-1.2.1-orig/src/binpac/pac_expr.h bro-1.2.1-ssl-binpac/src/binpac/pac_expr.h
+--- bro-1.2.1-orig/src/binpac/pac_expr.h	2006-07-26 15:02:39.000000000 -0700
++++ bro-1.2.1-ssl-binpac/src/binpac/pac_expr.h	2007-05-04 14:16:31.624287000 -0700
+@@ -56,6 +56,11 @@
+ 	//
+ 	const char *EvalExpr(Output *out, Env *env);
+ 
++	// force evaulation of IDs contained in this expression;
++	// necessary with case expr and conditional let fields (&if)
++	// for correct parsing of fields
++	void ForceIDEval(Output *out_cc, Env *env);
++
+ 	// Returns the set_* function of the expression. 
+ 	// The expression must be of form ID or x.ID.
+ 	string SetFunc(Output *out, Env *env);
+diff -urN bro-1.2.1-orig/src/binpac/pac_let.cc bro-1.2.1-ssl-binpac/src/binpac/pac_let.cc
+--- bro-1.2.1-orig/src/binpac/pac_let.cc	2006-07-26 15:02:39.000000000 -0700
++++ bro-1.2.1-ssl-binpac/src/binpac/pac_let.cc	2007-05-04 15:32:09.695568000 -0700
+@@ -80,7 +80,12 @@
+ 	if ( type_->attr_if_expr() )
+ 		{
+ 		// A conditional field
++
+ 		env->Evaluate(out_cc, type_->has_value_var());
++
++		// force evaluation of IDs contained in this expr
++		expr()->ForceIDEval(out_cc, env);
++
+ 		out_cc->println("if ( %s )", 
+ 			env->RValue(type_->has_value_var()));
+ 		out_cc->inc_indent();

tools/binpac/patches/binpac-3.patch

@@ -0,0 +1,37 @@
+diff -urN bro-1.2.1-orig/src/binpac/pac_let.cc bro-1.2.1-ssl-binpac/src/binpac/pac_let.cc
+--- bro-1.2.1-orig/src/binpac/pac_let.cc	2006-07-26 15:02:39.000000000 -0700
++++ bro-1.2.1-ssl-binpac/src/binpac/pac_let.cc	2007-05-04 15:32:09.695568000 -0700
+@@ -108,11 +108,6 @@
+ void LetField::GenEval(Output* out_cc, Env* env)
+ 	{
+ 	GenParseCode(out_cc, env);
+-	if ( type_->attr_if_expr() )
+-		{
+-		out_cc->println("BINPAC_ASSERT(%s);", 
+-			env->RValue(type_->has_value_var()));
+-		}
+ 	}
+ 
+ LetDecl::LetDecl(ID *id, Type *type, Expr *expr)
+diff -urN bro-1.2.1-orig/src/binpac/pac_type.cc bro-1.2.1-ssl-binpac/src/binpac/pac_type.cc
+--- bro-1.2.1-orig/src/binpac/pac_type.cc	2006-07-26 15:02:40.000000000 -0700
++++ bro-1.2.1-ssl-binpac/src/binpac/pac_type.cc	2007-05-24 10:56:42.140658000 -0700
+@@ -316,9 +316,15 @@
+ 	{
+ 	if ( DefineValueVar() )
+ 		{
+-		out_h->println("%s %s const	{ return %s; }", 
+-			DataTypeConstRefStr().c_str(), 
+-			env->RValue(value_var()), lvalue());
++		if ( attr_if_expr_ )
++		        out_h->println("%s %s const { BINPAC_ASSERT(%s); return %s; }",
++			        DataTypeConstRefStr().c_str(), 
++			        env->RValue(value_var()),
++			        env->RValue(has_value_var()), lvalue());
++		else
++		        out_h->println("%s %s const { return %s; }",
++			        DataTypeConstRefStr().c_str(), 
++			        env->RValue(value_var()), lvalue());
+ 		}
+ 
+ 	foreach (i, FieldList, fields_)

tools/binpac/patches/binpac-4.patch

@@ -0,0 +1,12 @@
+diff -urN bro-1.2.1-orig/src/binpac/pac_record.cc bro-1.2.1-ssl-binpac/src/binpac/pac_record.cc
+--- bro-1.2.1-orig/src/binpac/pac_record.cc	2006-07-26 15:02:40.000000000 -0700
++++ bro-1.2.1-ssl-binpac/src/binpac/pac_record.cc	2007-05-08 16:13:33.373850000 -0700
+@@ -123,7 +123,7 @@
+ void RecordType::DoGenParseCode(Output* out_cc, Env* env, 
+ 		const DataPtr& data, int flags)
+ 	{
+-	if ( StaticSize(env) >= 0 )
++	if ( !incremental_input() && StaticSize(env) >= 0 )
+ 		GenBoundaryCheck(out_cc, env, data);
+ 
+ 	if ( incremental_parsing() )

tools/binpac/patches/binpac-5.patch

@@ -0,0 +1,66 @@
+diff -urN bro-1.2.1-orig/src/binpac/pac_paramtype.cc bro-1.2.1-ssl-binpac/src/binpac/pac_paramtype.cc
+--- bro-1.2.1-orig/src/binpac/pac_paramtype.cc	2006-07-26 15:02:40.000000000 -0700
++++ bro-1.2.1-ssl-binpac/src/binpac/pac_paramtype.cc	2007-05-10 15:09:47.470104000 -0700
+@@ -208,7 +208,13 @@
+ 	const char *parse_func;
+ 	string parse_params;
+ 
+-	if ( ref_type->incremental_input() )
++	if ( buffer_mode() == BUFFER_NOTHING )
++	        {
++		ASSERT(!ref_type->incremental_input());
++		parse_func = kParseFuncWithoutBuffer;
++		parse_params = "0, 0";
++		}
++	else if ( ref_type->incremental_input() )
+ 		{
+ 		parse_func = kParseFuncWithBuffer;
+ 		parse_params = env->RValue(flow_buffer_id);
+@@ -239,15 +245,24 @@
+ 
+ 	if ( incremental_input() )
+ 		{
+-		ASSERT(parsing_complete_var());
+-		out_cc->println("%s = %s;",
+-			env->LValue(parsing_complete_var()),
+-			call_parse_func.c_str());
+-
+-		// parsing_complete_var might have been already
+-		// evaluated when set to false
+-		if ( ! env->Evaluated(parsing_complete_var()) )
+-			env->SetEvaluated(parsing_complete_var());
++		if ( buffer_mode() == BUFFER_NOTHING )
++		        {
++		        out_cc->println("%s;", call_parse_func.c_str());
++			out_cc->println("%s = true;", 
++				env->LValue(parsing_complete_var()));
++			}
++		else
++		        {
++			ASSERT(parsing_complete_var());
++			out_cc->println("%s = %s;",
++				env->LValue(parsing_complete_var()),
++				call_parse_func.c_str());
++
++			// parsing_complete_var might have been already
++			// evaluated when set to false
++			if ( ! env->Evaluated(parsing_complete_var()) )
++			        env->SetEvaluated(parsing_complete_var());
++			}
+ 		}
+ 	else
+ 		{
+diff -urN bro-1.2.1-orig/src/binpac/pac_type.cc bro-1.2.1-ssl-binpac/src/binpac/pac_type.cc
+--- bro-1.2.1-orig/src/binpac/pac_type.cc	2006-07-26 15:02:40.000000000 -0700
++++ bro-1.2.1-ssl-binpac/src/binpac/pac_type.cc	2007-05-24 10:56:42.140658000 -0700
+@@ -501,8 +501,8 @@
+ 
+ 		if ( buffer_mode() == BUFFER_NOTHING )
+ 			{
+-			out_cc->println("%s = true;", 
+-				env->LValue(parsing_complete_var()));
++			// this is the empty type
++			DoGenParseCode(out_cc, env, data, flags);
+ 			}
+ 		else if ( buffer_input() )
+ 			{

tools/binpac/patches/binpac-6.patch

@@ -0,0 +1,28 @@
+diff -urN bro-1.2.1-orig/src/binpac/lib/binpac_buffer.h bro-1.2.1-ssl-binpac/src/binpac/lib/binpac_buffer.h
+--- bro-1.2.1-orig/src/binpac/lib/binpac_buffer.h	2006-07-26 15:02:38.000000000 -0700
++++ bro-1.2.1-ssl-binpac/src/binpac/lib/binpac_buffer.h	2007-05-09 16:14:54.501656000 -0700
+@@ -59,6 +59,11 @@
+ 			return frame_length_;
+ 		}
+ 
++	inline bool data_available() const
++	        {
++		return buffer_n_ > 0 || orig_data_end_ > orig_data_begin_;
++		}
++
+ 	void NewLine();
+ 	// A negative frame_length represents a frame till EOF
+ 	void NewFrame(int frame_length, bool chunked_);
+diff -urN bro-1.2.1-orig/src/binpac/pac_flow.cc bro-1.2.1-ssl-binpac/src/binpac/pac_flow.cc
+--- bro-1.2.1-orig/src/binpac/pac_flow.cc	2006-10-12 14:13:12.000000000 -0700
++++ bro-1.2.1-ssl-binpac/src/binpac/pac_flow.cc	2007-05-22 16:43:55.997562000 -0700
+@@ -272,7 +272,8 @@
+ 		env_->RValue(begin_of_data), 
+ 		env_->RValue(end_of_data)); 
+ 
+-	out_cc->println("while ( true )");
++	out_cc->println("while ( %s->data_available() )",
++		env_->LValue(flow_buffer_id));
+ 	out_cc->inc_indent();
+ 	out_cc->println("{");
+ 

tools/binpac/patches/binpac-7.patch

@@ -0,0 +1,21 @@
+diff -urN bro-1.2.1-orig/src/binpac/pac_type.cc bro-1.2.1-ssl-binpac/src/binpac/pac_type.cc
+--- bro-1.2.1-orig/src/binpac/pac_type.cc	2006-07-26 15:02:40.000000000 -0700
++++ bro-1.2.1-ssl-binpac/src/binpac/pac_type.cc	2007-05-24 10:56:42.140658000 -0700
+@@ -393,7 +393,7 @@
+ 			break;
+ 
+ 		case BUFFER_BY_LENGTH:
+-			if ( buffering_state_var_field_ )
++		        if ( env->GetDataType(buffering_state_id) )
+ 				{
+ 				out_cc->println("if ( %s == 0 )",
+ 					env->RValue(buffering_state_id));
+@@ -421,7 +421,7 @@
+ 				frame_buffer_arg.c_str(),
+ 				attr_chunked() ? "true" : "false");
+ 
+-			if ( buffering_state_var_field_ )
++			if ( env->GetDataType(buffering_state_id) )
+ 				{
+ 				out_cc->println("%s = 1;",
+ 					env->LValue(buffering_state_id));

tools/binpac/patches/binpac-8.patch

@@ -0,0 +1,190 @@
+diff -urN bro-1.2.1-orig/src/binpac/pac_analyzer.cc bro-1.2.1-ssl-binpac/src/binpac/pac_analyzer.cc
+--- bro-1.2.1-orig/src/binpac/pac_analyzer.cc	2006-07-26 15:02:40.000000000 -0700
++++ bro-1.2.1-ssl-binpac/src/binpac/pac_analyzer.cc	2007-05-22 17:00:10.091531000 -0700
+@@ -26,8 +26,9 @@
+ 	helpers_ = new AnalyzerHelperList();
+ 	functions_ = new FunctionList();
+ 
+-	constructor_helper_ = 0;
+-	destructor_helper_ = 0;
++	constructor_helpers_ = new AnalyzerHelperList();
++	destructor_helpers_ = new AnalyzerHelperList();
++	eof_helpers_ = new AnalyzerHelperList();
+ 
+ 	SetAnalyzerContext();
+ 
+@@ -41,6 +42,9 @@
+ 	delete_list(AnalyzerHelperList, helpers_);
+ 	delete_list(FunctionList, functions_);
+ 	delete_list(ParamList, params_);
++	delete_list(AnalyzerHelperList, constructor_helpers_);
++	delete_list(AnalyzerHelperList, destructor_helpers_);
++	delete_list(AnalyzerHelperList, eof_helpers_);
+ 	}
+ 
+ void AnalyzerDecl::AddElements(AnalyzerElementList *elemlist)
+@@ -75,28 +79,20 @@
+ 				AnalyzerHelper *helper_elem = 
+ 					(AnalyzerHelper *) elem;
+ 
+-				if ( helper_elem->helper_type() == 
+-				     AnalyzerHelper::INIT_CODE)
+-					{
+-					if ( constructor_helper_ )
+-						{
+-						throw Exception(elem,
+-							"Repeated definition of %init code");
+-						}
+-					constructor_helper_ = helper_elem;
++				switch ( helper_elem->helper_type() )
++				        {
++					case AnalyzerHelper::INIT_CODE:
++					        constructor_helpers_->push_back(helper_elem);
++						break;
++					case AnalyzerHelper::CLEANUP_CODE:
++ 				                destructor_helpers_->push_back(helper_elem);
++						break;
++					case AnalyzerHelper::EOF_CODE:
++					        eof_helpers_->push_back(helper_elem);
++						break;
++					default:
++					        helpers_->push_back(helper_elem);
+ 					}
+-				else if ( helper_elem->helper_type() == 
+-				          AnalyzerHelper::CLEANUP_CODE)
+-					{
+-					if ( destructor_helper_ )
+-						{
+-						throw Exception(elem,
+-							"Repeated definition of %cleanup code");
+-						}
+-					destructor_helper_ = helper_elem;
+-					}
+-				else
+-					helpers_->push_back(helper_elem);
+ 				}
+ 				break;
+ 			case AnalyzerElement::FUNCTION:
+@@ -217,15 +213,19 @@
+ void AnalyzerDecl::GenInitCode(Output *out_cc)
+ 	{
+ 	TypeDecl::GenInitCode(out_cc);
+-	if ( constructor_helper_ )
+-		constructor_helper_->GenCode(0, out_cc, this);
++	foreach(i, AnalyzerHelperList, constructor_helpers_)
++		{
++		(*i)->GenCode(0, out_cc, this);
++		}
+ 	}
+ 
+ void AnalyzerDecl::GenCleanUpCode(Output *out_cc)
+ 	{
+ 	TypeDecl::GenCleanUpCode(out_cc);
+-	if ( destructor_helper_ )
+-		destructor_helper_->GenCode(0, out_cc, this);
++	foreach(i, AnalyzerHelperList, destructor_helpers_)
++		{
++		(*i)->GenCode(0, out_cc, this);
++		}
+ 	}
+ 
+ void AnalyzerDecl::GenStateVarDecls(Output *out_h)
+@@ -295,6 +295,7 @@
+ 			break;
+ 		case INIT_CODE:
+ 		case CLEANUP_CODE:
++		case EOF_CODE:
+ 			out = out_cc;
+ 			break;
+ 		}
+diff -urN bro-1.2.1-orig/src/binpac/pac_analyzer.h bro-1.2.1-ssl-binpac/src/binpac/pac_analyzer.h
+--- bro-1.2.1-orig/src/binpac/pac_analyzer.h	2006-07-26 15:02:39.000000000 -0700
++++ bro-1.2.1-ssl-binpac/src/binpac/pac_analyzer.h	2007-05-22 16:32:08.397926000 -0700
+@@ -76,8 +76,9 @@
+ 	AnalyzerHelperList *helpers_;
+ 	FunctionList *functions_;
+ 
+-	AnalyzerHelper *constructor_helper_;
+-	AnalyzerHelper *destructor_helper_;
++	AnalyzerHelperList *constructor_helpers_;
++	AnalyzerHelperList *destructor_helpers_;
++	AnalyzerHelperList *eof_helpers_;
+ };
+ 
+ class AnalyzerElement : public Object
+@@ -117,6 +118,7 @@
+ 		MEMBER_DECLS,
+ 		INIT_CODE,
+ 		CLEANUP_CODE,
++		EOF_CODE,
+ 	};
+ 	AnalyzerHelper(Type helper_type, EmbeddedCode *code)
+ 		: AnalyzerElement(HELPER),
+diff -urN bro-1.2.1-orig/src/binpac/pac_conn.cc bro-1.2.1-ssl-binpac/src/binpac/pac_conn.cc
+--- bro-1.2.1-orig/src/binpac/pac_conn.cc	2006-07-26 15:02:40.000000000 -0700
++++ bro-1.2.1-ssl-binpac/src/binpac/pac_conn.cc	2007-05-22 16:42:35.406135000 -0700
+@@ -97,6 +97,12 @@
+ 	out_cc->println("%s->%s();",
+ 	                env_->LValue(downflow_id),
+ 	                kFlowEOF);
++
++	foreach(i, AnalyzerHelperList, eof_helpers_)
++		{
++		(*i)->GenCode(0, out_cc, this);
++		}
++
+ 	out_cc->dec_indent();
+ 	
+ 	out_cc->println("}");
+diff -urN bro-1.2.1-orig/src/binpac/pac_flow.cc bro-1.2.1-ssl-binpac/src/binpac/pac_flow.cc
+--- bro-1.2.1-orig/src/binpac/pac_flow.cc	2006-10-12 14:13:12.000000000 -0700
++++ bro-1.2.1-ssl-binpac/src/binpac/pac_flow.cc	2007-05-22 16:43:55.997562000 -0700
+@@ -151,6 +151,11 @@
+ 	out_cc->inc_indent();
+ 	out_cc->println("{");
+ 
++	foreach(i, AnalyzerHelperList, eof_helpers_)
++		{
++		(*i)->GenCode(0, out_cc, this);
++		}
++
+ 	if ( dataunit_->type() == AnalyzerDataUnit::FLOWUNIT )
+ 		{
+ 		out_cc->println("%s->set_eof();", 
+diff -urN bro-1.2.1-orig/src/binpac/pac_parse.yy bro-1.2.1-ssl-binpac/src/binpac/pac_parse.yy
+--- bro-1.2.1-orig/src/binpac/pac_parse.yy	2006-10-12 14:13:12.000000000 -0700
++++ bro-1.2.1-ssl-binpac/src/binpac/pac_parse.yy	2007-05-22 16:56:09.280526000 -0700
+@@ -22,7 +22,7 @@
+ %token TOK_STATE TOK_ACTION TOK_WHEN TOK_HELPER 
+ %token TOK_DATAUNIT TOK_FLOWDIR TOK_WITHCONTEXT
+ %token TOK_LPB_EXTERN TOK_LPB_HEADER TOK_LPB_CODE 
+-%token TOK_LPB_MEMBER TOK_LPB_INIT TOK_LPB_CLEANUP 
++%token TOK_LPB_MEMBER TOK_LPB_INIT TOK_LPB_CLEANUP TOK_LPB_EOF
+ %token TOK_LPB TOK_RPB 
+ %token TOK_EMBEDDED_ATOM TOK_EMBEDDED_STRING
+ %token TOK_PAC_VAL TOK_PAC_SET TOK_PAC_TYPE TOK_PAC_TYPEOF TOK_PAC_CONST_DEF 
+@@ -795,6 +795,10 @@
+ 				{
+ 				$$ = new AnalyzerHelper(AnalyzerHelper::CLEANUP_CODE, $2);
+ 				}
++		|	TOK_LPB_EOF embedded_code TOK_RPB
++				{
++				$$ = new AnalyzerHelper(AnalyzerHelper::EOF_CODE, $2);
++				}
+ 		|	TOK_FLOWDIR '=' tok_id optargs ';'
+ 				{
+ 				$$ = new AnalyzerFlow((AnalyzerFlow::Direction) $1, $3, $4);
+diff -urN bro-1.2.1-orig/src/binpac/pac_scan.ll bro-1.2.1-ssl-binpac/src/binpac/pac_scan.ll
+--- bro-1.2.1-orig/src/binpac/pac_scan.ll	2006-07-26 15:02:40.000000000 -0700
++++ bro-1.2.1-ssl-binpac/src/binpac/pac_scan.ll	2007-05-22 16:55:19.349644000 -0700
+@@ -96,6 +96,10 @@
+ 				BEGIN(EC);
+ 				return TOK_LPB_MEMBER;
+ 				}
++<INITIAL>"%eof{"		{
++				BEGIN(EC);
++				return TOK_LPB_EOF;
++				}
+ <INITIAL>"%{"			{
+ 				BEGIN(EC);
+ 				return TOK_LPB;

tools/binpac/patches/binpac-patch-doc.txt

@@ -0,0 +1,87 @@
+binpac fixes
+----------------
+
+numbers of issues below correspond to the patch numbers
+
+(1) correct calculation of minimal header size in pac_expr.cc
+- problem: EXPR_CALLARGS and EXPR_CASE not considered for the calculation
+  of minimal header size
+- solution: added two cases in switch stmt of Expr::MinimalHeaderSize
+  for EXPR_CALLARGS and EXPR_CASE
+
+
+(2) ensure parsing of fields first referenced in a case expression or
+    let field with an &if attribute
+- problem: in cases where the if expression evaluates to false or the
+  proper case does not occur, fields get not parsed at all
+- solution: force evaluation of all IDs referenced in a let field with
+  if attribute or a case expression before the body of the corresponding
+  switch stmt or the if stmt
+- added public method Expr::ForceIDEval, properly called before
+  generating the code of a field with if attribute or the case expression
+
+
+(3) properly assert the use of fields with an if attribute
+- problem: the use of fields with an if attribute was not asserted in all
+  cases and asserted in the wrong way in some others due to the
+  corresponding BINPAC_ASSERT only called upon parsing the field
+- solution: perform BINPAC_ASSERT upon calling the fields accessor
+  function
+- moved BINPAC_ASSERT statement from LetField::GenEval to
+  Type::GenPubDecls
+
+
+(4) incremental input with records with a non-negative StaticSize
+- problem: incremental input with records with a StaticSize >= 0
+  cannot be performed due to necessary length attribute, leading to
+  an invalid call of GenBoundaryCheck in RecordType::DoGenParseCode
+- solution: added a check for incremental input in
+  RecordType::DoGenParseCode before calling GenBoundaryCheck
+
+
+(5) empty type with incremental input
+- problem: with an empty type and incremental input, although the
+  Parse function is created, it is never called, leading to problems,
+  if additional actions are to be performed when encountering that
+  empty type
+- solution: generate call to Parse of empty type in Type::GenParseBuffer
+
+
+(6) parsing loop in flow ParseBuffer (while(true))
+- problem: while(true) leads to problems after parsing of a type is
+  complete; at this time, it is unexpected that parsing continues, even
+  if no data is available in the flow buffer
+- solution: check if data is available before starting a new parsing
+  cycle
+- added a method data_available to FlowBuffer
+- changed while(true) in FlowDecl::GenCodeFlowUnit to
+  while(flow_buffer_->data_available())
+
+
+(7) initialization of flow buffer in CaseType with bufferable fields
+    in cases
+- problem: initialization of buffer occurs in every Parse call,
+  regardless if it was initialized before or not; initialization
+  is correct only on first such occurence
+- solution: check to buffer_state is to be created always when
+  buffering_state_id is in environment in Type::GenBufferConfig
+- changed condition from buffering_state_var_field_ to
+  env->GetDataType(buffering_state_id)
+
+
+(8) allowing init and cleanup code to be redefined, as well as addition
+    of code to FlowEOF calls in analyzer and flow
+- problem 1: when refining an analyzer or flow definition, additional
+  init and cleanup code was not allowed, if these were already defined
+  before; this leads to problems when adding new members, as these
+  cannot be initialized and destroyed properly
+- solution: allow init and cleanup code to be specified more than once
+- changed deifnitions and usage of constructor_helper and
+  destructor_helper to allow for lists of constructor and destructor
+  helpers (similar to member declarations) in pac_analyzer.h and
+  pac_analyzer.cc
+- problem 2: in some cases, it is desirable to execute code when
+  encountering the end of the input stream, which is not possible in
+  binpac
+- solution: added a %eof binpac primitive similar to %init, which adds
+  code to the FlowEOF function of an analyzer or a flow

tools/binpac/patches/nadi-bittorrent.patch

@@ -0,0 +1,172 @@
+Index: pac_type.h
+===================================================================
+--- pac_type.h	(revision 4130)
++++ pac_type.h	(working copy)
+@@ -78,12 +78,6 @@
+ 	string EvalByteOrder(Output *out_cc, Env *env) const;
+ 
+ 	virtual string EvalMember(const ID *member_id) const;
+-#if 0
+-	// member_env() is used for finding a member of the type.
+-	// Thus member_env() of a ParameterizedType should return 
+-	// ReferredDataType()->env()
+-	// virtual Env *member_env() const;
+-#endif
+ 
+ 	// The variable defined by the type
+ 	const ID *value_var() const		{ return value_var_; }
+@@ -223,6 +217,8 @@
+ 
+ 	virtual bool ByteOrderSensitive() const = 0;
+ 
++	bool NeedsBufferingStateVar() const;
++
+ 	void GenBufferingLoop(Output* out_cc, Env* env, int flags);
+ 	void GenParseBuffer(Output* out_cc, Env* env, int flags);
+ 	void GenParseCode2(Output* out_cc, Env* env, const DataPtr& data, int flags);
+Index: lib/binpac_buffer.h
+===================================================================
+--- lib/binpac_buffer.h	(revision 4130)
++++ lib/binpac_buffer.h	(working copy)
+@@ -24,18 +24,18 @@
+ 	void DiscardData();
+ 
+ 	// Whether there is enough data for the frame
+-	bool ready() const{ return message_complete_; }
++	bool ready() const{ return message_complete_ || mode_ == UNKNOWN_MODE; }
+ 
+ 	inline const_byteptr begin() const
+ 		{
+-		BINPAC_ASSERT(message_complete_);
++		BINPAC_ASSERT(ready());
+ 		return ( buffer_n_ == 0 ) ? 
+ 			orig_data_begin_ : buffer_;
+ 		}
+ 
+ 	inline const_byteptr end() const
+ 		{
+-		BINPAC_ASSERT(message_complete_);
++		BINPAC_ASSERT(ready());
+ 		if ( buffer_n_ == 0 )
+ 			{
+ 			BINPAC_ASSERT(frame_length_ >= 0);
+Index: pac_type.cc
+===================================================================
+--- pac_type.cc	(revision 4130)
++++ pac_type.cc	(working copy)
+@@ -285,9 +285,8 @@
+ 			parsing_complete_var, extern_type_bool->Clone());
+ 		parsing_complete_var_field_->Prepare(env);
+ 
+-		if ( ( buffer_mode() == BUFFER_BY_LENGTH ||
+-		       buffer_mode() == BUFFER_BY_LINE ) &&
+-		       ! env->GetDataType(buffering_state_id) )
++		if ( NeedsBufferingStateVar() && 
++		     !env->GetDataType(buffering_state_id) )
+ 			{
+ 			buffering_state_var_field_ = new PrivVarField(
+ 				buffering_state_id->clone(), 
+@@ -387,17 +386,17 @@
+ 			break;
+ 
+ 		case BUFFER_BY_LENGTH:
+-			if ( buffering_state_var_field_ )
+-				{
+-				out_cc->println("if ( %s == 0 )",
+-					env->RValue(buffering_state_id));
+-				out_cc->inc_indent();
+-				out_cc->println("{");
+-				}
++			if ( !NeedsBufferingStateVar() )
++				break;
+ 
++			ASSERT(env->GetDataType(buffering_state_id));
++			out_cc->println("if ( %s == 0 )",
++				env->RValue(buffering_state_id));
++			out_cc->inc_indent();
++			out_cc->println("{");
++
+ 			if ( attr_length_expr_ )
+ 				{
+-				// frame_buffer_arg = attr_length_expr_->EvalExpr(out_cc, env);
+ 				frame_buffer_arg = strfmt("%d", InitialBufferLength());
+ 				}
+ 			else if ( attr_restofflow_ )
+@@ -407,7 +406,7 @@
+ 				}
+ 			else
+ 				{
+-				frame_buffer_arg = strfmt("%d", InitialBufferLength());
++				ASSERT(0);
+ 				}
+ 
+ 			out_cc->println("%s->NewFrame(%s, %s);",
+@@ -415,16 +414,14 @@
+ 				frame_buffer_arg.c_str(),
+ 				attr_chunked() ? "true" : "false");
+ 
+-			if ( buffering_state_var_field_ )
+-				{
+-				out_cc->println("%s = 1;",
+-					env->LValue(buffering_state_id));
+-				out_cc->println("}");
+-				out_cc->dec_indent();
+-				}
++			out_cc->println("%s = 1;",
++				env->LValue(buffering_state_id));
++			out_cc->println("}");
++			out_cc->dec_indent();
+ 			break;
+ 
+ 		case BUFFER_BY_LINE:
++			ASSERT(env->GetDataType(buffering_state_id));
+ 			out_cc->println("if ( %s == 0 )",
+ 				env->RValue(buffering_state_id));
+ 			out_cc->inc_indent();
+@@ -890,6 +887,25 @@
+ 	return ! attr_byteorder_expr() && ByteOrderSensitive(); 
+ 	}
+ 
++bool Type::NeedsBufferingStateVar() const
++	{
++	if ( !incremental_input() )
++		return false;
++	switch ( buffer_mode() )
++		{
++		case BUFFER_NOTHING:
++		case NOT_BUFFERABLE:
++			return false;
++		case BUFFER_BY_LINE:
++			return true;
++		case BUFFER_BY_LENGTH:
++			return ( attr_length_expr_ || attr_restofflow_ );
++		default:
++			ASSERT(0);
++			return false;
++		}
++	}
++
+ bool Type::DoTraverse(DataDepVisitor *visitor)
+ 	{
+ 	foreach (i, FieldList, fields_)
+Index: pac_flow.cc
+===================================================================
+--- pac_flow.cc	(revision 4130)
++++ pac_flow.cc	(working copy)
+@@ -224,15 +224,13 @@
+ 	out_cc->println("catch ( Exception const &e )");
+ 	out_cc->inc_indent();
+ 	out_cc->println("{");
+-	out_cc->println("DEBUG_MSG(\"%%.6f binpac exception: %%s\\n\", network_time(), e.c_msg());");
+ 	GenCleanUpCode(out_cc);
+ 	if ( dataunit_->type() == AnalyzerDataUnit::FLOWUNIT )
+ 		{
+ 		out_cc->println("%s->DiscardData();",
+ 			env_->LValue(flow_buffer_id));
+-		out_cc->println("BINPAC_ASSERT(!%s->ready());",
+-			env_->RValue(flow_buffer_id));
+ 		}
++	out_cc->println("throw e;");
+ 	out_cc->println("}");
+ 	out_cc->dec_indent();
+ 

tools/binpac/src/Makefile.am

@@ -0,0 +1,62 @@
+## Process this file with automake to produce Makefile.in
+
+AM_YFLAGS = -d -t -v
+AM_CPPFLAGS = -W -Wall -Wno-unused
+
+noinst_PROGRAMS = binpac
+
+binpac_SOURCES = \
+	pac_scan.ll pac_parse.yy \
+	pac_action.cc \
+	pac_analyzer.cc \
+	pac_array.cc \
+	pac_attr.cc \
+	pac_btype.cc \
+	pac_case.cc \
+	pac_conn.cc \
+	pac_context.cc \
+	pac_cstr.cc \
+	pac_datadep.cc \
+	pac_dataptr.cc \
+	pac_dataunit.cc \
+	pac_decl.cc \
+	pac_embedded.cc \
+	pac_enum.cc \
+	pac_expr.cc \
+	pac_exttype.cc \
+	pac_field.cc \
+	pac_flow.cc \
+	pac_func.cc \
+	pac_id.cc \
+	pac_inputbuf.cc \
+	pac_let.cc \
+	pac_param.cc \
+	pac_paramtype.cc \
+	pac_primitive.cc \
+	pac_record.cc \
+	pac_redef.cc \
+	pac_regex.cc \
+	pac_state.cc \
+	pac_strtype.cc \
+	pac_type.cc \
+	pac_typedecl.cc \
+	pac_withinput.cc \
+	pac_output.cc pac_utils.cc pac_exception.cc \
+	pac_main.cc \
+	pac_action.h pac_analyzer.h pac_array.h pac_attr.h pac_btype.h \
+	pac_case.h pac_cclass.h pac_common.h pac_conn.h pac_context.h \
+	pac_cstr.h pac_ctype.h pac_datadep.h pac_dataptr.h pac_dataunit.h \
+	pac_dbg.h pac_decl-inl.h pac_decl.h pac_embedded.h pac_enum.h \
+	pac_exception.h pac_expr.h pac_exttype.h pac_field.h pac_flow.h \
+	pac_func.h pac_id.h pac_inputbuf.h pac_let.h pac_number.h \
+	pac_output.h pac_param.h pac_paramtype.h pac_parse.h pac_primitive.h \
+	pac_record.h pac_redef.h pac_regex.h pac_state.h pac_strtype.h \
+	pac_type.h pac_typedecl.h pac_utils.h pac_varfield.h pac_withinput.h
+
+EXTRA_DIST = pac_expr.def pac_type.def pac_externtype.def
+
+DISTCLEANFILES = pac_parse.cc pac_parse.h pac_scan.cc y.output
+
+# Manual rules below:
+
+pac_scan.o:	pac_parse.h

tools/binpac/src/pac_action.cc

@@ -0,0 +1,119 @@
+#include "pac_embedded.h"
+#include "pac_exception.h"
+#include "pac_id.h"
+#include "pac_output.h"
+#include "pac_type.h"
+#include "pac_typedecl.h"
+#include "pac_utils.h"
+
+#include "pac_action.h"
+
+AnalyzerAction::AnalyzerAction(ID *action_id, 
+	                       When when, 
+	                       ActionParam *param, 
+	                       EmbeddedCode *code)
+	: AnalyzerElement(ACTION),
+	  action_id_(action_id),
+	  when_(when),
+	  param_(param),
+	  code_(code),
+	  analyzer_(0) 
+	{
+	}
+
+AnalyzerAction::~AnalyzerAction()
+	{
+	delete action_id_;
+	delete param_;
+	delete code_;
+	}
+
+string AnalyzerAction::action_function() const
+	{
+	return strfmt("Action_%s", action_id_->Name());
+	}
+
+void AnalyzerAction::InstallHook(AnalyzerDecl *analyzer)
+	{
+	ASSERT(0);
+	analyzer_ = analyzer;
+	// param_->MainDataType()->InstallAction(this);
+	}
+
+void AnalyzerAction::GenCode(Output *out_h, Output *out_cc, AnalyzerDecl *decl)
+	{
+	Env action_func_env(decl->env(), this);
+	action_func_env.AddID(param_->id(), 
+	                      TEMP_VAR, 
+	                      param_->DataType());
+	action_func_env.SetEvaluated(param_->id());
+
+	string action_func_proto = 
+		strfmt("%s(%s)", 
+		       action_function().c_str(), 
+		       ParamDecls(&action_func_env).c_str());
+
+	out_h->println("void %s;", action_func_proto.c_str());
+
+	out_cc->println("void %s::%s", 
+	                decl->class_name().c_str(), 
+	                action_func_proto.c_str());
+	out_cc->inc_indent();
+	out_cc->println("{");
+
+	code_->GenCode(out_cc, &action_func_env);
+
+	out_cc->println("");
+	out_cc->println("}");
+	out_cc->dec_indent();
+	out_cc->println("");
+	}
+
+string AnalyzerAction::ParamDecls(Env *env) const
+	{
+	return param_->DeclStr(env);
+	}
+
+Type *ActionParam::MainDataType() const
+	{
+	// Note: this is not equal to DataType()
+	Type *main_type = TypeDecl::LookUpType(type()->type_id());
+
+	if ( ! main_type )
+		{
+		throw Exception(type()->type_id(), 
+		                "type not defined");
+		}
+
+	return main_type;
+	}
+
+Type *ActionParam::DataType() const
+	{
+	Type *main_type = MainDataType();
+
+	if ( ! type()->field_id() )
+		{
+		return main_type;
+		}
+	else
+		{
+		Type *member_type = 
+			main_type->MemberDataType(type()->field_id());
+		if ( ! member_type )
+			{
+			throw Exception(type()->field_id(),
+				fmt("cannot find member type for `%s.%s'",
+				    type()->type_id()->Name(),
+				    type()->field_id()->Name()));
+			}
+		return member_type;
+		}
+	}
+
+string ActionParam::DeclStr(Env *env) const
+	{
+	return strfmt("%s %s", 
+	              DataType()->DataTypeStr().c_str(), 
+	              env->LValue(id()));
+	}

tools/binpac/src/pac_action.h

@@ -0,0 +1,74 @@
+#ifndef pac_action_h
+#define pac_action_h
+
+// Classes representing analyzer actions.
+
+#include "pac_common.h"
+#include "pac_analyzer.h"
+
+class AnalyzerAction : public AnalyzerElement
+{
+public:
+	enum When { BEFORE, AFTER };
+
+	AnalyzerAction(ID *action_id, 
+	               When when, 
+	               ActionParam *param, 
+	               EmbeddedCode *code);
+
+	~AnalyzerAction();
+
+	When when() const		{ return when_; }
+	ActionParam *param() const	{ return param_; }
+	AnalyzerDecl *analyzer() const	{ return analyzer_; }
+	string action_function() const;
+
+	// Generate function prototype and code for the action
+	void GenCode(Output *out_h, Output *out_cc, AnalyzerDecl *decl);
+
+	// Install the hook at the corresponding data type parsing
+	// function to invoke the action.
+	void InstallHook(AnalyzerDecl *analyzer);
+
+private:
+	string ParamDecls(Env *env) const;
+
+	ID *action_id_;
+	When when_;
+	ActionParam *param_;
+	EmbeddedCode *code_;
+	AnalyzerDecl *analyzer_;
+};
+
+class ActionParam
+{
+public:
+	ActionParam(const ID *id, ActionParamType *type)
+		: id_(id), type_(type) {}
+
+	const ID *id() const		{ return id_; }
+	ActionParamType *type() const	{ return type_; }
+
+	Type *MainDataType() const;
+	Type *DataType() const;
+	string DeclStr(Env *env) const;
+
+private:
+	const ID *id_;
+	ActionParamType *type_;
+};
+
+class ActionParamType
+{
+public:
+	ActionParamType(const ID *type_id, const ID *field_id = 0)
+		: type_id_(type_id), field_id_(field_id) {}
+
+	const ID *type_id() const	{ return type_id_; }
+	const ID *field_id() const	{ return field_id_; }
+
+protected:
+	const ID *type_id_, *field_id_;
+};
+
+#endif  // pac_action_h

tools/binpac/src/pac_analyzer.cc

@@ -0,0 +1,358 @@
+#include "pac_action.h"
+#include "pac_context.h"
+#include "pac_embedded.h"
+#include "pac_exception.h"
+#include "pac_expr.h"
+#include "pac_flow.h"
+#include "pac_func.h"
+#include "pac_output.h"
+#include "pac_param.h"
+#include "pac_paramtype.h"
+#include "pac_state.h"
+#include "pac_type.h"
+#include "pac_varfield.h"
+
+#include "pac_analyzer.h"
+
+AnalyzerDecl::AnalyzerDecl(ID *id, 
+                           DeclType decl_type,
+                           ParamList *params)
+	: TypeDecl(id, params, new DummyType())
+	{
+	decl_type_ = decl_type;
+
+	statevars_ = new StateVarList();
+	actions_ = new AnalyzerActionList();
+	helpers_ = new AnalyzerHelperList();
+	functions_ = new FunctionList();
+
+	constructor_helpers_ = new AnalyzerHelperList();
+	destructor_helpers_ = new AnalyzerHelperList();
+	eof_helpers_ = new AnalyzerHelperList();
+
+	SetAnalyzerContext();
+
+	env_ = 0;
+	}
+
+AnalyzerDecl::~AnalyzerDecl()
+	{
+	delete_list(StateVarList, statevars_);
+	delete_list(AnalyzerActionList, actions_);
+	delete_list(AnalyzerHelperList, helpers_);
+	delete_list(FunctionList, functions_);
+	delete_list(ParamList, params_);
+	delete_list(AnalyzerHelperList, constructor_helpers_);
+	delete_list(AnalyzerHelperList, destructor_helpers_);
+	delete_list(AnalyzerHelperList, eof_helpers_);
+	}
+
+void AnalyzerDecl::AddElements(AnalyzerElementList *elemlist)
+	{
+	ASSERT(! env_);
+	foreach(i, AnalyzerElementList, elemlist)
+		{
+		AnalyzerElement *elem = *i;
+		switch ( elem->type() )
+			{
+			case AnalyzerElement::STATE:
+				{
+				ASSERT(0);
+				AnalyzerState *state_elem = 
+					(AnalyzerState *) elem;
+				statevars_->insert(
+					statevars_->end(),
+					state_elem->statevars()->begin(),
+					state_elem->statevars()->end());
+				}
+				break;
+			case AnalyzerElement::ACTION:
+				{
+				ASSERT(0);
+				AnalyzerAction *action_elem = 
+					(AnalyzerAction *) elem;
+				actions_->push_back(action_elem);
+				}
+				break;
+			case AnalyzerElement::HELPER:
+				{
+				AnalyzerHelper *helper_elem = 
+					(AnalyzerHelper *) elem;
+
+				switch ( helper_elem->helper_type() )
+				        {
+					case AnalyzerHelper::INIT_CODE:
+					        constructor_helpers_->push_back(helper_elem);
+						break;
+					case AnalyzerHelper::CLEANUP_CODE:
+ 				                destructor_helpers_->push_back(helper_elem);
+						break;
+					case AnalyzerHelper::EOF_CODE:
+					        eof_helpers_->push_back(helper_elem);
+						break;
+					default:
+					        helpers_->push_back(helper_elem);
+					}
+				}
+				break;
+			case AnalyzerElement::FUNCTION:
+				{
+				AnalyzerFunction *func_elem = 
+					(AnalyzerFunction *) elem;
+				Function *func = func_elem->function();
+				func->set_analyzer_decl(this);
+				functions_->push_back(func);
+				}
+				break;
+			case AnalyzerElement::FLOW:
+				{
+				AnalyzerFlow *flow_elem = 
+					(AnalyzerFlow *) elem;
+				ProcessFlowElement(flow_elem);
+				}
+				break;
+			case AnalyzerElement::DATAUNIT:
+				{
+				AnalyzerDataUnit *dataunit_elem = 
+					(AnalyzerDataUnit *) elem;
+				ProcessDataUnitElement(dataunit_elem);
+				}
+				break;
+			}
+		}
+	}
+
+string AnalyzerDecl::class_name() const
+	{ 
+	return id_->Name(); 
+	}
+
+void AnalyzerDecl::Prepare()
+	{
+	TypeDecl::Prepare();
+
+	ASSERT(statevars_->empty());
+	ASSERT(actions_->empty());
+
+	foreach(i, FunctionList, functions_)
+		{
+		Function *function = *i;
+		function->Prepare(env_);
+		}
+	foreach(i, StateVarList, statevars_)
+		{
+		StateVar *statevar = *i;
+		env_->AddID(statevar->id(), STATE_VAR, statevar->type());
+		}
+	foreach(i, AnalyzerActionList, actions_)
+		{
+		AnalyzerAction *action = *i;
+		action->InstallHook(this);
+		}
+	}
+
+void AnalyzerDecl::GenForwardDeclaration(Output* out_h)
+	{
+	out_h->println("class %s;", class_name().c_str());
+	foreach(i, FunctionList, functions_)
+		{
+		Function *function = *i;
+		function->GenForwardDeclaration(out_h);
+		}
+	}
+
+void AnalyzerDecl::GenActions(Output *out_h, Output *out_cc)
+	{
+	foreach(i, AnalyzerActionList, actions_)
+		{
+		(*i)->GenCode(out_h, out_cc, this);
+		}
+	}
+
+void AnalyzerDecl::GenHelpers(Output *out_h, Output *out_cc)
+	{
+	foreach(i, AnalyzerHelperList, helpers_)
+		{
+		(*i)->GenCode(out_h, out_cc, this);
+		}
+	}
+
+void AnalyzerDecl::GenPubDecls(Output *out_h, Output *out_cc)
+	{
+	TypeDecl::GenPubDecls(out_h, out_cc);
+
+	GenProcessFunc(out_h, out_cc);
+	GenGapFunc(out_h, out_cc);
+	GenEOFFunc(out_h, out_cc);
+	out_h->println("");
+
+	if ( ! functions_->empty() )
+		{
+		out_h->println("// Functions");
+		GenFunctions(out_h, out_cc);
+		out_h->println("");
+		}
+
+	// TODO: export public state variables
+	}
+
+void AnalyzerDecl::GenPrivDecls(Output *out_h, Output *out_cc)
+	{
+	TypeDecl::GenPrivDecls(out_h, out_cc);
+
+	if ( ! helpers_->empty() )
+		{
+		out_h->println("");
+		out_h->println("// Additional members");
+		GenHelpers(out_h, out_cc);
+		}
+
+	// TODO: declare state variables
+	}
+
+void AnalyzerDecl::GenInitCode(Output *out_cc)
+	{
+	TypeDecl::GenInitCode(out_cc);
+	foreach(i, AnalyzerHelperList, constructor_helpers_)
+		{
+		(*i)->GenCode(0, out_cc, this);
+		}
+	}
+
+void AnalyzerDecl::GenCleanUpCode(Output *out_cc)
+	{
+	TypeDecl::GenCleanUpCode(out_cc);
+	foreach(i, AnalyzerHelperList, destructor_helpers_)
+		{
+		(*i)->GenCode(0, out_cc, this);
+		}
+	}
+
+void AnalyzerDecl::GenStateVarDecls(Output *out_h)
+	{
+	foreach(i, StateVarList, statevars_)
+		{
+		StateVar *var = *i;
+		var->GenDecl(out_h, env_);
+		}
+	}
+
+void AnalyzerDecl::GenStateVarSetFunctions(Output *out_h)
+	{
+	foreach(i, StateVarList, statevars_)
+		{
+		StateVar *var = *i;
+		var->GenSetFunction(out_h, env_);
+		}
+	}
+
+void AnalyzerDecl::GenStateVarInitCode(Output *out_cc)
+	{
+	foreach(i, StateVarList, statevars_)
+		{
+		StateVar *var = *i;
+		var->GenInitCode(out_cc, env_);
+		}
+	}
+
+void AnalyzerDecl::GenStateVarCleanUpCode(Output *out_cc)
+	{
+	foreach(i, StateVarList, statevars_)
+		{
+		StateVar *var = *i;
+		var->GenCleanUpCode(out_cc, env_);
+		}
+	}
+
+void AnalyzerDecl::GenFunctions(Output *out_h, Output *out_cc)
+	{
+	foreach(i, FunctionList, functions_)
+		{
+		Function *function = *i;
+		function->GenCode(out_h, out_cc);
+		}
+	}
+
+AnalyzerState::~AnalyzerState()
+	{
+	// Note: do not delete elements of statevars_, because they
+	// are referenced by the AnalyzerDecl.
+	delete statevars_;
+	}
+
+AnalyzerHelper::~AnalyzerHelper()
+	{
+	delete code_;
+	}
+
+void AnalyzerHelper::GenCode(Output *out_h, Output *out_cc, AnalyzerDecl *decl)
+	{
+	Output *out = 0;
+	switch ( helper_type_ )
+		{
+		case MEMBER_DECLS:
+			out = out_h;
+			break;
+		case INIT_CODE:
+		case CLEANUP_CODE:
+		case EOF_CODE:
+			out = out_cc;
+			break;
+		}
+	ASSERT(out);
+	code()->GenCode(out, decl->env());
+	}
+
+FlowField::FlowField(ID *flow_id, ParameterizedType *flow_type)
+	: Field(FLOW_FIELD, 
+		TYPE_NOT_TO_BE_PARSED | CLASS_MEMBER | PUBLIC_READABLE, 
+		flow_id, flow_type)
+	{
+	}
+
+void FlowField::GenInitCode(Output *out_cc, Env *env)
+	{
+	type_->GenPreParsing(out_cc, env);
+	}
+
+AnalyzerFlow::AnalyzerFlow(Direction dir, ID *type_id, ExprList *params)
+	: AnalyzerElement(FLOW),
+	  dir_(dir),
+	  type_id_(type_id)
+	{
+	if ( ! params )
+		params = new ExprList();
+
+	// Add "this" to the list of params
+	params->insert(params->begin(), new Expr(this_id->clone()));
+
+	ID *flow_id = ((dir == UP) ? upflow_id : downflow_id)->clone();
+
+	ParameterizedType *flow_type = new ParameterizedType(type_id_, params);
+
+	flow_field_ = new FlowField(flow_id, flow_type);
+
+	flow_decl_ = 0;
+	}
+
+AnalyzerFlow::~AnalyzerFlow()
+	{
+	delete flow_field_;
+	}
+
+FlowDecl *AnalyzerFlow::flow_decl()
+	{
+	DEBUG_MSG("Getting flow_decl for %s\n", type_id_->Name());
+	if ( ! flow_decl_ )
+		{
+		Decl *decl = Decl::LookUpDecl(type_id_);
+		if ( decl && decl->decl_type() == Decl::FLOW )
+			flow_decl_ = static_cast<FlowDecl *>(decl);
+		if ( ! flow_decl_ )
+			{
+			throw Exception(this, 
+			                "cannot find the flow declaration");
+			}
+		}
+	return flow_decl_;
+	}

tools/binpac/src/pac_analyzer.h

@@ -0,0 +1,168 @@
+#ifndef pac_analyzer_h
+#define pac_analyzer_h
+
+#include "pac_common.h"
+#include "pac_field.h"
+#include "pac_typedecl.h"
+
+class AnalyzerElement;
+class AnalyzerState;
+class AnalyzerAction;	// defined in pac_action.h
+class AnalyzerHelper;
+class AnalyzerFlow;
+class AnalyzerDataUnit;
+class AnalyzerFunction;
+class ConnDecl;
+class FlowDecl;
+typedef vector<AnalyzerHelper *> AnalyzerHelperList;
+typedef vector<Function *> FunctionList;
+
+class AnalyzerDecl : public TypeDecl
+{
+public:
+	AnalyzerDecl(ID *id, DeclType decl_type, ParamList *params);
+	~AnalyzerDecl();
+
+	void AddElements(AnalyzerElementList *elemlist);
+
+	void Prepare();
+	void GenForwardDeclaration(Output *out_h);
+	// void GenCode(Output *out_h, Output *out_cc);
+
+	void GenInitCode(Output *out_cc);
+	void GenCleanUpCode(Output *out_cc);
+
+	string class_name() const;
+	// string cookie_name() const;
+
+protected:
+	virtual void ProcessFlowElement(AnalyzerFlow *flow_elem) = 0;
+	virtual void ProcessDataUnitElement(AnalyzerDataUnit *dataunit_elem) = 0;
+
+	// Generate public/private declarations for member functions and 
+	// variables
+	void GenPubDecls(Output *out_h, Output *out_cc);
+	void GenPrivDecls(Output *out_h, Output *out_cc);
+
+	// Generate the NewData() function
+	virtual void GenProcessFunc(Output *out_h, Output *out_cc) = 0;
+
+	// Generate the NewGap() function
+	virtual void GenGapFunc(Output *out_h, Output *out_cc) = 0;
+
+	// Generate the FlowEOF() function
+	virtual void GenEOFFunc(Output *out_h, Output *out_cc) = 0;
+
+	// Generate the functions
+	void GenFunctions(Output *out_h, Output *out_cc);
+
+	// Generate the action functions
+	void GenActions(Output *out_h, Output *out_cc);
+
+	// Generate the helper code segments
+	void GenHelpers(Output *out_h, Output *out_cc);
+
+	// Generate declarations for state variables and their set functions
+	void GenStateVarDecls(Output *out_h);
+	void GenStateVarSetFunctions(Output *out_h);
+
+	// Generate code for initializing and cleaning up (including
+	// memory de-allocating) state variables
+	void GenStateVarInitCode(Output *out_cc);
+	void GenStateVarCleanUpCode(Output *out_cc);
+
+	StateVarList *statevars_;
+	AnalyzerActionList *actions_;
+	AnalyzerHelperList *helpers_;
+	FunctionList *functions_;
+
+	AnalyzerHelperList *constructor_helpers_;
+	AnalyzerHelperList *destructor_helpers_;
+	AnalyzerHelperList *eof_helpers_;
+};
+
+class AnalyzerElement : public Object
+{
+public:
+	enum ElementType { STATE, ACTION, FUNCTION, HELPER, FLOW, DATAUNIT };
+	AnalyzerElement(ElementType type)
+		: type_(type) {}
+	virtual ~AnalyzerElement() {}
+
+	ElementType type() const	{ return type_; }
+
+private:
+	ElementType type_;
+};
+
+// A collection of variables representing analyzer states.
+class AnalyzerState : public AnalyzerElement
+{
+public:
+	AnalyzerState(StateVarList *statevars)
+		: AnalyzerElement(STATE),
+		  statevars_(statevars) {}
+	~AnalyzerState();
+
+	StateVarList *statevars() const	{ return statevars_; }
+
+private:
+	StateVarList *statevars_;
+};
+
+// A collection of embedded C++ code
+class AnalyzerHelper : public AnalyzerElement
+{
+public:
+	enum Type {
+		MEMBER_DECLS,
+		INIT_CODE,
+		CLEANUP_CODE,
+		EOF_CODE,
+	};
+	AnalyzerHelper(Type helper_type, EmbeddedCode *code)
+		: AnalyzerElement(HELPER),
+		  helper_type_(helper_type), 
+		  code_(code) {}
+	~AnalyzerHelper();
+
+	Type helper_type() const	{ return helper_type_; }
+
+	void GenCode(Output *out_h, Output *out_cc, AnalyzerDecl *decl);
+
+	EmbeddedCode *code() const	{ return code_; }
+
+private:
+	Type helper_type_;
+	EmbeddedCode *code_;
+};
+
+// The type and parameters of (uni-directional) flows of a connection.
+
+class FlowField : public Field
+{
+public:
+	FlowField(ID *flow_id, ParameterizedType *flow_type);
+	void GenInitCode(Output *out, Env *env);
+};
+
+class AnalyzerFlow : public AnalyzerElement
+{
+public:
+	enum Direction { UP, DOWN };
+	AnalyzerFlow(Direction dir, ID *type_id, ExprList *params);
+	~AnalyzerFlow();
+
+	Direction dir() const		{ return dir_; }
+	FlowField *flow_field() const	{ return flow_field_; }
+
+	FlowDecl *flow_decl();
+
+private:
+	Direction dir_;
+	ID *type_id_;
+	FlowField *flow_field_;
+	FlowDecl *flow_decl_;
+};
+
+#endif  // pac_analyzer_h

tools/binpac/src/pac_array.cc

@@ -0,0 +1,700 @@
+#include "pac_attr.h"
+#include "pac_dataptr.h"
+#include "pac_exception.h"
+#include "pac_expr.h"
+#include "pac_exttype.h"
+#include "pac_id.h"
+#include "pac_number.h"
+#include "pac_output.h"
+#include "pac_utils.h"
+#include "pac_varfield.h"
+
+#include "pac_array.h"
+
+ArrayType::ArrayType(Type *elemtype, Expr *length)
+	: Type(ARRAY), elemtype_(elemtype), length_(length)
+	{
+	init();
+
+	switch ( elemtype_->tot() )
+		{
+		case BUILTIN:
+		case PARAMETERIZED:
+		case STRING:
+		case EXTERN:
+			break;
+
+		case ARRAY:
+		case CASE:
+		case DUMMY:
+		case EMPTY:
+		case RECORD:
+		case UNDEF:
+			ASSERT(0);
+			break;
+		}
+	}
+
+void ArrayType::init()
+	{
+	arraylength_var_field_ = 0;
+	elem_it_var_field_ = 0;
+	elem_var_field_ = 0;
+	elem_dataptr_var_field_ = 0;
+	elem_input_var_field_ = 0;
+
+	elem_dataptr_until_expr_ = 0;
+
+	end_of_array_loop_label_ = "@@@";
+
+	vector_str_ = strfmt("vector<%s>", elemtype_->DataTypeStr().c_str());
+
+	datatype_str_ = strfmt("%s *", vector_str_.c_str());
+
+	attr_generic_until_expr_ = 0;
+	attr_until_element_expr_ = 0;
+	attr_until_input_expr_ = 0;
+	}
+
+ArrayType::~ArrayType()
+	{
+	delete arraylength_var_field_;
+	delete elem_it_var_field_;
+	delete elem_var_field_;
+	delete elem_dataptr_var_field_;
+	delete elem_input_var_field_;
+
+	delete elem_dataptr_until_expr_;
+	}
+
+Type *ArrayType::DoClone() const
+	{
+	Type *elemtype = elemtype_->Clone();
+	if ( ! elemtype )
+		return 0;
+	return new ArrayType(elemtype, length_);
+	}
+
+bool ArrayType::DefineValueVar() const
+	{
+	return true;
+	}
+
+string ArrayType::DataTypeStr() const
+	{
+	return datatype_str_;
+	}
+
+Type *ArrayType::ElementDataType() const
+	{
+	return elemtype_;
+	}
+
+string ArrayType::EvalElement(const string &array, const string &index) const
+	{
+	return strfmt("(*(%s))[%s]", array.c_str(), index.c_str());
+	}
+
+const ID *ArrayType::arraylength_var() const
+	{
+	return arraylength_var_field_ ? arraylength_var_field_->id() : 0;
+	}
+
+const ID *ArrayType::elem_it_var() const
+	{
+	return elem_it_var_field_ ? elem_it_var_field_->id() : 0;
+	}
+
+const ID *ArrayType::elem_var() const
+	{
+	return elem_var_field_ ? elem_var_field_->id() : 0;
+	}
+
+const ID *ArrayType::elem_dataptr_var() const
+	{
+	return elem_dataptr_var_field_ ? elem_dataptr_var_field_->id() : 0;
+	}
+
+const ID *ArrayType::elem_input_var() const
+	{
+	return elem_input_var_field_ ? elem_input_var_field_->id() : 0;
+	}
+
+void ArrayType::ProcessAttr(Attr *a)
+	{
+	Type::ProcessAttr(a);
+
+	switch ( a->type() )
+		{
+		case ATTR_RESTOFDATA:
+			{
+			if ( elemtype_->StaticSize(env()) != 1 )
+				{
+				throw Exception(elemtype_, 
+					"&restofdata can be applied"
+					" to only byte arrays");
+				}
+			if ( length_ )
+				{
+				throw Exception(length_, 
+					"&restofdata cannot be applied"
+					" to arrays with specified length");
+				}
+			attr_restofdata_ = true;
+			// As the array automatically extends to the end of 
+			// data, we do not have to check boundary.
+			SetBoundaryChecked();
+			}
+			break;
+
+		case ATTR_RESTOFFLOW:
+			attr_restofflow_ = true;
+			// TODO: handle &restofflow
+			break;
+
+		case ATTR_UNTIL:
+			{
+			bool ref_element = a->expr()->HasReference(element_macro_id);
+			bool ref_input = a->expr()->HasReference(input_macro_id);
+			if ( ref_element && ref_input )
+				{
+				throw Exception(a->expr(), 
+					"cannot reference both $element and $input "
+					"in the same &until---please separate them.");
+				}
+
+			if ( ref_element )
+				{
+				if ( attr_until_element_expr_ )
+					{
+					throw Exception(a->expr(), 
+						"multiple &until on $element");
+					}
+				attr_until_element_expr_ = a->expr();
+				}
+			else if ( ref_input )
+				{
+				if ( attr_until_input_expr_ )
+					{
+					throw Exception(a->expr(), 
+						"multiple &until on $input");
+					}
+				attr_until_input_expr_ = a->expr();
+				}
+			else
+				{
+				if ( attr_generic_until_expr_ )
+					{
+					throw Exception(a->expr(), 
+						"multiple &until condition");
+					}
+				attr_generic_until_expr_ = a->expr();
+				}
+			}
+			break;
+
+		default:
+			break;
+		}
+	}
+
+void ArrayType::Prepare(Env *env, int flags)
+	{
+	if ( flags & TO_BE_PARSED )
+		{
+		ID *arraylength_var = new ID(fmt("%s__arraylength", value_var()->Name()));
+		ID *elem_var = new ID(fmt("%s__elem", value_var()->Name()));
+		ID *elem_it_var = new ID(fmt("%s__it", elem_var->Name()));
+
+		elem_var_field_ = 
+			new ParseVarField(Field::CLASS_MEMBER, elem_var, elemtype_);
+		AddField(elem_var_field_);
+
+		if ( incremental_parsing() )
+			{
+			arraylength_var_field_ = 
+				new PrivVarField(arraylength_var, extern_type_int->Clone());
+			elem_it_var_field_ = 
+				new PrivVarField(elem_it_var, extern_type_int->Clone());
+
+			AddField(arraylength_var_field_);
+			AddField(elem_it_var_field_);
+			}
+		else
+			{
+			arraylength_var_field_ = 
+				new TempVarField(arraylength_var, extern_type_int->Clone());
+			elem_it_var_field_ = 
+				new TempVarField(elem_it_var, extern_type_int->Clone());
+
+			arraylength_var_field_->Prepare(env);
+			elem_it_var_field_->Prepare(env);
+
+			// Add elem_dataptr_var only when not parsing incrementally
+			ID *elem_dataptr_var = 
+				new ID(fmt("%s__dataptr", elem_var->Name()));
+			elem_dataptr_var_field_ = new TempVarField(
+				elem_dataptr_var, 
+				extern_type_const_byteptr->Clone());
+			elem_dataptr_var_field_->Prepare(env);
+
+			// until(dataptr >= end_of_data)
+			elem_dataptr_until_expr_ = new Expr(
+				Expr::EXPR_GE, 
+				new Expr(elem_dataptr_var->clone()),
+				new Expr(end_of_data->clone()));
+			}
+
+		if ( attr_until_input_expr_ )
+			{
+			elemtype_->SetUntilCheck(this);
+			}
+
+		end_of_array_loop_label_ = strfmt("end_of_%s", value_var()->Name());
+		}
+
+	Type::Prepare(env, flags);
+	}
+
+void ArrayType::GenArrayLength(Output *out_cc, Env *env, const DataPtr& data)
+	{
+	if ( env->Evaluated(arraylength_var()) )
+		return;
+
+	if ( ! incremental_parsing() )
+		{
+		arraylength_var_field_->GenTempDecls(out_cc, env);
+		arraylength_var_field_->GenInitCode(out_cc, env);
+		}
+
+	if ( length_ )
+		{
+		out_cc->println("%s = %s;", 
+			env->LValue(arraylength_var()), 
+			length_->EvalExpr(out_cc, env));
+
+		env->SetEvaluated(arraylength_var());
+
+		// Check for overlong array length. We cap it at the
+		// maximum data size as we won't store more elements.
+		out_cc->println("if ( t_begin_of_data + %s > t_end_of_data + 1 )",
+			env->LValue(arraylength_var()));
+		out_cc->inc_indent();
+		out_cc->println("{");
+		out_cc->println("%s = t_end_of_data - t_begin_of_data + 1;",
+			env->LValue(arraylength_var()));
+		out_cc->println("}");
+		out_cc->dec_indent();
+		
+		// Check negative array length
+		out_cc->println("if ( %s < 0 )",
+			env->LValue(arraylength_var())); 
+		out_cc->inc_indent();
+		out_cc->println("{");
+		out_cc->println("%s = 0;",
+			env->LValue(arraylength_var())); 
+		out_cc->println("}");
+		out_cc->dec_indent();
+		}
+	else if ( attr_restofdata_ )
+		{
+		ASSERT(elemtype_->StaticSize(env) == 1);
+		out_cc->println("%s = (%s) - (%s);", 
+			env->LValue(arraylength_var()), 
+			env->RValue(end_of_data),
+			data.ptr_expr());
+		env->SetEvaluated(arraylength_var());
+		}
+	}
+
+void ArrayType::GenPubDecls(Output *out_h, Env *env)
+	{
+	Type::GenPubDecls(out_h, env);
+
+	if ( declared_as_type() )
+		{
+		out_h->println("int size() const	{ return %s ? %s->size() : 0; }",
+			env->RValue(value_var()),
+			env->RValue(value_var()));
+		out_h->println("%s operator[](int index) const { BINPAC_ASSERT(%s); return (*%s)[index]; }",
+			elemtype_->DataTypeConstRefStr().c_str(),
+			env->RValue(value_var()),
+			env->RValue(value_var()));
+		}
+	}
+
+void ArrayType::GenPrivDecls(Output *out_h, Env *env)
+	{
+	ASSERT(elem_var_field_->type() == elemtype_);
+	ASSERT(elemtype_->value_var());
+	Type::GenPrivDecls(out_h, env);
+	}
+
+void ArrayType::GenInitCode(Output *out_cc, Env *env)
+	{
+	// Do not initiate the array here
+	// out_cc->println("%s = new %s;", lvalue(), vector_str_.c_str());
+	out_cc->println("%s = 0;", lvalue());
+
+	Type::GenInitCode(out_cc, env);
+	if ( incremental_parsing() )
+		{
+		out_cc->println("%s = -1;", 
+			env->LValue(elem_it_var()));
+		}
+	}
+
+void ArrayType::GenCleanUpCode(Output *out_cc, Env *env)
+	{
+	Type::GenCleanUpCode(out_cc, env);
+	if ( elemtype_->NeedsCleanUp() )
+		{
+		if ( ! elem_var_field_ )
+			{
+			ID *elem_var = new ID(fmt("%s__elem", value_var()->Name()));
+			elem_var_field_ = 
+				new ParseVarField(
+					Field::NOT_CLASS_MEMBER, 
+					elem_var, 
+					elemtype_);
+			elem_var_field_->Prepare(env);
+			}
+
+		out_cc->println("if ( %s )", env->RValue(value_var()));
+		out_cc->inc_indent();
+		out_cc->println("{");
+
+		out_cc->println("for ( int i = 0; i < (int) %s->size(); ++i )",
+			env->RValue(value_var()));
+		out_cc->inc_indent();
+		out_cc->println("{");
+		out_cc->println("%s %s = (*%s)[i];", 
+			elemtype_->DataTypeStr().c_str(), 
+			env->LValue(elem_var()), 
+			lvalue());
+		elemtype_->GenCleanUpCode(out_cc, env);
+		out_cc->println("}");
+		out_cc->dec_indent();
+
+		out_cc->println("}");
+		out_cc->dec_indent();
+		}
+	out_cc->println("delete %s;", lvalue());
+	}
+
+string ArrayType::GenArrayInit(Output *out_cc, Env *env, bool known_array_length)
+	{
+	string array_str;
+
+	array_str = lvalue();
+	if ( incremental_parsing() )
+		{
+		out_cc->println("if ( %s < 0 )", 
+			env->LValue(elem_it_var()));
+		out_cc->inc_indent();
+		out_cc->println("{");
+		out_cc->println("// Initialize only once");
+		out_cc->println("%s = 0;", env->LValue(elem_it_var()));
+		}
+
+	out_cc->println("%s = new %s;", 
+		lvalue(), vector_str_.c_str());
+
+	if ( known_array_length )
+		{
+		out_cc->println("%s->reserve(%s);", 
+			lvalue(), env->RValue(arraylength_var()));
+		}
+
+	if ( incremental_parsing() )
+		{
+		out_cc->println("}");
+		out_cc->dec_indent();
+		}
+
+	return array_str;
+	}
+
+void ArrayType::GenElementAssignment(Output *out_cc, Env *env,
+		string const &array_str, bool use_vector)
+	{
+	// Assign the element
+	if ( ! use_vector )
+		{
+		out_cc->println("%s[%s] = %s;", 
+			array_str.c_str(), 
+			env->LValue(elem_it_var()), 
+			env->LValue(elem_var()));
+		}
+	else
+		{
+		out_cc->println("%s->push_back(%s);", 
+			array_str.c_str(), 
+			env->LValue(elem_var()));
+		}
+	}
+
+void ArrayType::DoGenParseCode(Output *out_cc, Env *env, 
+		const DataPtr& data, int flags)
+	{
+	GenArrayLength(out_cc, env, data);
+
+	// Otherwise these variables are declared as member variables
+	if ( ! incremental_parsing() )
+		{
+		// Declare and initialize temporary variables
+		elem_var_field_->GenInitCode(out_cc, env);
+		elem_it_var_field_->GenTempDecls(out_cc, env);
+		out_cc->println("%s = 0;", env->LValue(elem_it_var()));
+		env->SetEvaluated(elem_it_var());
+		}
+
+	/*
+	If the input length can be determined without parsing
+	individual elements, generate the boundary checking before
+	parsing (unless in the case of incremental parsing).
+
+	There are two cases when the input length can be determined:
+	1. The array has a static size;
+	2. The array length can be computed before parsing and
+   	each element is of constant size.
+	*/
+
+	bool compute_size_var = false;
+
+	if ( incremental_input() )
+		{
+		// Do not compute size_var on incremental input
+		compute_size_var = false;
+		
+		if ( ! incremental_parsing() &&
+		     ( StaticSize(env) >= 0 ||
+		       ( env->Evaluated(arraylength_var()) && 
+		         elemtype_->StaticSize(env) >= 0 ) ) )
+			{
+			GenBoundaryCheck(out_cc, env, data);
+			}
+		}
+	else
+		{
+		compute_size_var = AddSizeVar(out_cc, env);
+		}
+
+	bool known_array_length = env->Evaluated(arraylength_var());
+	string array_str = GenArrayInit(out_cc, env, known_array_length);
+
+	bool use_vector = true;
+
+	ASSERT(elem_it_var());
+
+	DataPtr elem_data(env, 0, 0);
+
+	if ( elem_dataptr_var() )
+		{
+		out_cc->println("const_byteptr %s = %s;", 
+			env->LValue(elem_dataptr_var()), data.ptr_expr());
+		env->SetEvaluated(elem_dataptr_var());
+
+		elem_data = DataPtr(env, elem_dataptr_var(), 0);
+		}
+
+	string for_condition = known_array_length ?
+		strfmt("%s < %s", 
+			env->LValue(elem_it_var()), 
+			env->RValue(arraylength_var())) :
+		"/* forever */";
+
+	out_cc->println("for (; %s; ++%s)", 
+		for_condition.c_str(), 
+		env->LValue(elem_it_var()));
+	out_cc->inc_indent();
+	out_cc->println("{");
+
+	if ( attr_generic_until_expr_ )
+		GenUntilCheck(out_cc, env, attr_generic_until_expr_, true);
+
+	if ( elem_dataptr_var() )
+		GenUntilCheck(out_cc, env, elem_dataptr_until_expr_, false);
+	
+	elemtype_->GenPreParsing(out_cc, env);
+	elemtype_->GenParseCode(out_cc, env, elem_data, flags);
+
+	if ( incremental_parsing() ) 
+		{
+		out_cc->println("if ( ! %s )", 
+			elemtype_->parsing_complete(env).c_str());
+		out_cc->inc_indent();
+		out_cc->println("goto %s;", kNeedMoreData);
+		out_cc->dec_indent();
+		}
+
+	GenElementAssignment(out_cc, env, array_str, use_vector);
+
+	if ( elem_dataptr_var() )
+		{
+		out_cc->println("%s += %s;", 
+			env->LValue(elem_dataptr_var()), 
+			elemtype_->DataSize(0, env, elem_data).c_str());
+		out_cc->println("BINPAC_ASSERT(%s <= %s);",
+			env->RValue(elem_dataptr_var()), 
+			env->RValue(end_of_data));
+		}
+
+	if ( attr_until_element_expr_ )
+		GenUntilCheck(out_cc, env, attr_until_element_expr_, false);
+
+	if ( elemtype_->IsPointerType() )
+		out_cc->println("%s = 0;", env->LValue(elem_var()));
+
+	out_cc->println("}");
+	out_cc->dec_indent();
+
+	out_cc->dec_indent();
+	out_cc->println("%s: ;", end_of_array_loop_label_.c_str());
+	out_cc->inc_indent();
+
+	if ( compute_size_var && elem_dataptr_var() && ! env->Evaluated(size_var()) )
+		{
+		// Compute the data size
+		out_cc->println("%s = %s - (%s);", 
+			env->LValue(size_var()),
+			env->RValue(elem_dataptr_var()),
+			data.ptr_expr());
+		env->SetEvaluated(size_var());
+		}
+	}
+
+void ArrayType::GenUntilInputCheck(Output *out_cc, Env *env)
+	{
+	ID *elem_input_var_id = new ID(
+		fmt("%s__elem_input", value_var()->Name()));
+	elem_input_var_field_ = new TempVarField(
+		elem_input_var_id, extern_type_const_bytestring->Clone());
+	elem_input_var_field_->Prepare(env);
+
+	out_cc->println("%s %s(%s, %s);", 
+		extern_type_const_bytestring->DataTypeStr().c_str(),
+		env->LValue(elem_input_var()),
+		env->RValue(begin_of_data),
+		env->RValue(end_of_data));
+	env->SetEvaluated(elem_input_var());
+
+	GenUntilCheck(out_cc, env, attr_until_input_expr_, true);
+	}
+
+void ArrayType::GenUntilCheck(Output *out_cc, Env *env, 
+		Expr *until_expr, bool delete_elem)
+	{
+	ASSERT(until_expr);
+
+	Env check_env(env, this);
+	check_env.AddMacro(element_macro_id, 
+		new Expr(elem_var()->clone()));
+	if ( elem_input_var() )
+		{
+		check_env.AddMacro(input_macro_id, 
+			new Expr(elem_input_var()->clone()));
+		}
+
+	out_cc->println("// Check &until(%s)", until_expr->orig());
+	out_cc->println("if ( %s )", 
+		until_expr->EvalExpr(out_cc, &check_env));
+	out_cc->inc_indent();
+	out_cc->println("{");
+	if ( parsing_complete_var() )
+		{
+		out_cc->println("%s = true;",
+			env->LValue(parsing_complete_var()));
+		}
+
+	if ( elemtype_->IsPointerType() )
+		{
+		if ( delete_elem )
+			elemtype_->GenCleanUpCode(out_cc, env);
+		else
+			out_cc->println("%s = 0;", env->LValue(elem_var()));
+		}
+
+	out_cc->println("goto %s;", end_of_array_loop_label_.c_str());
+	out_cc->println("}");
+	out_cc->dec_indent();
+	}
+
+void ArrayType::GenDynamicSize(Output *out_cc, Env *env,
+		const DataPtr& data)
+	{
+	ASSERT(! incremental_input());
+	DEBUG_MSG("Generating dynamic size for array `%s'\n", 
+		value_var()->Name());
+
+	int elem_w = elemtype_->StaticSize(env);
+	if ( elem_w >= 0 &&
+	     ! attr_until_element_expr_ && 
+	     ! attr_until_input_expr_ &&
+	     ( length_ || attr_restofdata_ ) )
+		{
+		// If the elements have a fixed size,
+		// we only need to compute the number of elements
+		bool compute_size_var = AddSizeVar(out_cc, env);
+		ASSERT(compute_size_var);
+		GenArrayLength(out_cc, env, data);
+		ASSERT(env->Evaluated(arraylength_var()));
+		out_cc->println("%s = %d * %s;",
+			env->LValue(size_var()), elem_w, env->RValue(arraylength_var()));
+		env->SetEvaluated(size_var());
+		}
+	else
+		{
+		// Otherwise we need parse the array dynamically
+		GenParseCode(out_cc, env, data, 0);
+		}
+	}
+
+int ArrayType::StaticSize(Env *env) const
+	{
+	int num = 0;
+
+	if ( ! length_ || ! length_->ConstFold(env, &num) )
+		return -1;
+
+	int elem_w = elemtype_->StaticSize(env);
+	if ( elem_w < 0 )
+		return -1;
+
+	DEBUG_MSG("static size of %s:%s = %d * %d\n", 
+		decl_id()->Name(), lvalue(), elem_w, num);
+
+	return num * elem_w;
+	}
+
+void ArrayType::SetBoundaryChecked()
+	{
+	Type::SetBoundaryChecked();
+	elemtype_->SetBoundaryChecked();
+	}
+
+void ArrayType::DoMarkIncrementalInput()
+	{
+	elemtype_->MarkIncrementalInput(); 
+	}
+
+bool ArrayType::RequiresAnalyzerContext()
+	{ 
+	return Type::RequiresAnalyzerContext() ||
+	       ( length_ && length_->RequiresAnalyzerContext() ) || 
+	       elemtype_->RequiresAnalyzerContext(); 
+	}
+
+bool ArrayType::DoTraverse(DataDepVisitor *visitor)
+	{ 
+	if ( ! Type::DoTraverse(visitor) )
+		return false;
+
+	if ( length_ && ! length_->Traverse(visitor) )
+		return false;
+
+	if ( ! elemtype_->Traverse(visitor) )
+		return false; 
+
+	return true;
+	}

tools/binpac/src/pac_array.h

@@ -0,0 +1,92 @@
+#ifndef pac_array_h
+#define pac_array_h
+
+#include "pac_common.h"
+#include "pac_type.h"
+
+// Fixed-length array and variable length sequence with an ending pattern
+
+class ArrayType : public Type
+{
+public:
+	ArrayType(Type *arg_elemtype, Expr *arg_length = 0);
+	~ArrayType();
+
+	bool DefineValueVar() const;
+	string DataTypeStr() const;
+	string DefaultValue() const	{ return "0"; }
+	Type *ElementDataType() const;
+
+	string EvalElement(const string &array, const string &index) const;
+
+	void ProcessAttr(Attr *a);
+
+	void Prepare(Env *env, int flags);
+
+	void GenPubDecls(Output *out, Env *env);
+	void GenPrivDecls(Output *out, Env *env);
+
+	void GenInitCode(Output *out, Env *env);
+	void GenCleanUpCode(Output *out, Env *env);
+
+	int StaticSize(Env *env) const;
+
+	void SetBoundaryChecked();
+	void GenUntilInputCheck(Output *out_cc, Env *env);
+
+	bool IsPointerType() const	{ return true; }
+
+protected:
+	void init();
+
+	void DoGenParseCode(Output *out, Env *env, const DataPtr& data, int flags);
+	void GenDynamicSize(Output *out, Env *env, const DataPtr& data);
+	void GenArrayLength(Output *out_cc, Env *env, const DataPtr& data);
+	string GenArrayInit(Output *out_cc, Env *env, bool known_array_length);
+	void GenElementAssignment(Output *out_cc, Env *env,
+		string const &array_str, bool use_vector);
+	void GenUntilCheck(Output *out_cc, Env *env, 
+		Expr *until_condition, bool delete_elem);
+
+	bool ByteOrderSensitive() const 
+		{ 
+		return elemtype_->RequiresByteOrder(); 
+		}
+	bool RequiresAnalyzerContext();
+
+	Type *DoClone() const;
+
+	void DoMarkIncrementalInput();
+
+	const ID *arraylength_var() const;
+	const ID *elem_it_var() const;
+	const ID *elem_var() const;
+	const ID *elem_dataptr_var() const;
+	const ID *elem_input_var() const;
+
+protected:
+	bool DoTraverse(DataDepVisitor *visitor);
+
+private:
+	Type *elemtype_;
+	Expr *length_;
+
+	string vector_str_;
+	string datatype_str_;
+	string end_of_array_loop_label_;
+
+	Field *arraylength_var_field_;
+	Field *elem_it_var_field_;
+	Field *elem_var_field_;
+	Field *elem_dataptr_var_field_;
+	Field *elem_input_var_field_;
+
+	// This does not come from &until, but is internally generated
+	Expr *elem_dataptr_until_expr_;
+
+	Expr *attr_generic_until_expr_;
+	Expr *attr_until_element_expr_;
+	Expr *attr_until_input_expr_;
+};
+
+#endif  // pac_array_h

tools/binpac/src/pac_attr.cc

@@ -0,0 +1,57 @@
+#include "pac_attr.h"
+#include "pac_expr.h"
+
+bool Attr::DoTraverse(DataDepVisitor *visitor)
+	{
+	if ( expr_ && ! expr_->Traverse(visitor) )
+		return false;
+	return true;
+	}
+
+bool Attr::RequiresAnalyzerContext() const
+	{
+	return (expr_ && expr_->RequiresAnalyzerContext());
+	}
+
+void Attr::init()
+	{
+	expr_ = 0;
+	seqend_ = 0;
+	}
+
+Attr::Attr(AttrType type)
+	: DataDepElement(DataDepElement::ATTR)
+	{
+	type_ = type;
+	init();
+	}
+
+Attr::Attr(AttrType type, Expr *expr)
+	: DataDepElement(DataDepElement::ATTR)
+	{
+	type_ = type;
+	init();
+	expr_ = expr;
+	}
+
+Attr::Attr(AttrType type, ExprList *exprlist)
+	: DataDepElement(DataDepElement::ATTR)
+	{
+	type_ = type;
+	init();
+	expr_ = new Expr(exprlist);
+	}
+
+Attr::Attr(AttrType type, SeqEnd *seqend)
+	: DataDepElement(DataDepElement::ATTR)
+	{
+	type_ = type;
+	init();
+	seqend_ = seqend;
+	}
+
+LetAttr::LetAttr(FieldList *letfields)
+	: Attr(ATTR_LET)
+	{
+	letfields_ = letfields;
+	}

tools/binpac/src/pac_attr.h

@@ -0,0 +1,61 @@
+#ifndef pac_attr_h
+#define pac_attr_h
+
+#include "pac_common.h"
+#include "pac_datadep.h"
+
+enum AttrType { 
+	ATTR_BYTEORDER, 
+	ATTR_CHECK, 
+	ATTR_CHUNKED,
+	ATTR_EXPORTSOURCEDATA,
+	ATTR_IF,
+	ATTR_LENGTH, 
+	ATTR_LET,
+	ATTR_LINEBREAKER,
+	ATTR_MULTILINE,
+	ATTR_ONELINE,
+	ATTR_REFCOUNT,
+	ATTR_REQUIRES,
+	ATTR_RESTOFDATA, 
+	ATTR_RESTOFFLOW, 
+	ATTR_TRANSIENT,
+	ATTR_UNTIL,
+};
+
+class Attr : public Object, public DataDepElement
+{
+public:
+	Attr(AttrType type);
+	Attr(AttrType type, Expr *expr);
+	Attr(AttrType type, ExprList *exprlist);
+	Attr(AttrType type, SeqEnd *seqend);
+
+	AttrType type() const 		{ return type_; }
+	Expr *expr() const		{ return expr_; }
+	SeqEnd *seqend() const		{ return seqend_; }
+
+	bool RequiresAnalyzerContext() const;
+
+protected:
+	bool DoTraverse(DataDepVisitor *visitor);
+
+protected:
+	void init();
+
+	AttrType type_;
+	Expr *expr_;
+	SeqEnd *seqend_;
+};
+
+class LetAttr : public Attr
+{
+public:
+	LetAttr(FieldList *letfields);
+	FieldList *letfields() const	{ return letfields_; }
+
+private:
+	FieldList *letfields_;
+};
+
+#endif  // pac_attr_h

tools/binpac/src/pac_btype.cc

@@ -0,0 +1,144 @@
+#include "pac_btype.h"
+#include "pac_dataptr.h"
+#include "pac_id.h"
+#include "pac_output.h"
+
+Type *BuiltInType::DoClone() const
+	{
+	return new BuiltInType(bit_type());
+	}
+
+bool BuiltInType::IsNumericType() const
+	{
+	BITType t = bit_type();
+	return (t == INT8 || t == INT16 || t == INT32 ||
+	        t == UINT8 || t == UINT16 || t == UINT32);
+	}
+
+bool BuiltInType::CompatibleBuiltInTypes(BuiltInType *type1, 
+	                                 BuiltInType *type2)
+	{
+	return type1->IsNumericType() && type2->IsNumericType();
+	}
+
+static const char* basic_pactype_name[] = {
+#	define TYPE_DEF(name, pactype, ctype, size)	pactype,
+#	include "pac_type.def"
+#	undef TYPE_DEF
+	0,
+};
+
+void BuiltInType::static_init()
+	{
+	for ( int bit_type = 0; basic_pactype_name[bit_type]; ++bit_type )
+		{
+		Type::AddPredefinedType(
+			basic_pactype_name[bit_type], 
+			new BuiltInType((BITType) bit_type));
+		}
+	}
+
+int BuiltInType::LookUpByName(const char* name)
+	{
+	ASSERT(0);
+	for ( int i = 0; basic_pactype_name[i]; ++i )
+		if ( strcmp(basic_pactype_name[i], name) == 0 )
+			return i;
+	return -1;
+	}
+
+static const char* basic_ctype_name[] = {
+#	define TYPE_DEF(name, pactype, ctype, size)	ctype,
+#	include "pac_type.def"
+#	undef TYPE_DEF
+	0,
+};
+
+bool BuiltInType::DefineValueVar() const
+	{
+	return bit_type_ != EMPTY;
+	}
+
+string BuiltInType::DataTypeStr() const
+	{
+	return basic_ctype_name[bit_type_];
+	}
+
+int BuiltInType::StaticSize(Env* /* env */) const
+	{
+	static const size_t basic_type_size[] = 
+		{
+#		define TYPE_DEF(name, pactype, ctype, size)	size,
+#		include "pac_type.def"
+#		undef TYPE_DEF
+		};
+
+	return basic_type_size[bit_type_];
+	}
+
+void BuiltInType::DoMarkIncrementalInput()
+	{
+	if ( bit_type_ == EMPTY )
+		return;
+	Type::DoMarkIncrementalInput();
+	}
+
+void BuiltInType::GenInitCode(Output* out_cc, Env* env)
+	{
+	if ( bit_type_ != EMPTY )
+		out_cc->println("%s = 0;", env->LValue(value_var()));
+	Type::GenInitCode(out_cc, env);
+	}
+
+void BuiltInType::GenDynamicSize(Output* out_cc, Env* env, const DataPtr& data)
+	{
+	/* should never be called */
+	ASSERT(0);
+	}
+
+void BuiltInType::DoGenParseCode(Output* out_cc, Env* env,
+		const DataPtr& data, int flags)
+	{
+	if ( bit_type_ == EMPTY )
+		return;
+
+	// There is no need to generate the size variable
+	// out_cc->println("%s = sizeof(%s);", size_var(), DataTypeStr().c_str());
+
+	GenBoundaryCheck(out_cc, env, data);
+
+	if ( anonymous_value_var() )
+		return;
+
+	switch ( bit_type_ ) 
+		{
+		case EMPTY:
+			// do nothing
+			break;
+
+		case INT8:
+		case UINT8:
+			out_cc->println("%s = *((%s const *) (%s));",
+				lvalue(), DataTypeStr().c_str(), data.ptr_expr()); 
+			break;
+		case INT16:
+		case UINT16:
+		case INT32:
+		case UINT32:
+#if 0
+			out_cc->println("%s = UnMarshall<%s>(%s, %s);",
+				lvalue(), 
+				DataTypeStr().c_str(), 
+				data.ptr_expr(),
+				EvalByteOrder(out_cc, env).c_str());
+#else
+			out_cc->println("%s = FixByteOrder(%s, *((%s const *) (%s)));",
+				lvalue(), 
+				EvalByteOrder(out_cc, env).c_str(),
+				DataTypeStr().c_str(), 
+				data.ptr_expr());
+#endif
+			break;
+		}
+	}
+

tools/binpac/src/pac_btype.h

@@ -0,0 +1,52 @@
+#ifndef pac_btype_h
+#define pac_btype_h
+
+#include "pac_type.h"
+
+class BuiltInType : public Type
+{
+public:
+	enum BITType {
+#		define TYPE_DEF(name, pactype, ctype, size)	name,
+#		include "pac_type.def"
+#		undef TYPE_DEF
+	};
+
+	static int LookUpByName(const char *name);
+
+	BuiltInType(BITType bit_type)
+		: Type(bit_type == BuiltInType::EMPTY ? Type::EMPTY : BUILTIN),
+		  bit_type_(bit_type) {}
+
+	BITType bit_type() const	{ return bit_type_; }
+
+	bool IsNumericType() const;
+
+	bool DefineValueVar() const;
+	string DataTypeStr() const;
+	string DefaultValue() const	{ return "0"; }
+
+	int StaticSize(Env *env) const;
+
+	bool IsPointerType() const	{ return false; }
+
+	bool ByteOrderSensitive() const { return StaticSize(0) >= 2; }
+
+	void GenInitCode(Output *out_cc, Env *env);
+
+	void DoMarkIncrementalInput();
+
+protected:
+	void DoGenParseCode(Output *out, Env *env, const DataPtr& data, int flags);
+	void GenDynamicSize(Output *out, Env *env, const DataPtr& data);
+	Type *DoClone() const;
+
+	BITType bit_type_;
+
+public:
+	static void static_init();
+	static bool CompatibleBuiltInTypes(BuiltInType *type1, 
+	                                   BuiltInType *type2);
+};
+
+#endif  // pac_btype_h

tools/binpac/src/pac_case.cc

@@ -0,0 +1,391 @@
+#include "pac_exception.h"
+#include "pac_expr.h"
+#include "pac_exttype.h"
+#include "pac_id.h"
+#include "pac_output.h"
+#include "pac_typedecl.h"
+#include "pac_utils.h"
+
+#include "pac_case.h"
+
+CaseType::CaseType(Expr* index_expr, CaseFieldList* cases)
+	: Type(CASE), index_expr_(index_expr), cases_(cases) 
+	{
+	index_var_ = 0;
+	foreach(i, CaseFieldList, cases_)
+		AddField(*i);
+	}
+
+CaseType::~CaseType()
+	{
+	delete index_var_;
+	delete index_expr_;
+	delete cases_;
+	}
+
+void CaseType::AddCaseField(CaseField *f)
+	{
+	// All fields must be added before Prepare()
+	ASSERT(!env());
+
+	AddField(f);
+	cases_->push_back(f);
+	}
+
+bool CaseType::DefineValueVar() const
+	{
+	return false;
+	}
+
+string CaseType::DataTypeStr() const
+	{
+	ASSERT(type_decl());
+	return strfmt("%s *", type_decl()->class_name().c_str());
+	}
+
+Type *CaseType::ValueType() const
+	{
+	foreach (i, CaseFieldList, cases_)
+		{
+		CaseField *c = *i;
+		return c->type();
+		}
+	ASSERT(0);
+	return 0;
+	}
+
+string CaseType::DefaultValue() const
+	{
+	return ValueType()->DefaultValue();
+	}
+
+void CaseType::Prepare(Env* env, int flags)
+	{
+	ASSERT(flags & TO_BE_PARSED);
+
+	index_var_ = new ID(fmt("%s_case_index", value_var()->Name()));
+	env->AddID(index_var_, MEMBER_VAR, extern_type_int);
+
+	// Sort the cases_ to put the default case at the end of the list
+	CaseFieldList::iterator default_case_it = 
+		cases_->end(); // to avoid warning
+	CaseField *default_case = 0;
+
+	foreach (i, CaseFieldList, cases_)
+		{
+		CaseField *c = *i;
+		if ( ! c->index() )
+			{
+			if ( default_case )
+				throw Exception(c, "duplicate default case");
+			default_case_it = i;
+			default_case = c;
+			}
+		}
+	if ( default_case )
+		{
+		cases_->erase(default_case_it);
+		cases_->push_back(default_case);
+		}
+
+	foreach (i, CaseFieldList, cases_)
+		{
+		CaseField *c = *i;
+		c->set_index_var(index_var_);
+		c->set_case_type(this);
+		}
+
+	Type::Prepare(env, flags);
+	}
+
+void CaseType::GenPrivDecls(Output* out_h, Env* env)
+	{
+	out_h->println("int %s;", env->LValue(index_var_));
+	Type::GenPrivDecls(out_h, env);
+	}
+
+void CaseType::GenPubDecls(Output* out_h, Env* env)
+	{
+	out_h->println("int %s const	{ return %s; }",
+		env->RValue(index_var_), env->LValue(index_var_));
+	Type::GenPubDecls(out_h, env);
+	}
+
+void CaseType::GenInitCode(Output* out_cc, Env* env)
+	{
+	out_cc->println("%s = -1;", env->LValue(index_var_));
+	Type::GenInitCode(out_cc, env);
+	}
+
+void CaseType::GenCleanUpCode(Output* out_cc, Env* env)
+	{
+	Type::GenCleanUpCode(out_cc, env);
+
+	env->set_in_branch(true);
+	out_cc->println("switch ( %s )", env->RValue(index_var_));
+	out_cc->inc_indent();
+	out_cc->println("{");
+	foreach (i, CaseFieldList, cases_)
+		{
+		CaseField *c = *i;
+		c->GenCleanUpCode(out_cc, env);
+		}
+	out_cc->println("}");
+	out_cc->dec_indent();
+	env->set_in_branch(false);
+	}
+
+void CaseType::DoGenParseCode(Output* out_cc, Env* env,
+		const DataPtr& data, int flags)
+	{
+	if ( StaticSize(env) >= 0 )
+		GenBoundaryCheck(out_cc, env, data);
+
+	bool compute_size_var = false;
+
+	if ( ! incremental_input() )
+		compute_size_var = AddSizeVar(out_cc, env);
+
+	out_cc->println("%s = %s;", 
+		env->LValue(index_var_), index_expr_->EvalExpr(out_cc, env));
+	env->SetEvaluated(index_var_);
+	
+	env->set_in_branch(true);
+	out_cc->println("switch ( %s )", env->RValue(index_var_));
+	out_cc->inc_indent();
+	out_cc->println("{");
+	bool has_default_case = false;
+	foreach (i, CaseFieldList, cases_)
+		{
+		CaseField *c = *i;
+		c->GenParseCode(out_cc, env, data, 
+			compute_size_var ? size_var() : 0);
+		if ( c->IsDefaultCase() )
+			has_default_case = true;
+		}
+
+	if ( ! has_default_case )
+		{
+		out_cc->println("default:");
+		out_cc->inc_indent();
+		out_cc->println("throw ExceptionInvalidCaseIndex(\"%s\", %s);", 
+			decl_id()->Name(), env->RValue(index_var_));
+		out_cc->println("break;");
+		out_cc->dec_indent();
+		}
+	out_cc->println("}");
+	out_cc->dec_indent();
+	env->set_in_branch(false);
+
+	if ( compute_size_var )
+		env->SetEvaluated(size_var());
+	}
+
+void CaseType::GenDynamicSize(Output* out_cc, Env* env,
+		const DataPtr& data)
+	{
+	GenParseCode(out_cc, env, data, 0);
+	}
+
+int CaseType::StaticSize(Env* env) const
+	{
+	int static_w = -1;
+	foreach (i, CaseFieldList, cases_)
+		{
+		CaseField *c = *i;
+		int w = c->StaticSize(env);
+		if ( w < 0 || ( static_w >= 0 && w != static_w ) )
+			return -1;
+		static_w = w;
+		}
+	return static_w;
+	}
+
+void CaseType::SetBoundaryChecked()
+	{
+	Type::SetBoundaryChecked();
+	foreach (i, CaseFieldList, cases_)
+		{
+		CaseField *c = *i;
+		c->SetBoundaryChecked();
+		}
+	}
+
+void CaseType::DoMarkIncrementalInput()
+	{
+	foreach (i, CaseFieldList, cases_)
+		{
+		CaseField *c = *i;
+		c->type()->MarkIncrementalInput();
+		}
+	}
+
+bool CaseType::ByteOrderSensitive() const
+	{
+	foreach (i, CaseFieldList, cases_)
+		{
+		CaseField *c = *i;
+		if ( c->RequiresByteOrder() )
+			return true;
+		}
+	return false;
+	}
+
+CaseField::CaseField(ExprList* index, ID* id, Type* type)
+	: Field(CASE_FIELD, 
+		TYPE_TO_BE_PARSED | CLASS_MEMBER | PUBLIC_READABLE, 
+		id, type), 
+	  index_(index)
+	{
+	ASSERT(type_);
+	type_->set_value_var(id, MEMBER_VAR);
+	case_type_ = 0;
+	}
+
+CaseField::~CaseField()
+	{
+	delete_list(ExprList, index_);
+	}
+
+void GenCaseStr(ExprList *index_list, Output *out_cc, Env *env)
+	{
+	if ( index_list )
+		{
+		foreach(i, ExprList, index_list)
+			{
+			Expr *index_expr = *i;
+			int index_const;
+			
+			if ( ! index_expr->ConstFold(env, &index_const) )
+				throw ExceptionNonConstExpr(index_expr);
+			out_cc->println("case %d:", index_const);
+			}
+		}
+	else
+		{
+		out_cc->println("default:");
+		}
+	}
+
+void CaseField::Prepare(Env* env)
+	{
+	ASSERT(index_var_);
+	Field::Prepare(env);
+	}
+
+void CaseField::GenPubDecls(Output* out_h, Env* env)
+	{
+	if ( ! ((flags_ & PUBLIC_READABLE) && (flags_ & CLASS_MEMBER)) )
+		return;
+
+	// Skip type "empty"
+	if ( type_->DataTypeStr().empty() )
+		return;
+
+	out_h->println("%s %s const",
+		type_->DataTypeConstRefStr().c_str(), env->RValue(id_));
+
+	out_h->inc_indent();
+	out_h->println("{");
+
+	if ( ! index_ )
+		out_h->println("return %s;", lvalue());
+	else
+		{
+		out_h->println("switch ( %s )", env->RValue(index_var_));
+		out_h->inc_indent();
+		out_h->println("{");
+		GenCaseStr(index_, out_h, env);
+		out_h->inc_indent();
+		out_h->println("break;  // OK");
+		out_h->dec_indent();
+
+		out_h->println("default:");
+		out_h->inc_indent();
+		out_h->println("throw ExceptionInvalidCase(\"%s\", %s, \"%s\");", 
+		               id_->LocName(), 
+		               env->RValue(index_var_), 
+		               OrigExprList(index_).c_str());
+		out_h->println("break;");
+		out_h->dec_indent();
+
+		out_h->println("}");
+		out_h->dec_indent();
+
+		out_h->println("return %s;", lvalue());
+		}
+
+	out_h->println("}");
+	out_h->dec_indent();
+	}
+
+void CaseField::GenInitCode(Output* out_cc, Env* env)
+	{
+	// GenCaseStr(index_, out_cc, env);
+	// out_cc->inc_indent();
+	// out_cc->println("{");
+	// out_cc->println("// Initialize \"%s\"", id_->Name());
+	type_->GenInitCode(out_cc, env);
+	// out_cc->println("}");
+	// out_cc->println("break;");
+	// out_cc->dec_indent();
+	}
+
+void CaseField::GenCleanUpCode(Output* out_cc, Env* env)
+	{
+	GenCaseStr(index_, out_cc, env);
+	out_cc->inc_indent();
+	out_cc->println("// Clean up \"%s\"", id_->Name());
+	out_cc->println("{");
+	if ( ! anonymous_field() )
+		type_->GenCleanUpCode(out_cc, env);
+	out_cc->println("}");
+	out_cc->println("break;");
+	out_cc->dec_indent();
+	}
+
+void CaseField::GenParseCode(Output* out_cc, Env* env, 
+		const DataPtr& data, const ID* size_var)
+	{
+	GenCaseStr(index_, out_cc, env);
+	out_cc->inc_indent();
+	out_cc->println("// Parse \"%s\"", id_->Name());
+	out_cc->println("{");
+	
+	{
+	Env case_env(env, this);
+	Env *env = &case_env;
+
+	type_->GenPreParsing(out_cc, env);
+	type_->GenParseCode(out_cc, env, data, 0);
+	if ( size_var )
+		{
+		out_cc->println("%s = %s;", 
+			env->LValue(size_var),
+			type_->DataSize(out_cc, env, data).c_str());
+		}
+	if ( type_->incremental_input() )
+		{
+		ASSERT(case_type()->parsing_complete_var());
+		out_cc->println("%s = %s;", 
+			env->LValue(case_type()->parsing_complete_var()),
+			env->RValue(type_->parsing_complete_var()));
+		}
+	out_cc->println("}");
+	}
+
+	out_cc->println("break;");
+	out_cc->dec_indent();
+	}
+
+bool CaseField::DoTraverse(DataDepVisitor *visitor)
+	{ 
+	return Field::DoTraverse(visitor) &&
+	       type()->Traverse(visitor); 
+	}
+
+bool CaseField::RequiresAnalyzerContext() const 
+	{ 
+	return Field::RequiresAnalyzerContext() ||
+	       type()->RequiresAnalyzerContext(); 
+	}

tools/binpac/src/pac_case.h

@@ -0,0 +1,99 @@
+#ifndef pac_case_h
+#define pac_case_h
+
+#include "pac_common.h"
+#include "pac_field.h"
+#include "pac_id.h"
+#include "pac_type.h"
+
+class CaseType : public Type
+{
+public:
+	CaseType(Expr *index, CaseFieldList *cases);
+	~CaseType();
+
+	void AddCaseField(CaseField *f);
+
+	bool DefineValueVar() const;
+	string DataTypeStr() const;
+	string DefaultValue() const;
+
+	void Prepare(Env *env, int flags);
+
+	void GenPubDecls(Output *out, Env *env);
+	void GenPrivDecls(Output *out, Env *env);
+
+	void GenInitCode(Output *out, Env *env);
+	void GenCleanUpCode(Output *out, Env *env);
+
+	int StaticSize(Env *env) const;
+
+	void SetBoundaryChecked();
+
+	Type *ValueType() const;
+
+	bool IsPointerType() const	{ return ValueType()->IsPointerType(); }
+
+protected:
+	void DoGenParseCode(Output *out, Env *env, const DataPtr& data, int flags);
+	void GenDynamicSize(Output *out, Env *env, const DataPtr& data);
+	Type *DoClone() const 	{ return 0; }
+	void DoMarkIncrementalInput();
+
+	bool ByteOrderSensitive() const;
+
+	Expr *index_expr_;
+	ID *index_var_;
+	CaseFieldList *cases_;
+
+	typedef map<const ID*, CaseField*, ID_ptr_cmp> member_map_t;
+	member_map_t member_map_;
+};
+
+class CaseField : public Field
+{
+public:
+	CaseField(ExprList *index, ID *id, Type *type);
+	~CaseField();
+
+	CaseType *case_type() const	{ return case_type_; }
+	void set_case_type(CaseType *t)	{ case_type_ = t; }
+
+	ExprList *index() const		{ return index_; }
+
+	const char *lvalue() const	{ return type_->lvalue(); }
+
+	const char *CaseStr(Env *env);
+	void set_index_var(const ID *var) { index_var_ = var; }
+	
+	void Prepare(Env *env);
+
+	void GenPubDecls(Output *out, Env *env);
+
+	void GenInitCode(Output *out, Env *env);
+	void GenCleanUpCode(Output *out, Env *env);
+	void GenParseCode(Output *out, Env *env, 
+		const DataPtr& data, const ID *size_var);
+
+	int StaticSize(Env *env) const 	{ return type_->StaticSize(env); }
+
+	bool IsDefaultCase() const	{ return ! index_; }
+	void SetBoundaryChecked()	{ type_->SetBoundaryChecked(); }
+
+	bool RequiresByteOrder() const	{ return type_->RequiresByteOrder(); }
+	bool RequiresAnalyzerContext() const;
+
+protected:
+	bool DoTraverse(DataDepVisitor *visitor);
+
+protected:
+	CaseType *case_type_;
+	ExprList *index_;
+	const ID *index_var_;
+};
+
+// Generate a list of "case X:" lines from index_list. Each index
+// expression must be constant foldable.
+void GenCaseStr(ExprList *index_list, Output *out_cc, Env *env);
+
+#endif  // pac_case_h

tools/binpac/src/pac_cclass.h

@@ -0,0 +1,81 @@
+#ifndef pac_cclass_h
+#define pac_cclass_h
+
+class CClass;
+class CClassMember;
+class CClassMethod;
+class CType;
+class CVariable;
+
+typedef vector<CClassMember *> CClassMemberList;
+typedef vector<CClassMethod *> CClassMethodList;
+typedef vector<CVariable *> CVariableList;
+
+#include "pac_common.h"
+
+// Represents a C++ class.
+// 
+// For now we adopt a simple model:
+// 
+// 1. All members have a protected member variable "name_" and a
+// public constant access method "name()".
+// 
+// 2. All methods are public.
+//
+// 3. We do not check repeated names.
+
+class CClass
+{
+public:
+	CClass(const string &class_name);
+
+	void AddMember(CClassMember *member);
+	void AddMethod(CClassMember *method);
+
+	void GenForwardDeclaration(Output *out_h);
+	void GenCode(Output *out_h, Output *out_cc);
+
+protected:
+	string class_name_;
+	CClassMemberList *members_;
+	CClassMethodList *methods_;
+};
+
+class CVariable
+{
+public:
+	CClassMember(const string &name, CType *type);
+
+	string name() const { return name_; }
+	CType *type() const { return type_; }
+
+protected:
+	string name_;
+	CType *type_;
+};
+
+class CClassMember
+{
+public:
+	CClassMember(CVariable *var);
+	void GenCode(Output *out_h, Output *out_cc);
+
+	string decl() const;
+
+protected:
+	CVariable *var_;
+};
+
+class CClassMethod
+{
+public:
+	CClassMethod(CVariable *var, CVariableList *params);
+
+	string decl() const;
+
+protected:
+	CVariable *var_;
+	CVariableList *params_;
+};
+
+#endif  // pac_cclass_h

tools/binpac/src/pac_common.h

@@ -0,0 +1,134 @@
+#ifndef pac_common_h
+#define pac_common_h
+
+#include "pac_utils.h"
+
+#include <vector>
+
+#include <ctype.h>
+#include <string.h>
+#include <stdlib.h>
+
+using namespace std;
+
+extern bool FLAGS_pac_debug;
+extern vector<string> FLAGS_include_directories;
+extern string input_filename;
+extern int line_number;
+
+// Definition of class Object, which is the base class for all objects
+// representing language elements -- identifiers, types, expressions,
+// etc.
+
+class Object
+{
+public:
+	Object()
+		{
+		filename = input_filename;
+		line_num = line_number;
+		location = strfmt("%s:%d", filename.c_str(), line_number);
+		}
+
+	~Object()
+		{
+		}
+
+	const char* Location() const	{ return location.c_str(); }
+
+protected:
+	string filename;
+	int line_num;
+	string location;
+};
+
+class ActionParam;
+class ActionParamType;
+class AnalyzerAction;
+class AnalyzerContextDecl;
+class AnalyzerDecl;
+class AnalyzerElement;
+class ArrayType;
+class Attr;
+class CClass;
+class CType;
+class ConstString;
+class CaseExpr;
+class CaseField;
+class ContextField;
+class DataPtr;
+class Decl;
+class EmbeddedCode;
+class Enum;
+class Env;
+class ExternType;
+class Expr;
+class Field;
+class Function;
+class InputBuffer;
+class LetDef;
+class LetField;
+class ID;
+class Number;
+class Output;
+class PacPrimitive;
+class Param;
+class ParameterizedType;
+class RecordType;
+class RecordField;
+class RecordDataField;
+class RecordPaddingField;
+class RegEx;
+class SeqEnd;
+class StateVar;
+class Type;
+class TypeDecl;
+class WithInputField;
+
+// The ID of the current declaration.
+extern const ID* current_decl_id;
+
+typedef vector<ActionParam*>		ActionParamList;
+typedef vector<AnalyzerAction*> 	AnalyzerActionList;
+typedef vector<AnalyzerElement*>	AnalyzerElementList;
+typedef vector<Attr*> 			AttrList;
+typedef vector<CaseExpr*> 		CaseExprList;
+typedef vector<CaseField*> 		CaseFieldList;
+typedef vector<ContextField*> 		ContextFieldList;
+typedef vector<Decl*> 			DeclList;
+typedef vector<Enum*>			EnumList;
+typedef vector<Expr*> 			ExprList;
+typedef vector<Field*> 			FieldList;
+typedef vector<LetField*> 		LetFieldList;
+typedef vector<Number*>			NumList;
+typedef vector<Param*> 			ParamList;
+typedef vector<RecordField*> 		RecordFieldList;
+typedef vector<StateVar*>		StateVarList;
+
+#define foreach(i, ct, pc) \
+	if ( pc ) \
+		for ( ct::iterator i = (pc)->begin(); i != (pc)->end(); ++i )
+
+#define delete_list(ct, pc) \
+	{ \
+	foreach(delete_list_i, ct, pc) 		\
+		delete *delete_list_i;		\
+	delete pc;				\
+	pc = 0;					\
+	}
+
+// Constants
+const char * const kComputeFrameLength = "compute_frame_length";
+const char * const kFlowBufferClass = "FlowBuffer";
+const char * const kFlowBufferVar = "flow_buffer";
+const char * const kFlowEOF = "FlowEOF";
+const char * const kFlowGap = "NewGap";
+const char * const kInitialBufferLengthFunc = "initial_buffer_length";
+const char * const kNeedMoreData = "need_more_data";
+const char * const kNewData = "NewData";
+const char * const kParseFuncWithBuffer = "ParseBuffer";
+const char * const kParseFuncWithoutBuffer = "Parse";
+const char * const kRefCountClass = "binpac::RefCount";
+const char * const kTypeWithLengthClass = "binpac::TypeWithLength";
+
+#endif  // pac_common_h

tools/binpac/src/pac_conn.cc

@@ -0,0 +1,169 @@
+#include "pac_analyzer.h"
+#include "pac_dataunit.h"
+#include "pac_embedded.h"
+#include "pac_exception.h"
+#include "pac_expr.h"
+#include "pac_flow.h"
+#include "pac_output.h"
+#include "pac_paramtype.h"
+#include "pac_type.h"
+
+#include "pac_conn.h"
+
+ConnDecl::ConnDecl(ID *conn_id, 
+                   ParamList *params,
+                   AnalyzerElementList *elemlist)
+	: AnalyzerDecl(conn_id, CONN, params)
+	{
+	flows_[0] = flows_[1] = 0;
+	AddElements(elemlist);
+	data_type_ = new ParameterizedType(conn_id->clone(), 0);
+	}
+
+ConnDecl::~ConnDecl()
+	{
+	delete flows_[0];
+	delete flows_[1];
+	}
+
+void ConnDecl::AddBaseClass(vector<string> *base_classes) const
+	{
+	base_classes->push_back("binpac::ConnectionAnalyzer"); 
+	} 
+
+void ConnDecl::ProcessFlowElement(AnalyzerFlow *flow_elem)
+	{
+	int flow_index;
+
+	if ( flow_elem->dir() == AnalyzerFlow::UP )
+		flow_index = 0;
+	else
+		flow_index = 1;
+
+	if ( flows_[flow_index] )
+		{
+		throw Exception(flow_elem,
+		                fmt("%sflow already defined", 
+		                    flow_index == 0 ? "up" : "down"));
+		}
+
+	flows_[flow_index] = flow_elem;
+	type_->AddField(flow_elem->flow_field());
+	}
+
+void ConnDecl::ProcessDataUnitElement(AnalyzerDataUnit *dataunit_elem)
+	{
+	throw Exception(
+		dataunit_elem, 
+		"dataunit should be defined in only a flow declaration");
+	}
+
+void ConnDecl::Prepare()
+	{
+	AnalyzerDecl::Prepare();
+
+	flows_[0]->flow_decl()->set_conn_decl(this);
+	flows_[1]->flow_decl()->set_conn_decl(this);
+	}
+
+void ConnDecl::GenPubDecls(Output *out_h, Output *out_cc)
+	{
+	AnalyzerDecl::GenPubDecls(out_h, out_cc);
+	}
+
+void ConnDecl::GenPrivDecls(Output *out_h, Output *out_cc)
+	{
+	AnalyzerDecl::GenPrivDecls(out_h, out_cc);
+	}
+
+void ConnDecl::GenEOFFunc(Output *out_h, Output *out_cc)
+	{
+	string proto = strfmt("%s(bool is_orig)", kFlowEOF);
+
+	out_h->println("void %s;", proto.c_str());
+
+	out_cc->println("void %s::%s", class_name().c_str(), proto.c_str());
+	out_cc->inc_indent();
+	out_cc->println("{");
+
+	out_cc->println("if ( is_orig )");
+	out_cc->inc_indent();
+	out_cc->println("%s->%s();",
+	                env_->LValue(upflow_id),
+	                kFlowEOF);
+	out_cc->dec_indent();
+	out_cc->println("else");
+	out_cc->inc_indent();
+	out_cc->println("%s->%s();",
+	                env_->LValue(downflow_id),
+	                kFlowEOF);
+
+	foreach(i, AnalyzerHelperList, eof_helpers_)
+		{
+		(*i)->GenCode(0, out_cc, this);
+		}
+
+	out_cc->dec_indent();
+	
+	out_cc->println("}");
+	out_cc->dec_indent();
+	out_cc->println("");
+	}
+
+void ConnDecl::GenGapFunc(Output *out_h, Output *out_cc)
+	{
+	string proto = strfmt("%s(bool is_orig, int gap_length)", kFlowGap);
+
+	out_h->println("void %s;", proto.c_str());
+
+	out_cc->println("void %s::%s", class_name().c_str(), proto.c_str());
+	out_cc->inc_indent();
+	out_cc->println("{");
+
+	out_cc->println("if ( is_orig )");
+	out_cc->inc_indent();
+	out_cc->println("%s->%s(gap_length);",
+	                env_->LValue(upflow_id),
+	                kFlowGap);
+	out_cc->dec_indent();
+	out_cc->println("else");
+	out_cc->inc_indent();
+	out_cc->println("%s->%s(gap_length);",
+	                env_->LValue(downflow_id),
+	                kFlowGap);
+	out_cc->dec_indent();
+	
+	out_cc->println("}");
+	out_cc->dec_indent();
+	out_cc->println("");
+	}
+
+void ConnDecl::GenProcessFunc(Output *out_h, Output *out_cc)
+	{
+	string proto = 
+		strfmt("%s(bool is_orig, const_byteptr begin, const_byteptr end)",
+		       kNewData);
+
+	out_h->println("void %s;", proto.c_str());
+
+	out_cc->println("void %s::%s", class_name().c_str(), proto.c_str());
+	out_cc->inc_indent();
+	out_cc->println("{");
+
+	out_cc->println("if ( is_orig )");
+	out_cc->inc_indent();
+	out_cc->println("%s->%s(begin, end);",
+	                env_->LValue(upflow_id),
+	                kNewData);
+	out_cc->dec_indent();
+	out_cc->println("else");
+	out_cc->inc_indent();
+	out_cc->println("%s->%s(begin, end);",
+	                env_->LValue(downflow_id),
+	                kNewData);
+	out_cc->dec_indent();
+	
+	out_cc->println("}");
+	out_cc->dec_indent();
+	out_cc->println("");
+	}

tools/binpac/src/pac_conn.h

@@ -0,0 +1,34 @@
+#ifndef pac_conn_h
+#define pac_conn_h
+
+#include "pac_decl.h"
+#include "pac_analyzer.h"
+
+class ConnDecl : public AnalyzerDecl
+{
+public:
+	ConnDecl(ID *conn_id, ParamList *params, AnalyzerElementList *elemlist);
+	~ConnDecl();
+
+	void Prepare();
+
+	Type* DataType() const	{ return data_type_; }
+
+protected:
+	void AddBaseClass(vector<string> *base_classes) const;
+
+	void GenProcessFunc(Output *out_h, Output *out_cc);
+	void GenGapFunc(Output *out_h, Output *out_cc);
+	void GenEOFFunc(Output *out_h, Output *out_cc);
+
+	void GenPubDecls(Output *out_h, Output *out_cc);
+	void GenPrivDecls(Output *out_h, Output *out_cc);
+
+	void ProcessFlowElement(AnalyzerFlow *flow_elem);
+	void ProcessDataUnitElement(AnalyzerDataUnit *dataunit_elem);
+
+	AnalyzerFlow *flows_[2];
+	Type *data_type_;
+};
+
+#endif  // pac_conn_h

tools/binpac/src/pac_context.cc

@@ -0,0 +1,120 @@
+#include "pac_analyzer.h"
+#include "pac_exception.h"
+#include "pac_exttype.h"
+#include "pac_flow.h"
+#include "pac_id.h"
+#include "pac_output.h"
+#include "pac_param.h"
+#include "pac_paramtype.h"
+#include "pac_type.h"
+#include "pac_utils.h"
+
+#include "pac_context.h"
+
+ContextField::ContextField(ID *id, Type *type)
+	: Field(CONTEXT_FIELD, 
+		TYPE_NOT_TO_BE_PARSED | CLASS_MEMBER | PUBLIC_READABLE, 
+		id, type)
+	{
+	}
+
+AnalyzerContextDecl *AnalyzerContextDecl::current_analyzer_context_ = 0;
+
+namespace {
+	ParamList *ContextFieldsToParams(ContextFieldList *context_fields)
+		{
+		// Convert context fields to parameters
+		ParamList *params = new ParamList();
+		foreach(i, ContextFieldList, context_fields)
+			{
+			ContextField *f = *i;
+			params->push_back(
+				new Param(f->id()->clone(), 
+				f->type()));
+			}
+		return params;
+		}
+} // namespace private
+
+AnalyzerContextDecl::AnalyzerContextDecl(
+		ID *id, 
+		ContextFieldList *context_fields)
+	: TypeDecl(new ID(fmt("Context%s", id->Name())), 
+		ContextFieldsToParams(context_fields), 
+		new DummyType())
+	{
+	context_name_id_ = id;
+	if ( current_analyzer_context_ != 0 )
+		{
+		throw Exception(this, 
+		                fmt("multiple declaration of analyzer context; "
+		                    "the previous one is `%s'",
+		                    current_analyzer_context_->id()->Name()));
+		}
+	else
+		current_analyzer_context_ = this;
+
+	context_fields_ = context_fields;
+
+	param_type_ = new ParameterizedType(id_->clone(), 0);
+
+	flow_buffer_added_ = false;
+
+	DEBUG_MSG("Context type: %s\n", param_type()->class_name().c_str());
+	}
+
+AnalyzerContextDecl::~AnalyzerContextDecl()
+	{
+	delete context_name_id_;
+	delete_list(ContextFieldList, context_fields_);
+	}
+
+void AnalyzerContextDecl::GenForwardDeclaration(Output *out_h)
+	{
+	GenNamespaceBegin(out_h);
+	TypeDecl::GenForwardDeclaration(out_h);
+	}
+
+void AnalyzerContextDecl::GenCode(Output *out_h, Output *out_cc)
+	{
+	GenNamespaceBegin(out_h);
+	GenNamespaceBegin(out_cc);
+	TypeDecl::GenCode(out_h, out_cc);
+	}
+
+void AnalyzerContextDecl::GenNamespaceBegin(Output *out) const
+	{
+	out->println("namespace %s {", context_name_id()->Name());
+	}
+
+void AnalyzerContextDecl::GenNamespaceEnd(Output *out) const
+	{
+	out->println("} // namespace %s", context_name_id()->Name());
+	}
+
+void AnalyzerContextDecl::AddFlowBuffer()
+	{
+	if ( flow_buffer_added_ )
+		return;
+
+	AddParam(new Param(
+		new ID(kFlowBufferVar), 
+		FlowDecl::flow_buffer_type()->Clone()));
+
+	flow_buffer_added_ = true;
+	}
+
+string AnalyzerContextDecl::mb_buffer(Env *env)
+	{
+	// A hack. The orthodox way would be to build an Expr of
+	// context.flow_buffer_var, and then EvalExpr.
+	return fmt("%s->%s()", 
+		env->RValue(analyzer_context_id), 
+		kFlowBufferVar);
+	}
+
+Type *DummyType::DoClone() const
+	{
+	// Fields will be copied in Type::Clone().
+	return new DummyType();
+	}

tools/binpac/src/pac_context.h

@@ -0,0 +1,97 @@
+#ifndef pac_context_h
+#define pac_context_h
+
+#include "pac_common.h"
+#include "pac_field.h"
+#include "pac_type.h"
+#include "pac_typedecl.h"
+
+// AnalyzerContext represents a cookie that an analyzer gives to
+// parse functions of various message types. The cookie is parsed
+// to every parse function (if necessary) as parameter 'binpac_context'.
+// 
+// The members of the cookie is declared through 'analyzer' declarations,
+// such as in:
+//
+// analyzer SunRPC withcontext {
+//        connection:     RPC_Conn;
+//        flow:           RPC_Flow;
+// };      
+//
+// The cookie usually contains the connection and flow in which 
+// the message appears, and the context information can be 
+// accessed as members of the cookie, such as 
+// ``binpac_context.connection''.
+
+class ContextField : public Field
+{
+public:
+	ContextField(ID *id, Type *type);
+};
+
+class AnalyzerContextDecl : public TypeDecl
+{
+public:
+	AnalyzerContextDecl(ID *id, ContextFieldList *context_fields);
+	~AnalyzerContextDecl();
+
+	void AddFlowBuffer();
+
+	const ID *context_name_id() const { return context_name_id_; }
+
+	// The type of analyzer context as a parameter
+	ParameterizedType *param_type() const { return param_type_; }
+
+	void GenForwardDeclaration(Output *out_h);
+	void GenCode(Output *out_h, Output *out_cc);
+
+	void GenNamespaceBegin(Output *out) const;
+	void GenNamespaceEnd(Output *out) const;
+
+private:
+	ID *context_name_id_;
+	ContextFieldList *context_fields_;
+	ParameterizedType *param_type_;
+	bool flow_buffer_added_;
+
+// static members
+public:
+	static AnalyzerContextDecl *current_analyzer_context()
+		{
+		return current_analyzer_context_;
+		}
+
+	static string mb_buffer(Env *env);
+
+private:
+	static AnalyzerContextDecl *current_analyzer_context_;
+};
+
+class DummyType : public Type 
+{
+public:
+	DummyType() : Type(DUMMY) {}
+
+	bool DefineValueVar() const 	{ return false; }
+	string DataTypeStr() const 	{ ASSERT(0); return ""; }
+
+	int StaticSize(Env* env) const 	{ ASSERT(0); return -1; }
+
+	bool ByteOrderSensitive() const { return false; }
+
+	bool IsPointerType() const 	{ ASSERT(0); return false; }
+
+	void DoGenParseCode(Output* out, Env* env,
+			const DataPtr& data, int flags)
+		{ ASSERT(0); }
+
+	// Generate code for computing the dynamic size of the type
+	void GenDynamicSize(Output* out, Env* env, const DataPtr& data)
+		{ ASSERT(0); }
+
+protected:
+	Type *DoClone()	const;
+	void DoMarkIncrementalInput()		{ ASSERT(0); }
+};
+
+#endif  // pac_context_h

tools/binpac/src/pac_cstr.cc

@@ -0,0 +1,127 @@
+#include "pac_cstr.h"
+#include "pac_dbg.h"
+#include "pac_exception.h"
+
+namespace {
+
+class EscapeException
+{
+public:
+	explicit EscapeException(const string &s)
+		{
+		msg_ = s;
+		}
+
+	const string &msg() const	{ return msg_; }
+
+private:
+	string msg_;
+};
+
+// Copied from util.cc of Bro
+int expand_escape(const char*& s)
+	{
+	switch ( *(s++) ) {
+	case 'b': return '\b';
+	case 'f': return '\f';
+	case 'n': return '\n';
+	case 'r': return '\r';
+	case 't': return '\t';
+	case 'a': return '\a';
+	case 'v': return '\v';
+
+	case '0': case '1': case '2': case '3': case '4':
+	case '5': case '6': case '7':
+		{ // \<octal>{1,3}
+		--s;	// put back the first octal digit
+		const char* start = s;
+
+		// Don't increment inside loop control
+		// because if isdigit() is a macro it might
+		// expand into multiple increments ...
+
+		// Here we define a maximum length for escape sequence
+		// to allow easy handling of string like: "^H0" as
+		// "\0100".
+
+		for ( int len = 0; len < 3 && isascii(*s) && isdigit(*s); ++s, ++len)
+			;
+
+		int result;
+		if ( sscanf(start, "%3o", &result) != 1 )
+			{
+			throw EscapeException(fmt("bad octal escape: \"%s", start));
+			result = 0;
+			}
+
+		return result;
+		}
+
+	case 'x':
+		{ /* \x<hex> */
+		const char* start = s;
+
+		// Look at most 2 characters, so that "\x0ddir" -> "^Mdir".
+		for ( int len = 0; len < 2 && isascii(*s) && isxdigit(*s);
+		      ++s, ++len)
+			;
+
+		int result;
+		if ( sscanf(start, "%2x", &result) != 1 )
+			{
+			throw EscapeException(fmt("bad hexadecimal escape: \"%s", start));
+			result = 0;
+			}
+
+		return result;
+		}
+
+	default:
+		return s[-1];
+	}
+	}
+
+}  // private namespace
+
+ConstString::ConstString(const string &s)
+	: str_(s)
+	{
+	// Copied from scan.l of Bro
+	try 
+		{
+		const char* text = str_.c_str();
+		int len = strlen(text) + 1;
+		int i = 0;
+
+		char* s = new char[len];
+
+		// Skip leading quote.
+		for ( ++text; *text; ++text )
+			{
+			if ( *text == '\\' )
+				{
+				++text;	// skip '\'
+				s[i++] = expand_escape(text);
+				--text;	// point to end of sequence
+				}
+			else
+				{
+				s[i++] = *text;
+				}
+			}
+		ASSERT(i < len);
+
+		// Get rid of trailing quote.
+		ASSERT(s[i-1] == '"');
+		s[i-1] = '\0';
+
+		unescaped_ = s;
+		delete [] s;
+		}
+	catch(EscapeException const &e)
+		{
+		// Throw again with the object
+		throw Exception(this, e.msg().c_str());
+		}
+	}
+

tools/binpac/src/pac_cstr.h

@@ -0,0 +1,23 @@
+#ifndef pac_cstr_h
+#define pac_cstr_h
+
+#include "pac_common.h"
+
+class ConstString : public Object
+{
+public:
+	ConstString(const string &s);
+
+	// The string in its escaped form, with surrounding '"'s
+	const string &str() const	{ return str_; }
+	const char *c_str() const	{ return str_.c_str(); }
+
+	// The unescaped string, without surrounding '"'s
+	const string &unescaped() const { return unescaped_; }
+
+private:
+	string str_;
+	string unescaped_;
+};
+
+#endif  // pac_cstr_h

tools/binpac/src/pac_ctype.cc

@@ -0,0 +1,21 @@
+#include "pac_ctype.h"
+
+string CType::DeclareInstance(const string &var) const
+	{
+	return strfmt("%s %s", name().c_str(), var.c_str());
+	}
+
+string CType::DeclareConstReference(const string &var) const
+	{
+	return strfmt("%s const &%s", name().c_str(), var.c_str());
+	}
+
+string CType::DeclareConstPointer(const string &var) const
+	{
+	return strfmt("%s const *%s", name().c_str(), var.c_str());
+	}
+
+string CType::DeclarePointer(const string &var) const
+	{
+	return strfmt("%s *%s", name().c_str(), var.c_str());
+	}

tools/binpac/src/pac_ctype.h

@@ -0,0 +1,23 @@
+#ifndef pac_ctype_h
+#define pac_ctype_h
+
+#include "pac_common.h"
+
+// Represents a C++ type
+class CType
+{
+public:
+	CType(const string &name);
+
+	string name() const 	{ return name_; }
+
+	string DeclareInstance(const string &var) const;
+	string DeclareConstReference(const string &var) const;
+	string DeclareConstPointer(const string &var) const;
+	string DeclarePointer(const string &var) const;
+
+protected:
+	string name_;
+};
+
+#endif  // pac_ctype_h

tools/binpac/src/pac_datadep.cc

@@ -0,0 +1,76 @@
+#include "pac_datadep.h"
+#include "pac_expr.h"
+#include "pac_id.h"
+#include "pac_type.h"
+
+DataDepElement::DataDepElement(DDE_Type type) 
+	: dde_type_(type), in_traversal(false)
+	{ 
+	}
+
+bool DataDepElement::Traverse(DataDepVisitor *visitor)
+	{
+	// Avoid infinite loop
+	if ( in_traversal )
+		return true;
+	if ( ! visitor->PreProcess(this) )
+		return false;
+
+	in_traversal = true;
+	bool cont = DoTraverse(visitor);
+	in_traversal = false;
+
+	if ( ! cont )
+		return false;
+	if ( ! visitor->PostProcess(this) )
+		return false;
+	return true;
+	}
+
+Expr *DataDepElement::expr()
+	{ 
+	return static_cast<Expr *>(this); 
+	}
+
+Type *DataDepElement::type()
+	{ 
+	return static_cast<Type *>(this); 
+	}
+
+bool RequiresAnalyzerContext::PreProcess(DataDepElement *element)
+	{
+	switch ( element->dde_type() ) 
+		{
+		case DataDepElement::EXPR:
+			ProcessExpr(element->expr());
+			break;
+		default:
+			break;
+		}
+
+	// Continue traversal until we know the answer is 'yes'
+	return ! requires_analyzer_context_;
+	}
+
+bool RequiresAnalyzerContext::PostProcess(DataDepElement *element)
+	{
+	return ! requires_analyzer_context_;
+	}
+
+void RequiresAnalyzerContext::ProcessExpr(Expr *expr)
+	{
+	if ( expr->expr_type() == Expr::EXPR_ID )
+		{
+		requires_analyzer_context_ = 
+			(requires_analyzer_context_ || 
+		       	 *expr->id() == *analyzer_context_id ||
+		       	 *expr->id() == *context_macro_id);
+		}
+	}
+
+bool RequiresAnalyzerContext::compute(DataDepElement *element)
+	{
+	RequiresAnalyzerContext visitor;
+	element->Traverse(&visitor);
+	return visitor.requires_analyzer_context_;
+	}

tools/binpac/src/pac_datadep.h

@@ -0,0 +1,71 @@
+#ifndef pac_datadep_h
+#define pac_datadep_h
+
+// To provide a way to traverse through the data dependency graph.
+// That is, to evaluate X, what must be evaluated.
+
+#include "pac_common.h"
+#include "pac_dbg.h"
+
+class DataDepVisitor;
+
+class DataDepElement {
+public:
+	enum DDE_Type {
+		ATTR,
+		CASEEXPR,
+		EXPR,
+		FIELD,
+		INPUT_BUFFER,
+		PARAM,
+		TYPE,
+	};
+
+	DataDepElement(DDE_Type type);
+	virtual ~DataDepElement() {}
+
+	// Returns whether to continue traversal
+	bool Traverse(DataDepVisitor *visitor);
+
+	// Returns whether to continue traversal
+	virtual bool DoTraverse(DataDepVisitor *visitor) = 0;
+
+	DDE_Type dde_type() const	{ return dde_type_; }
+	Expr *expr();
+	Type *type();
+
+protected:
+	DDE_Type dde_type_;
+	bool in_traversal;
+};
+
+class DataDepVisitor {
+public:
+	virtual ~DataDepVisitor() {}
+	// Returns whether to continue traversal
+	virtual bool PreProcess(DataDepElement *element) = 0;
+	virtual bool PostProcess(DataDepElement *element) = 0;
+};
+
+class RequiresAnalyzerContext : public DataDepVisitor {
+public:
+	RequiresAnalyzerContext() : requires_analyzer_context_(false) {}
+
+	// Returns whether to continue traversal
+	bool PreProcess(DataDepElement *element);
+	bool PostProcess(DataDepElement *element);
+
+	bool requires_analyzer_context() const
+		{
+		return requires_analyzer_context_;
+		}
+
+	static bool compute(DataDepElement *element);
+
+protected:
+	void ProcessExpr(Expr *expr);
+
+	bool requires_analyzer_context_;
+};
+
+#endif // pac_datadep_h

tools/binpac/src/pac_dataptr.cc

@@ -0,0 +1,66 @@
+#include "pac_exception.h"
+#include "pac_id.h"
+#include "pac_output.h"
+#include "pac_utils.h"
+
+#include "pac_dataptr.h"
+
+DataPtr::DataPtr(Env* env, const ID* id, const int offset)
+	: id_(id), offset_(offset)
+	{
+	if ( id_ )
+		{
+		if ( ! env->Evaluated(id_) )
+			throw ExceptionIDNotEvaluated(id_);
+
+		if ( offset_ == 0 )
+			ptr_expr_ = strfmt("%s", env->RValue(id_));
+		else
+			ptr_expr_ = strfmt("(%s + %d)", env->RValue(id_), offset_);
+		}
+	else
+		ptr_expr_ = "(null id)";
+	}
+
+int DataPtr::AbsOffset(const ID* base_ptr) const
+	{
+	return ( id() == base_ptr ) ? offset() : -1;
+	}
+
+char* DataPtr::AbsOffsetExpr(Env* env, const ID* base_ptr) const
+	{
+	if ( AbsOffset(base_ptr) >= 0 )
+		return nfmt("%d", offset());
+	else
+		return nfmt("(%s - %s)", ptr_expr(), env->RValue(base_ptr));
+	}
+
+void DataPtr::GenBoundaryCheck(Output* out_cc, Env* env,
+		const char* data_size, const char* data_name) const
+	{
+	ASSERT(id_);
+
+	out_cc->println("// Checking out-of-bound for \"%s\"", data_name);
+	out_cc->println("if ( %s + (%s) > %s )",
+		ptr_expr(),
+		data_size,
+		env->RValue(end_of_data));
+
+	out_cc->inc_indent(); 
+	out_cc->println("{");
+
+	char* data_offset = AbsOffsetExpr(env, begin_of_data); 
+
+	out_cc->println("// Handle out-of-bound condition");
+	out_cc->println("throw ExceptionOutOfBound(\"%s\",", data_name);
+	out_cc->println("	(%s) + (%s), ", 
+		data_offset, data_size);
+	out_cc->println("	(%s) - (%s));", 
+		env->RValue(end_of_data), env->RValue(begin_of_data));
+
+	delete [] data_offset;
+
+	out_cc->println("}");
+	out_cc->dec_indent(); 
+	}
+

tools/binpac/src/pac_dataptr.h

@@ -0,0 +1,47 @@
+#ifndef pac_dataptr_h
+#define pac_dataptr_h
+
+#include <string>
+#include "pac_common.h"
+
+// A data pointer is represented by an data pointer variable
+// plus a constant offset.
+
+class DataPtr
+{
+public:
+	DataPtr(Env* env, const ID* arg_id, const int arg_off);
+
+	DataPtr const &operator=(DataPtr const &x)
+		{
+		id_ = x.id();
+		offset_ = x.offset();
+		ptr_expr_ = x.ptr_expr();
+
+		return *this;
+		}
+
+	const ID* id() const 	{ return id_; }
+	int offset() const 	{ return offset_; }
+
+	const char* ptr_expr() const
+		{
+		ASSERT(id_); 
+		return ptr_expr_.c_str(); 
+		}
+
+	int AbsOffset(const ID* base_ptr) const;
+	char* AbsOffsetExpr(Env* env, const ID* base_ptr) const;
+
+	void GenBoundaryCheck(Output* out, 
+                              Env* env, 
+                              const char* data_size, 
+                              const char* data_name) const;
+
+protected:
+	const ID* id_;
+	int offset_;
+	string ptr_expr_;
+};
+
+#endif  // pac_dataptr_h

tools/binpac/src/pac_dataunit.cc

@@ -0,0 +1,60 @@
+#include "pac_context.h"
+#include "pac_dataunit.h"
+#include "pac_output.h"
+#include "pac_paramtype.h"
+#include "pac_varfield.h"
+
+AnalyzerDataUnit::AnalyzerDataUnit(
+		DataUnitType type, 
+		ID *id, 
+		ExprList *type_params,
+		ExprList *context_params)
+	: AnalyzerElement(DATAUNIT),
+	  type_(type),
+	  id_(id),
+	  type_params_(type_params),
+	  context_params_(context_params) 
+	{
+	data_type_ = new ParameterizedType(id_, type_params_);
+	context_type_ = new ParameterizedType(
+		AnalyzerContextDecl::current_analyzer_context()->id()->clone(),
+		context_params_);
+
+	dataunit_var_field_ = new ParseVarField(
+		Field::CLASS_MEMBER,
+		dataunit_id->clone(),
+		data_type());
+	context_var_field_ = new PrivVarField(
+		analyzer_context_id->clone(),
+		context_type());
+	}
+
+AnalyzerDataUnit::~AnalyzerDataUnit()
+	{
+	delete dataunit_var_field_;
+	delete context_var_field_;
+	}
+
+void AnalyzerDataUnit::Prepare(Env *env)
+	{
+	dataunit_var_field_->Prepare(env);
+	context_var_field_->Prepare(env);
+	}
+
+void AnalyzerDataUnit::GenNewDataUnit(Output *out_cc, Env *env)
+	{
+	out_cc->println("%s = new %s(%s);",
+		env->LValue(dataunit_id),
+		data_type()->class_name().c_str(),
+		data_type()->EvalParameters(out_cc, env).c_str());
+	}
+
+void AnalyzerDataUnit::GenNewContext(Output *out_cc, Env *env)
+	{
+	out_cc->println("%s = new %s(%s);", 
+		env->LValue(analyzer_context_id),
+		context_type()->class_name().c_str(),
+		context_type()->EvalParameters(out_cc, env).c_str());
+	env->SetEvaluated(analyzer_context_id);
+	}
+

tools/binpac/src/pac_dataunit.h

@@ -0,0 +1,49 @@
+#ifndef pac_dataunit_h
+#define pac_dataunit_h
+
+#include "pac_analyzer.h"
+
+// The type and parameters of input data unit of a flow. For instance, the
+// data unit of a DCE/RPC flow is DCE_RPC_PDU.
+
+class AnalyzerDataUnit : public AnalyzerElement
+{
+public:
+	enum DataUnitType { DATAGRAM, FLOWUNIT };
+	AnalyzerDataUnit(
+		DataUnitType type, 
+		ID *id, 
+		ExprList *type_params, 
+		ExprList *context_params);
+	~AnalyzerDataUnit();
+
+	void Prepare(Env *env);
+
+	// Initializes dataunit_id
+	void GenNewDataUnit(Output *out_cc, Env *env);
+	// Initializes analyzer_context_id
+	void GenNewContext(Output *out_cc, Env *env);
+
+	DataUnitType type() const		{ return type_; }
+	const ID *id() const			{ return id_; }
+	ExprList *type_params() const		{ return type_params_; }
+	ExprList *context_params() const	{ return context_params_; }
+
+	ParameterizedType *data_type() const	{ return data_type_; }
+	ParameterizedType *context_type() const	{ return context_type_; }
+
+	Field *dataunit_var_field() const	{ return dataunit_var_field_; }
+	Field *context_var_field() const	{ return context_var_field_; }
+
+private:
+	DataUnitType type_;
+	ID *id_;
+	ExprList *type_params_;
+	ExprList *context_params_;
+	ParameterizedType *data_type_;
+	ParameterizedType *context_type_;
+	Field *dataunit_var_field_;
+	Field *context_var_field_;
+};
+
+#endif // pac_dataunit_h

tools/binpac/src/pac_dbg.h

@@ -0,0 +1,14 @@
+/* $Id: pac_dbg.h 3265 2006-06-09 21:16:12Z rpang $ */
+
+#ifndef pac_dbg_h
+#define pac_dbg_h
+
+#include <assert.h>
+#include <stdio.h>
+
+extern bool FLAGS_pac_debug;
+
+#define ASSERT(x)	assert(x)
+#define DEBUG_MSG(x...)	if ( FLAGS_pac_debug ) fprintf(stderr, x)
+
+#endif /* pac_dbg_h */

tools/binpac/src/pac_decl-inl.h

@@ -0,0 +1,6 @@
+#ifndef pac_decl_inl_h
+#define pac_decl_inl_h
+
+#include "pac_id.h"
+
+#endif  // pac_decl_inl_h

tools/binpac/src/pac_decl.cc

@@ -0,0 +1,191 @@
+#include "pac_attr.h"
+#include "pac_context.h"
+#include "pac_dataptr.h"
+#include "pac_embedded.h"
+#include "pac_exception.h"
+#include "pac_expr.h"
+#include "pac_exttype.h"
+#include "pac_id.h"
+#include "pac_output.h"
+#include "pac_param.h"
+#include "pac_record.h"
+#include "pac_type.h"
+#include "pac_utils.h"
+
+#include "pac_decl.h"
+
+DeclList *Decl::decl_list_ = 0;
+Decl::DeclMap Decl::decl_map_;
+
+Decl::Decl(ID* id, DeclType decl_type)
+	: id_(id), decl_type_(decl_type), attrlist_(0)
+	{
+	decl_map_[id_] = this;
+	if ( ! decl_list_ )
+		decl_list_ = new DeclList();
+	decl_list_->push_back(this);
+
+	DEBUG_MSG("Finished Decl %s\n", id_->Name());
+
+	analyzer_context_ = 0;
+	}
+
+Decl::~Decl()
+	{
+	delete id_;
+	delete_list(AttrList, attrlist_);
+	}
+
+void Decl::AddAttrs(AttrList* attrs)
+	{
+	if ( ! attrs )
+		return;
+	if ( ! attrlist_ )
+		attrlist_ = new AttrList();
+	foreach ( i, AttrList, attrs )
+		{
+		attrlist_->push_back(*i);
+		ProcessAttr(*i);
+		}
+	}
+
+void Decl::ProcessAttr(Attr *attr)
+	{
+	throw Exception(attr, "unhandled attribute");
+	}
+
+void Decl::SetAnalyzerContext()
+	{
+	analyzer_context_ = 
+		AnalyzerContextDecl::current_analyzer_context();
+	if ( ! analyzer_context_ )
+		{
+		throw Exception(this, 
+		                "analyzer context not defined");
+		}
+	}
+
+void Decl::ProcessDecls(Output *out_h, Output *out_cc)
+	{
+	if ( ! decl_list_ )
+		return;
+
+	foreach(i, DeclList, decl_list_)
+		{
+		Decl *decl = *i;
+		current_decl_id = decl->id();
+		decl->Prepare();
+		}
+
+	foreach(i, DeclList, decl_list_)
+		{
+		Decl *decl = *i;
+		current_decl_id = decl->id();
+		decl->GenExternDeclaration(out_h);
+		}
+
+	out_h->println("namespace binpac {\n");
+	out_cc->println("namespace binpac {\n");
+
+	AnalyzerContextDecl *analyzer_context =
+		AnalyzerContextDecl::current_analyzer_context();
+
+	foreach(i, DeclList, decl_list_)
+		{
+		Decl *decl = *i;
+		current_decl_id = decl->id();
+		decl->GenForwardDeclaration(out_h);
+		}
+
+	if ( analyzer_context )
+		analyzer_context->GenNamespaceEnd(out_h);
+
+	out_h->println("");
+
+	foreach(i, DeclList, decl_list_)
+		{
+		Decl *decl = *i;
+		current_decl_id = decl->id();
+		decl->GenCode(out_h, out_cc);
+		}
+
+	if ( analyzer_context )
+		{
+		analyzer_context->GenNamespaceEnd(out_h);
+		analyzer_context->GenNamespaceEnd(out_cc);
+		}
+
+	out_h->println("}  // namespace binpac");
+	out_cc->println("}  // namespace binpac");
+	}
+
+Decl* Decl::LookUpDecl(const ID* id)
+	{
+	DeclMap::iterator it = decl_map_.find(id);
+	if ( it == decl_map_.end() )
+		return 0;
+	return it->second;
+	}
+
+int HelperDecl::helper_id_seq = 0;
+
+HelperDecl::HelperDecl(HelperType helper_type, 
+                       ID* context_id, 
+                       EmbeddedCode* code)
+ 	: Decl(new ID(fmt("helper_%d", ++helper_id_seq)), HELPER), 
+	  helper_type_(helper_type),
+	  context_id_(context_id),
+	  code_(code)
+	{
+	}
+
+HelperDecl::~HelperDecl()
+	{
+	delete context_id_;
+	delete code_;
+	}
+
+void HelperDecl::Prepare()
+	{
+	// Do nothing
+	}
+
+void HelperDecl::GenExternDeclaration(Output *out_h)
+	{
+	if ( helper_type_ == EXTERN )
+		code_->GenCode(out_h, global_env());
+	}
+
+void HelperDecl::GenCode(Output *out_h, Output *out_cc)
+	{
+	Env *env = global_env();
+
+#if 0
+	if ( context_id_ )
+		{
+		Decl *decl = Decl::LookUpDecl(context_id_);
+		if ( ! decl )
+			{
+			throw Exception(context_id_, 
+			                fmt("cannot find declaration for %s", 
+			                    context_id_->Name()));
+			}
+		env = decl->env();
+		if ( ! env )
+			{
+			throw Exception(context_id_,
+			                fmt("not a type or analyzer: %s",
+			                    context_id_->Name()));
+			}
+		}
+#endif
+
+	if ( helper_type_ == HEADER )
+		code_->GenCode(out_h, env);
+	else if ( helper_type_ == CODE )
+		code_->GenCode(out_cc, env);
+	else if ( helper_type_ == EXTERN )
+		; // do nothing
+	else
+		ASSERT(0);
+	}

tools/binpac/src/pac_decl.h

@@ -0,0 +1,81 @@
+#ifndef pac_decl_h
+#define pac_decl_h
+
+#include "pac_common.h"
+#include "pac_id.h"
+
+class Decl : public Object
+{
+public:
+	// Note: ANALYZER is not for AnalyzerDecl (which is an
+	// abstract class) , but for AnalyzerContextDecl.
+	enum DeclType { ENUM, LET, TYPE, FUNC, CONN, FLOW, ANALYZER, HELPER, REGEX };
+
+	Decl(ID *id, DeclType decl_type);
+	virtual ~Decl();
+
+	const ID *id() const		{ return id_; }
+	DeclType decl_type() const	{ return decl_type_; }
+	AnalyzerContextDecl *analyzer_context() const 
+		{ return analyzer_context_; }
+
+	// NULL except for TypeDecl or AnalyzerDecl
+	virtual Env *env() const	{ return 0; }
+
+	virtual void Prepare() = 0;
+
+	// Generate declarations out of the "binpac" namespace
+	virtual void GenExternDeclaration(Output *out_h) { /* do nothing */ }
+
+	// Generate declarations before definition of classes
+	virtual void GenForwardDeclaration(Output *out_h) = 0;
+
+	virtual void GenCode(Output *out_h, Output *out_cc) = 0;
+
+	void TakeExprList();
+	void AddAttrs(AttrList *attrlist);
+	void SetAnalyzerContext();
+
+protected:
+	virtual void ProcessAttr(Attr *a);
+
+	ID *id_;
+	DeclType decl_type_;
+	AttrList *attrlist_;
+	ExprList *expr_list_;
+	AnalyzerContextDecl *analyzer_context_;
+
+public:
+	static void ProcessDecls(Output *out_h, Output *out_cc);
+	static Decl *LookUpDecl(const ID *id);
+
+private:
+	static DeclList *decl_list_;
+	typedef map<const ID *, Decl*, ID_ptr_cmp> DeclMap;
+	static DeclMap decl_map_;
+};
+
+class HelperDecl : public Decl
+{
+public:
+	enum HelperType {
+		HEADER, CODE, EXTERN,
+	};
+	HelperDecl(HelperType type, ID *context_id, EmbeddedCode *code);
+	~HelperDecl();
+
+	void Prepare();
+	void GenExternDeclaration(Output *out_h);
+	void GenForwardDeclaration(Output *out_h) { /* do nothing */ }
+	void GenCode(Output *out_h, Output *out_cc);
+
+private:
+	HelperType helper_type_;
+	ID *context_id_;
+	ID *helper_id_;
+	EmbeddedCode *code_;
+
+	static int helper_id_seq;
+};
+
+#endif  // pac_decl_h

tools/binpac/src/pac_embedded.cc

@@ -0,0 +1,82 @@
+#include "pac_id.h"
+#include "pac_primitive.h"
+#include "pac_output.h"
+
+#include "pac_embedded.h"
+
+EmbeddedCodeSegment::EmbeddedCodeSegment(const string &s)
+	: s_(s), primitive_(0)
+	{
+	}
+
+EmbeddedCodeSegment::EmbeddedCodeSegment(PacPrimitive *primitive)
+	: s_(""), primitive_(primitive)
+	{
+	}
+
+EmbeddedCodeSegment::~EmbeddedCodeSegment()
+	{
+	delete primitive_;
+	}
+
+string EmbeddedCodeSegment::ToCode(Env *env)
+	{
+	if ( primitive_ &&  s_.empty() )
+		s_ = primitive_->ToCode(env);
+	return s_;
+	}
+
+EmbeddedCode::EmbeddedCode()
+	{
+	segments_ = new EmbeddedCodeSegmentList();
+	}
+
+EmbeddedCode::~EmbeddedCode()
+	{
+	delete_list(EmbeddedCodeSegmentList, segments_);
+	}
+
+void EmbeddedCode::Append(int atom)
+	{
+	current_segment_ += static_cast<char>(atom);
+	}
+
+void EmbeddedCode::Append(const char *str)
+	{
+	current_segment_ += str;
+	}
+
+void EmbeddedCode::Append(PacPrimitive *primitive)
+	{
+	if ( ! current_segment_.empty() )
+		{
+		segments_->push_back(new EmbeddedCodeSegment(current_segment_));
+		current_segment_ = "";
+		}
+	segments_->push_back(new EmbeddedCodeSegment(primitive));
+	}
+
+void EmbeddedCode::GenCode(Output *out, Env *env)
+	{
+	if ( ! current_segment_.empty() )
+		{
+		segments_->push_back(new EmbeddedCodeSegment(current_segment_));
+		current_segment_ = "";
+		}
+
+	// TODO: return to the generated file after embedded code
+	// out->print("#line %d \"%s\"\n", line_num, filename.c_str());
+
+	// Allow use of RValue for undefined ID, in which case the
+	// ID's name is used as its RValue
+	env->set_allow_undefined_id(true);
+
+	foreach(i, EmbeddedCodeSegmentList, segments_)
+		{
+		EmbeddedCodeSegment *segment = *i;
+		out->print("%s", segment->ToCode(env).c_str());
+		}
+
+	env->set_allow_undefined_id(false);
+	out->print("\n");
+	}

tools/binpac/src/pac_embedded.h

@@ -0,0 +1,42 @@
+#ifndef pac_embedded_h
+#define pac_embedded_h
+
+#include "pac_common.h"
+
+class EmbeddedCodeSegment
+{
+public:
+	explicit EmbeddedCodeSegment(const string &s);
+	explicit EmbeddedCodeSegment(PacPrimitive *primitive);
+	~EmbeddedCodeSegment();
+
+	string ToCode(Env *env);
+
+private:
+	string s_;
+	PacPrimitive *primitive_;
+};
+
+typedef vector<EmbeddedCodeSegment *> EmbeddedCodeSegmentList;
+
+class EmbeddedCode : public Object
+{
+public:
+	EmbeddedCode();
+	~EmbeddedCode();
+
+	// Append a character
+	void Append(int atom);
+	void Append(const char *str);
+
+	// Append a PAC primitive
+	void Append(PacPrimitive *primitive);
+
+	void GenCode(Output *out, Env *env);
+
+private:
+	string current_segment_;
+	EmbeddedCodeSegmentList *segments_;
+};
+
+#endif  // pac_embedded_h

tools/binpac/src/pac_enum.cc

@@ -0,0 +1,70 @@
+#include "pac_exception.h"
+#include "pac_enum.h"
+#include "pac_expr.h"
+#include "pac_exttype.h"
+#include "pac_output.h"
+#include "pac_typedecl.h"
+
+Enum::Enum(ID* id, Expr* expr)
+	: id_(id), expr_(expr)
+	{
+	}
+
+Enum::~Enum()
+	{
+	delete id_;
+	delete expr_;
+	}
+
+void Enum::GenHeader(Output* out_h, int *pval)
+	{
+	ASSERT(pval);
+	if ( expr_ )
+		{
+		if ( ! expr_->ConstFold(global_env(), pval) )
+			throw ExceptionNonConstExpr(expr_);
+		out_h->println("%s = %d,", id_->Name(), *pval);
+		}
+	else
+		out_h->println("%s,", id_->Name());
+	global_env()->AddConstID(id_, *pval);
+	}
+
+EnumDecl::EnumDecl(ID *id, EnumList *enumlist)
+	: Decl(id, ENUM), enumlist_(enumlist) 
+	{
+	ID *type_id = id->clone();
+	datatype_ = new ExternType(type_id, ExternType::NUMBER);
+	extern_typedecl_ = new TypeDecl(type_id, 0, datatype_);
+	}
+
+EnumDecl::~EnumDecl()
+	{ 
+	delete_list(EnumList, enumlist_); 
+	delete extern_typedecl_;
+	}
+
+void EnumDecl::Prepare() 
+	{ 
+	// Do nothing 
+	}
+
+void EnumDecl::GenForwardDeclaration(Output *out_h)
+	{ 
+	out_h->println("enum %s {", id_->Name());
+	out_h->inc_indent();
+	int c = 0;
+	foreach(i, EnumList, enumlist_)
+		{
+		(*i)->GenHeader(out_h, &c);
+		++c;
+		}
+	out_h->dec_indent();
+	out_h->println("};");
+	}
+
+void EnumDecl::GenCode(Output* out_h, Output* /* out_cc */)
+	{
+	// Do nothing 
+	}
+

tools/binpac/src/pac_enum.h

@@ -0,0 +1,37 @@
+#ifndef pac_enum_h
+#define pac_enum_h
+
+#include "pac_decl.h"
+
+class Enum
+{
+public:
+	Enum(ID *id, Expr *expr = 0);
+	~Enum();
+
+	void GenHeader(Output *out_h, int *pval);
+
+private:
+	ID *id_;
+	Expr *expr_;
+};
+
+class EnumDecl : public Decl
+{
+public:
+	EnumDecl(ID *id, EnumList *enumlist);
+	~EnumDecl();
+
+	Type *DataType() const	{ return datatype_; }
+
+	void Prepare();
+	void GenForwardDeclaration(Output *out_h);
+	void GenCode(Output *out_h, Output *out_cc);
+
+private:
+	EnumList *enumlist_;
+	Type *datatype_;
+	TypeDecl *extern_typedecl_;
+};
+
+#endif // pac_enum_h

tools/binpac/src/pac_exception.cc

@@ -0,0 +1,70 @@
+#include "pac_exception.h"
+#include "pac_expr.h"
+#include "pac_id.h"
+#include "pac_utils.h"
+
+Exception::Exception(const Object* o, const char* msg)
+	{
+	if ( o )
+		{
+		msg_ = o->Location();
+		msg_ += ": error : ";
+		}
+	if ( msg )
+		msg_ += msg;
+	if ( FLAGS_pac_debug ) 
+		{
+		DEBUG_MSG("Exception: %s\n", msg_.c_str());
+		abort();
+		}
+	}
+
+ExceptionIDNotFound::ExceptionIDNotFound(const ID* id)
+	: Exception(id), id_(id) 
+	{
+	append(fmt("`%s' undeclared", id_->Name()));
+	}
+
+ExceptionIDRedefinition::ExceptionIDRedefinition(const ID* id)
+	: Exception(id), id_(id) 
+	{
+	append(fmt("`%s' redefined", id_->Name()));
+	}
+
+ExceptionIDNotEvaluated::ExceptionIDNotEvaluated(const ID* id)
+	: Exception(id), id_(id) 
+	{
+	append(fmt("ID `%s' not evaluated before used", id->Name()));
+	}
+
+ExceptionIDNotField::ExceptionIDNotField(const ID* id)
+	: Exception(id), id_(id) 
+	{
+	append(fmt("ID `%s' is not a field", id_->Name()));
+	}
+
+ExceptionMemberNotFound::ExceptionMemberNotFound(const ID* type_id, 
+                                                 const ID *member_id)
+	: Exception(member_id), type_id_(type_id), member_id_(member_id)
+	{
+	append(fmt("type %s does not have member `%s'",
+	           type_id_->Name(), member_id_->Name()));
+	}
+
+ExceptionCyclicDependence::ExceptionCyclicDependence(const ID* id)
+	: Exception(id), id_(id) 
+	{
+	append(fmt("cyclic dependence through `%s'", id_->Name()));
+	}
+
+ExceptionPaddingError::ExceptionPaddingError(const Object* o, const char* msg)
+	: Exception(o)
+	{
+	append(msg);
+	}
+
+ExceptionNonConstExpr::ExceptionNonConstExpr(const Expr* expr)
+	: Exception(expr)
+	{
+	append(fmt("Expression `%s' is not constant", expr->orig()));
+	}

tools/binpac/src/pac_exception.h

@@ -0,0 +1,97 @@
+// $Id: pac_exception.h 3225 2006-06-08 00:00:01Z vern $
+
+#ifndef pac_exception_h
+#define pac_exception_h
+
+#include <string>
+using namespace std;
+
+#include "pac_common.h"
+
+class Exception
+{
+public:
+	Exception(const Object* o, const char* msg = 0);
+
+	const char* msg() const 	{ return msg_.c_str(); }
+	void append(const char* s) 	{ msg_ += s; }
+
+private:
+	string msg_;
+};
+
+class ExceptionIDNotFound : public Exception
+{
+public:
+	ExceptionIDNotFound(const ID* id);
+	const ID* id() const { return id_; }
+
+private:
+	const ID* id_;
+};
+
+class ExceptionIDRedefinition : public Exception
+{
+public:
+	ExceptionIDRedefinition(const ID* id);
+	const ID* id() const { return id_; }
+
+private:
+	const ID* id_;
+};
+
+class ExceptionIDNotEvaluated : public Exception
+{
+public:
+	ExceptionIDNotEvaluated(const ID* id);
+	const ID* id() const { return id_; }
+
+private:
+	const ID* id_;
+};
+
+class ExceptionCyclicDependence : public Exception
+{
+public:
+	ExceptionCyclicDependence(const ID* id);
+	const ID* id() const { return id_; }
+
+private:
+	const ID* id_;
+};
+
+class ExceptionPaddingError : public Exception
+{
+public:
+	ExceptionPaddingError(const Object* o, const char* msg);
+};
+
+class ExceptionIDNotField : public Exception
+{
+public:
+	ExceptionIDNotField(const ID* id);
+	const ID* id() const { return id_; }
+
+private:
+	const ID* id_;
+};
+
+class ExceptionMemberNotFound : public Exception
+{
+public:
+	ExceptionMemberNotFound(const ID* type_id, const ID *member_id);
+
+private:
+	const ID *type_id_, *member_id_;
+};
+
+class ExceptionNonConstExpr : public Exception
+{
+public:
+	ExceptionNonConstExpr(const Expr* expr);
+
+private:
+	const Expr *expr;
+};
+
+#endif /* pac_exception_h */

tools/binpac/src/pac_expr.def

@@ -0,0 +1,34 @@
+EXPR_DEF(EXPR_ID, 		0, "%s")
+EXPR_DEF(EXPR_NUM,		0, "%s")
+EXPR_DEF(EXPR_CSTR,		0, "%s")
+EXPR_DEF(EXPR_REGEX,		0, "REGEX(%s)")
+EXPR_DEF(EXPR_SUBSCRIPT,	2, "@element@(%s[%s])") 
+EXPR_DEF(EXPR_MEMBER,		2, "@%s->%s@") 
+EXPR_DEF(EXPR_PAREN,		1, " ( %s ) ") 
+EXPR_DEF(EXPR_CALL,		1, "%s(%s)")
+EXPR_DEF(EXPR_CALLARGS,		-1, "@custom@")
+EXPR_DEF(EXPR_SIZEOF,		1, "@sizeof(%s)@")
+EXPR_DEF(EXPR_OFFSETOF,		1, "@offsetof(%s)@")
+EXPR_DEF(EXPR_NEG,		1, "-%s") 
+EXPR_DEF(EXPR_PLUS,		2, "%s + %s")
+EXPR_DEF(EXPR_MINUS,		2, "%s - %s")
+EXPR_DEF(EXPR_TIMES,		2, "%s * %s")
+EXPR_DEF(EXPR_DIV,		2, "%s / %s")
+EXPR_DEF(EXPR_MOD,		2, "%s %% %s")
+EXPR_DEF(EXPR_BITNOT,		1, "~%s")
+EXPR_DEF(EXPR_BITAND,		2, "%s & %s")
+EXPR_DEF(EXPR_BITOR,		2, "%s | %s")
+EXPR_DEF(EXPR_BITXOR,		2, "%s ^ %s")
+EXPR_DEF(EXPR_LSHIFT,		2, "%s << %s")
+EXPR_DEF(EXPR_RSHIFT,		2, "%s >> %s")
+EXPR_DEF(EXPR_EQUAL,		2, "%s == %s")
+EXPR_DEF(EXPR_NEQ,		2, "%s != %s")
+EXPR_DEF(EXPR_GE,		2, "%s >= %s")
+EXPR_DEF(EXPR_LE,		2, "%s <= %s")
+EXPR_DEF(EXPR_GT,		2, "%s > %s")
+EXPR_DEF(EXPR_LT,		2, "%s < %s")
+EXPR_DEF(EXPR_NOT,		1, "! %s")
+EXPR_DEF(EXPR_AND,		2, "%s && %s")
+EXPR_DEF(EXPR_OR,		2, "%s || %s")
+EXPR_DEF(EXPR_COND,		3, "%s ? %s : %s")
+EXPR_DEF(EXPR_CASE,		-1, "@custom@")

tools/binpac/src/pac_expr.h

@@ -0,0 +1,139 @@
+#ifndef pac_expr_h
+#define pac_expr_h
+
+#include "pac_common.h"
+#include "pac_datadep.h"
+
+class CaseExpr;
+
+class Expr : public Object, public DataDepElement
+{
+public:
+	enum ExprType {
+#		define EXPR_DEF(type, x, y) type,
+#		include "pac_expr.def"
+#		undef EXPR_DEF
+	};
+
+	void init();
+
+	Expr(ID *id);
+	Expr(Number *num);
+	Expr(ConstString *s);
+	Expr(RegEx *regex);
+	Expr(ExprList *args);	// for EXPR_CALLARGS
+	Expr(Expr *index, CaseExprList *cases);
+
+	Expr(ExprType type, Expr *op1);
+	Expr(ExprType type, Expr *op1, Expr *op2);
+	Expr(ExprType type, Expr *op1, Expr *op2, Expr *op3);
+
+	virtual ~Expr();
+
+	const char *orig() const	{ return orig_.c_str(); }
+	const ID *id() const 		{ return id_; }
+	const char *str() const		{ return str_.c_str(); }
+	ExprType expr_type() const	{ return expr_type_; }
+
+	void AddCaseExpr(CaseExpr *case_expr);
+
+	// Returns the data "type" of the expression. Here we only
+	// do a serious job for the EXPR_MEMBER and EXPR_SUBSCRIPT
+	// operators. For arithmetic operations, we fall back
+	// to "int".
+	Type *DataType(Env *env) const;
+	string DataTypeStr(Env *env) const;
+
+	// Note: EvalExpr() may generate C++ statements in order to evaluate 
+	// variables in the expression, so the following is wrong:
+	//
+	// out->print("int x = ");
+	// out->println("%s", expr->EvalExpr(out, env));
+	//
+	// While putting them together is right:
+	//
+	// out->println("int x = %s", expr->EvalExpr(out, env));
+	//
+	const char *EvalExpr(Output *out, Env *env);
+
+	// force evaulation of IDs contained in this expression;
+	// necessary with case expr and conditional let fields (&if)
+	// for correct parsing of fields
+	void ForceIDEval(Output *out_cc, Env *env);
+
+	// Returns the set_* function of the expression. 
+	// The expression must be of form ID or x.ID.
+	string SetFunc(Output *out, Env *env);
+
+	// Returns true if the expression folds to an integer
+	// constant with env, and puts the constant in *pn.
+	//
+	bool ConstFold(Env *env, int *pn) const;
+
+	// Whether id is referenced in the expression
+	bool HasReference(const ID *id) const;
+
+	// Suppose the data for type might be incomplete, what is
+	// the minimal number of bytes from data head required to
+	// compute the expression? For example, how many bytes of frame
+	// header do we need to determine the length of the frame?
+	//
+	// The parameter <env> points to the Env of a type.
+	// 
+	// Returns -1 if the number is not a constant.
+	//
+	int MinimalHeaderSize(Env *env);
+
+	// Whether evaluation of the expression requires the analyzer context
+	bool RequiresAnalyzerContext() const;
+
+protected:
+	bool DoTraverse(DataDepVisitor *visitor);
+
+private:
+	ExprType expr_type_;
+
+	int num_operands_;
+	Expr *operand_[3];
+
+	ID *id_;		// EXPR_ID
+	Number *num_;		// EXPR_NUM
+	ConstString *cstr_;	// EXPR_CSTR
+	RegEx *regex_;		// EXPR_REGEX
+	ExprList *args_;	// EXPR_CALLARGS
+	CaseExprList *cases_;	// EXPR_CASE
+
+	string str_;		// value string
+	string orig_;		// original string for debugging info
+
+	void GenStrFromFormat(Env *env);
+	void GenEval(Output *out, Env *env);
+	void GenCaseEval(Output *out_cc, Env *env);
+};
+
+string OrigExprList(ExprList *exprlist);
+string EvalExprList(ExprList *exprlist, Output *out, Env *env);
+
+// An entry of the case expression, consisting of one or more constant
+// expressions for the case index and a value expression.
+class CaseExpr : public Object, public DataDepElement
+{
+public:
+	CaseExpr(ExprList *index, Expr *value);
+	virtual ~CaseExpr();
+
+	ExprList *index() const 	{ return index_; }
+	Expr *value() const 		{ return value_; }
+
+	bool HasReference(const ID *id) const;
+	bool RequiresAnalyzerContext() const;
+
+protected:
+	bool DoTraverse(DataDepVisitor *visitor);
+
+private:
+	ExprList *index_;
+	Expr *value_;
+};
+
+#endif  // pac_expr_h

tools/binpac/src/pac_externtype.def

@@ -0,0 +1,15 @@
+EXTERNTYPE(bool, bool, NUMBER)
+EXTERNTYPE(int, int, NUMBER)
+EXTERNTYPE(double, double, NUMBER)
+EXTERNTYPE(string, string, PLAIN)
+EXTERNTYPE(void, void, PLAIN)
+EXTERNTYPE(voidptr, void, POINTER)
+EXTERNTYPE(nullptr, nullptr, PLAIN)
+EXTERNTYPE(bytearray, bytearray, PLAIN)
+EXTERNTYPE(const_charptr, const_charptr, PLAIN)
+EXTERNTYPE(const_byteptr, const_byteptr, PLAIN)
+// EXTERNTYPE(const_byteseg, const_byteseg, PLAIN)
+EXTERNTYPE(const_bytestring, const_bytestring, PLAIN)
+// EXTERNTYPE(bytestring, bytestring, PLAIN)
+EXTERNTYPE(re_matcher, re_matcher, PLAIN)
+EXTERNTYPE(flowbuffer, FlowBuffer, POINTER)

tools/binpac/src/pac_exttype.cc

@@ -0,0 +1,76 @@
+#include "pac_exttype.h"
+#include "pac_id.h"
+#include "pac_decl.h"
+
+bool ExternType::DefineValueVar() const
+	{
+	return true;
+	}
+
+string ExternType::DataTypeStr() const
+	{ 
+	switch ( ext_type_ )
+		{
+		case PLAIN:
+		case NUMBER:
+			return id_->Name(); 
+		case POINTER:
+			return string(id_->Name()) + " *"; 
+		default:
+			ASSERT(0);
+			return "";
+		}
+	}
+
+int ExternType::StaticSize(Env* env) const
+	{
+	ASSERT(0);
+	return -1;
+	}
+
+bool ExternType::ByteOrderSensitive() const 
+	{ 
+	return false;
+	}
+
+string ExternType::EvalMember(const ID *member_id) const
+	{
+	return strfmt("%s%s", 
+		ext_type_ == POINTER ? "->" : ".",
+		member_id->Name());
+	}
+
+void ExternType::DoGenParseCode(Output* out, Env* env, const DataPtr& data, int flags)
+	{
+	ASSERT(0);
+	}
+
+void ExternType::GenDynamicSize(Output* out, Env* env, const DataPtr& data)
+	{
+	ASSERT(0);
+	}
+
+Type *ExternType::DoClone() const
+	{ 
+	return new ExternType(id_->clone(), ext_type_); 
+	}
+
+// Definitions of pre-defined external types
+
+#define EXTERNTYPE(name, ctype, exttype) ExternType *extern_type_##name = 0;
+#include "pac_externtype.def"
+#undef EXTERNTYPE
+
+void ExternType::static_init()
+	{
+	ID *id;
+	// TypeDecl *decl;
+	// decl = new TypeDecl(id, 0, extern_type_##name);
+
+#define EXTERNTYPE(name, ctype, exttype) \
+	id = new ID(#ctype); \
+	extern_type_##name = new ExternType(id, ExternType::exttype); \
+	Type::AddPredefinedType(#name, extern_type_##name); 
+#include "pac_externtype.def"
+#undef EXTERNTYPE
+	}

tools/binpac/src/pac_exttype.h

@@ -0,0 +1,47 @@
+#ifndef pac_exttype_h
+#define pac_exttype_h
+
+#include "pac_type.h"
+
+// ExternType represent external C++ types that are not defined in
+// PAC specification (therefore they cannot appear in data layout
+// spefication, e.g., in a record field). The type name is copied
+// literally to the compiled code. 
+
+class ExternType : public Type
+{
+public:
+	enum EXTType { PLAIN, NUMBER, POINTER };
+	ExternType(const ID *id, EXTType ext_type)
+		: Type(EXTERN), 
+		  id_(id), 
+		  ext_type_(ext_type) {}
+
+	bool DefineValueVar() const;
+	string DataTypeStr() const;
+	int StaticSize(Env *env) const;
+	bool ByteOrderSensitive() const;
+
+	string EvalMember(const ID *member_id) const;
+	bool IsNumericType() const		{ return ext_type_ == NUMBER; }
+	bool IsPointerType() const		{ return ext_type_ == POINTER; }
+
+protected:
+	void DoGenParseCode(Output *out, Env *env, const DataPtr& data, int flags);
+	void GenDynamicSize(Output *out, Env *env, const DataPtr& data);
+
+	Type *DoClone() const;
+
+private:
+	const ID *id_;
+	EXTType ext_type_;
+
+public:
+	static void static_init();
+};
+
+#define EXTERNTYPE(name, ctype, exttype) extern ExternType *extern_type_##name;
+#include "pac_externtype.def"
+#undef EXTERNTYPE
+
+#endif  // pac_exttype_h

tools/binpac/src/pac_field.cc

@@ -0,0 +1,143 @@
+#include "pac_attr.h"
+#include "pac_common.h"
+#include "pac_exception.h"
+#include "pac_field.h"
+#include "pac_id.h"
+#include "pac_type.h"
+
+Field::Field(FieldType tof, int flags, ID *id, Type *type)
+	: DataDepElement(DataDepElement::FIELD),
+	  tof_(tof), flags_(flags), id_(id), type_(type)
+	{
+	decl_id_ = current_decl_id;
+	field_id_str_ = strfmt("%s:%s", decl_id()->Name(), id_->Name());
+	attrs_ = 0;
+	}
+
+Field::~Field()
+	{
+	delete id_;
+	delete type_;
+	delete_list(AttrList, attrs_);
+	}
+
+void Field::AddAttr(AttrList* attrs)
+	{
+	if ( ! attrs_ )
+		{
+		attrs_ = attrs;
+		}
+	else
+		{
+		attrs_->insert(attrs_->end(), attrs->begin(), attrs->end());
+		delete attrs;
+		}
+
+	foreach(i, AttrList, attrs)
+		ProcessAttr(*i);
+	}
+
+void Field::ProcessAttr(Attr *a)
+	{
+	switch ( a->type() )
+		{
+		case ATTR_IF:
+			if ( tof() != LET_FIELD &&
+			     tof() != WITHINPUT_FIELD )
+				{
+				throw Exception(a, 
+					"&if can only be applied to a "
+					"let field");
+				}
+			break;
+		default:
+			break;
+		}
+
+	if ( type_ )
+		type_->ProcessAttr(a);
+	}
+
+bool Field::anonymous_field() const
+	{
+	return type_ && type_->anonymous_value_var();
+	}
+
+int Field::ValueVarType() const
+	{
+	if ( flags_ & CLASS_MEMBER )
+		return (flags_ & PUBLIC_READABLE) ? MEMBER_VAR : PRIV_MEMBER_VAR;
+	else
+		return TEMP_VAR;
+	}
+
+void Field::Prepare(Env *env)
+	{
+	if ( type_ )
+		{
+		if ( anonymous_field() )
+			flags_ &= ~(CLASS_MEMBER | PUBLIC_READABLE);
+		if ( ! type_->persistent() )
+			flags_ &= (~PUBLIC_READABLE);
+
+		type_->set_value_var(id(), ValueVarType());
+		type_->Prepare(env, 
+			flags_ & TYPE_TO_BE_PARSED ? 
+				Type::TO_BE_PARSED : 0);
+		env->SetField(id(), this);
+		}
+	}
+
+void Field::GenPubDecls(Output* out_h, Env* env)
+	{
+	if ( type_ && (flags_ & PUBLIC_READABLE) && (flags_ & CLASS_MEMBER) )
+		type_->GenPubDecls(out_h, env);
+	}
+
+void Field::GenPrivDecls(Output* out_h, Env* env)
+	{
+	// Generate private declaration only if it is a class member
+	if ( type_ && (flags_ & CLASS_MEMBER) )
+		type_->GenPrivDecls(out_h, env);
+	}
+
+void Field::GenTempDecls(Output* out_h, Env* env)
+	{
+	// Generate temp field
+	if ( type_ && !(flags_ & CLASS_MEMBER) )
+		type_->GenPrivDecls(out_h, env);
+	}
+
+void Field::GenInitCode(Output* out_cc, Env* env)
+	{
+	if ( type_ && ! anonymous_field() )
+		type_->GenInitCode(out_cc, env);
+	}
+
+void Field::GenCleanUpCode(Output* out_cc, Env* env)
+	{
+	if ( type_ && ! anonymous_field() )
+		type_->GenCleanUpCode(out_cc, env);
+	}
+
+bool Field::DoTraverse(DataDepVisitor *visitor)
+	{
+	// Check parameterized type
+	if ( type_ && ! type_->Traverse(visitor) )
+		return false;
+	foreach(i, AttrList, attrs_)
+		if ( ! (*i)->Traverse(visitor) )
+			return false;
+	return true;
+	}
+
+bool Field::RequiresAnalyzerContext() const
+	{
+	// Check parameterized type
+	if ( type_ && type_->RequiresAnalyzerContext() )
+		return true;
+	foreach(i, AttrList, attrs_)
+		if ( (*i)->RequiresAnalyzerContext() )
+			return true;
+	return false;
+	}

tools/binpac/src/pac_field.h

@@ -0,0 +1,84 @@
+#ifndef pac_field_h
+#define pac_field_h
+
+#include "pac_common.h"
+#include "pac_datadep.h"
+
+// A "field" is a member of class.
+
+enum FieldType {
+	CASE_FIELD, 
+	CONTEXT_FIELD,
+	FLOW_FIELD,
+	LET_FIELD, 
+	PADDING_FIELD, 
+	PARAM_FIELD,
+	RECORD_FIELD, 
+	PARSE_VAR_FIELD,
+	PRIV_VAR_FIELD,
+	PUB_VAR_FIELD,
+	TEMP_VAR_FIELD,
+	WITHINPUT_FIELD,
+};
+
+class Field : public Object, public DataDepElement
+{
+public:
+	Field(FieldType tof, int flags, ID *id, Type *type);
+	// Field flags
+
+	// Whether the field will be evaluated by calling the Parse()
+	// function of the type
+	static const int TYPE_TO_BE_PARSED = 1;	
+	static const int TYPE_NOT_TO_BE_PARSED = 0;
+
+	// Whether the field is a member of the class or a temp
+	// variable
+	static const int CLASS_MEMBER = 2;
+	static const int NOT_CLASS_MEMBER = 0;
+
+	// Whether the field is public readable
+	static const int PUBLIC_READABLE = 4;
+	static const int NOT_PUBLIC_READABLE = 0;
+
+	virtual ~Field();
+
+	FieldType tof() const 	{ return tof_; }
+	const ID* id() const		{ return id_; }
+	Type *type() const		{ return type_; }
+	const ID* decl_id() const	{ return decl_id_; }
+
+	bool anonymous_field() const;
+
+	void AddAttr(AttrList* attrs);
+
+	// The field interface
+	virtual void ProcessAttr(Attr *attr);
+	virtual void Prepare(Env* env);
+
+	virtual void GenPubDecls(Output* out, Env* env);
+	virtual void GenPrivDecls(Output* out, Env* env);
+	virtual void GenTempDecls(Output* out, Env* env);
+
+	virtual void GenInitCode(Output* out, Env* env);
+	virtual void GenCleanUpCode(Output* out, Env* env);
+
+	virtual bool RequiresAnalyzerContext() const;
+
+protected:
+	int ValueVarType() const;
+	bool ToBeParsed() const;
+
+	bool DoTraverse(DataDepVisitor *visitor);
+
+protected:
+	FieldType tof_;
+	int flags_;
+	ID* id_;
+	Type *type_;
+	const ID* decl_id_;
+	string field_id_str_;
+	AttrList* attrs_;
+};
+
+#endif  // pac_field_h

tools/binpac/src/pac_flow.cc

@@ -0,0 +1,340 @@
+#include "pac_analyzer.h"
+#include "pac_conn.h"
+#include "pac_context.h"
+#include "pac_dataptr.h"
+#include "pac_dataunit.h"
+#include "pac_embedded.h"
+#include "pac_exception.h"
+#include "pac_expr.h"
+#include "pac_exttype.h"
+#include "pac_flow.h"
+#include "pac_output.h"
+#include "pac_param.h"
+#include "pac_paramtype.h"
+#include "pac_type.h"
+#include "pac_varfield.h"
+
+
+FlowDecl::FlowDecl(ID *id, 
+                   ParamList *params, 
+                   AnalyzerElementList *elemlist)
+	: AnalyzerDecl(id, FLOW, params)
+	{
+	dataunit_ = 0;
+	conn_decl_ = 0;
+	flow_buffer_var_field_ = 0;
+	AddElements(elemlist);
+	}
+
+FlowDecl::~FlowDecl()
+	{
+	delete flow_buffer_var_field_;
+	delete dataunit_;
+	}
+
+ParameterizedType *FlowDecl::flow_buffer_type_ = 0;
+
+ParameterizedType *FlowDecl::flow_buffer_type()
+	{
+	if ( ! flow_buffer_type_ )
+		{
+		flow_buffer_type_ = new ParameterizedType(new ID(kFlowBufferClass), 0);
+		}
+	return flow_buffer_type_;
+	}
+
+void FlowDecl::AddBaseClass(vector<string> *base_classes) const
+	{
+	base_classes->push_back("binpac::FlowAnalyzer"); 
+	} 
+
+void FlowDecl::ProcessFlowElement(AnalyzerFlow *flow_elem)
+	{
+	throw Exception(
+		flow_elem, 
+		"flow should be defined in only a connection declaration");
+	}
+
+void FlowDecl::ProcessDataUnitElement(AnalyzerDataUnit *dataunit_elem)
+	{
+	if ( dataunit_ )
+		{
+		throw Exception(dataunit_elem,
+		                "dataunit already defined");
+		}
+	dataunit_ = dataunit_elem;
+
+	if ( dataunit_->type() == AnalyzerDataUnit::FLOWUNIT )
+		{
+		dataunit_->data_type()->MarkIncrementalInput();
+
+		flow_buffer_var_field_ = new PrivVarField(
+			flow_buffer_id->clone(),
+			FlowDecl::flow_buffer_type()->Clone());
+		type_->AddField(flow_buffer_var_field_);
+
+		ASSERT(AnalyzerContextDecl::current_analyzer_context());
+		AnalyzerContextDecl::current_analyzer_context()->AddFlowBuffer();
+
+		// Add an argument to the context initiation
+		dataunit_->context_type()->AddParamArg(
+			new Expr(flow_buffer_var_field_->id()->clone()));
+		}
+	}
+
+void FlowDecl::Prepare()
+	{
+	// Add the connection parameter
+	if ( ! conn_decl_ )
+		{
+		throw Exception(this, 
+		                "no connection is not declared for the flow");
+		}
+
+	if ( ! params_ )
+		params_ = new ParamList();
+
+	params_->insert(params_->begin(),
+	                new Param(connection_id->clone(), 
+	                          conn_decl_->DataType()));
+
+	AnalyzerDecl::Prepare();
+
+	dataunit_->Prepare(env_);
+	}
+
+void FlowDecl::GenPubDecls(Output *out_h, Output *out_cc)
+	{
+	AnalyzerDecl::GenPubDecls(out_h, out_cc);
+	}
+
+void FlowDecl::GenPrivDecls(Output *out_h, Output *out_cc)
+	{
+	// Declare the data unit
+	dataunit_->dataunit_var_field()->GenPrivDecls(out_h, env_);
+
+	// Declare the analyzer context
+	dataunit_->context_var_field()->GenPrivDecls(out_h, env_);
+
+	AnalyzerDecl::GenPrivDecls(out_h, out_cc);
+	}
+
+void FlowDecl::GenInitCode(Output *out_cc)
+	{
+	AnalyzerDecl::GenInitCode(out_cc);
+
+	out_cc->println("%s = 0;",
+	                env_->LValue(dataunit_id));
+	out_cc->println("%s = 0;",
+	                env_->LValue(analyzer_context_id));
+
+	if ( dataunit_->type() == AnalyzerDataUnit::FLOWUNIT )
+		{
+		flow_buffer_var_field_->type()->GenPreParsing(out_cc, env_);
+		env_->SetEvaluated(flow_buffer_var_field_->id());
+		}
+	}
+
+void FlowDecl::GenCleanUpCode(Output *out_cc)
+	{
+	GenDeleteDataUnit(out_cc);
+	AnalyzerDecl::GenCleanUpCode(out_cc);
+	}
+
+void FlowDecl::GenEOFFunc(Output *out_h, Output *out_cc)
+	{
+	string proto = strfmt("%s()", kFlowEOF);
+
+	out_h->println("void %s;", proto.c_str());
+
+	out_cc->println("void %s::%s", class_name().c_str(), proto.c_str());
+	out_cc->inc_indent();
+	out_cc->println("{");
+
+	foreach(i, AnalyzerHelperList, eof_helpers_)
+		{
+		(*i)->GenCode(0, out_cc, this);
+		}
+
+	if ( dataunit_->type() == AnalyzerDataUnit::FLOWUNIT )
+		{
+		out_cc->println("%s->set_eof();", 
+			env_->LValue(flow_buffer_id));
+		out_cc->println("%s(0, 0);", kNewData);
+		}
+	
+	out_cc->println("}");
+	out_cc->dec_indent();
+	}
+
+void FlowDecl::GenGapFunc(Output *out_h, Output *out_cc)
+	{
+	string proto = strfmt("%s(int gap_length)", kFlowGap);
+
+	out_h->println("void %s;", proto.c_str());
+
+	out_cc->println("void %s::%s", class_name().c_str(), proto.c_str());
+	out_cc->inc_indent();
+	out_cc->println("{");
+
+	if ( dataunit_->type() == AnalyzerDataUnit::FLOWUNIT )
+		{
+		out_cc->println("%s->NewGap(gap_length);", 
+			env_->LValue(flow_buffer_id));
+		}
+	
+	out_cc->println("}");
+	out_cc->dec_indent();
+	}
+
+void FlowDecl::GenProcessFunc(Output *out_h, Output *out_cc)
+	{
+	env_->AddID(begin_of_data, TEMP_VAR, extern_type_const_byteptr);
+	env_->AddID(end_of_data, TEMP_VAR, extern_type_const_byteptr);
+
+	string proto = 
+		strfmt("%s(const_byteptr %s, const_byteptr %s)",
+			kNewData,
+			env_->LValue(begin_of_data),
+			env_->LValue(end_of_data));
+
+	out_h->println("void %s;", proto.c_str());
+
+	out_cc->println("void %s::%s", class_name().c_str(), proto.c_str());
+	out_cc->inc_indent();
+	out_cc->println("{");
+
+	out_cc->println("try");
+	out_cc->inc_indent();
+	out_cc->println("{");
+
+	env_->SetEvaluated(begin_of_data);
+	env_->SetEvaluated(end_of_data);
+
+	switch ( dataunit_->type() )
+		{
+		case AnalyzerDataUnit::DATAGRAM:
+			GenCodeDatagram(out_cc);
+			break;
+		case AnalyzerDataUnit::FLOWUNIT:
+			GenCodeFlowUnit(out_cc);
+			break;
+		default:
+			ASSERT(0);
+		}
+
+	out_cc->println("}");
+	out_cc->dec_indent();
+
+	out_cc->println("catch ( Exception const &e )");
+	out_cc->inc_indent();
+	out_cc->println("{");
+	GenCleanUpCode(out_cc);
+	if ( dataunit_->type() == AnalyzerDataUnit::FLOWUNIT )
+		{
+		out_cc->println("%s->DiscardData();",
+			env_->LValue(flow_buffer_id));
+		}
+	out_cc->println("throw e;");
+	out_cc->println("}");
+	out_cc->dec_indent();
+
+	out_cc->println("}");
+	out_cc->dec_indent();
+	out_cc->println("");
+	}
+
+void FlowDecl::GenNewDataUnit(Output *out_cc)
+	{
+	Type *unit_datatype = dataunit_->data_type();
+	// dataunit_->data_type()->GenPreParsing(out_cc, env_);
+	dataunit_->GenNewDataUnit(out_cc, env_);
+	if ( unit_datatype->buffer_input() &&
+	     unit_datatype->buffer_mode() == Type::BUFFER_BY_LENGTH )
+		{
+		out_cc->println("%s->NewFrame(0, false);", 
+			env_->LValue(flow_buffer_id));
+		}
+	dataunit_->GenNewContext(out_cc, env_);
+	}
+
+void FlowDecl::GenDeleteDataUnit(Output *out_cc)
+	{
+	// Do not just delete dataunit, because we may just want to Unref it.
+	// out_cc->println("delete %s;", env_->LValue(dataunit_id));
+	dataunit_->data_type()->GenCleanUpCode(out_cc, env_);
+	dataunit_->context_type()->GenCleanUpCode(out_cc, env_);
+	}
+
+void FlowDecl::GenCodeFlowUnit(Output *out_cc)
+	{
+	Type *unit_datatype = dataunit_->data_type();
+
+	out_cc->println("%s->NewData(%s, %s);",
+		env_->LValue(flow_buffer_id),
+		env_->RValue(begin_of_data), 
+		env_->RValue(end_of_data)); 
+
+	out_cc->println("while ( %s->data_available() && ",
+		env_->LValue(flow_buffer_id));
+	out_cc->inc_indent();
+	out_cc->println("( !%s->have_pending_request() || %s->ready() ) )",
+		env_->LValue(flow_buffer_id), env_->LValue(flow_buffer_id));
+	out_cc->println("{");
+
+	// Generate a new dataunit if necessary
+	out_cc->println("if ( ! %s )", env_->LValue(dataunit_id));
+	out_cc->inc_indent();
+	out_cc->println("{");
+	out_cc->println("BINPAC_ASSERT(!%s);", 
+		env_->LValue(analyzer_context_id));
+	GenNewDataUnit(out_cc);
+	out_cc->println("}");
+	out_cc->dec_indent();
+
+	DataPtr data(env_, 0, 0);
+	unit_datatype->GenParseCode(out_cc, env_, data, 0);
+
+	out_cc->println("if ( %s )", 
+		unit_datatype->parsing_complete(env_).c_str());
+	out_cc->inc_indent();
+	out_cc->println("{");
+	out_cc->println("// Clean up the flow unit after parsing");
+	GenDeleteDataUnit(out_cc);
+	// out_cc->println("BINPAC_ASSERT(%s == 0);", env_->LValue(dataunit_id));
+	out_cc->println("}");
+	out_cc->dec_indent();
+	out_cc->println("else");
+	out_cc->inc_indent();
+	out_cc->println("{");
+	out_cc->println("// Resume upon next input segment");
+	out_cc->println("BINPAC_ASSERT(!%s->ready());",
+		env_->RValue(flow_buffer_id));
+	out_cc->println("break;");
+	out_cc->println("}");
+	out_cc->dec_indent();
+
+	out_cc->println("}");
+	out_cc->dec_indent();
+	}
+
+void FlowDecl::GenCodeDatagram(Output *out_cc)
+	{
+	Type *unit_datatype = dataunit_->data_type();
+	GenNewDataUnit(out_cc);
+
+	string parse_params = strfmt("%s, %s", 
+		env_->RValue(begin_of_data), 
+		env_->RValue(end_of_data)); 
+
+	if ( RequiresAnalyzerContext::compute(unit_datatype) )
+		{
+		parse_params += ", ";
+		parse_params += env_->RValue(analyzer_context_id);
+		}
+
+	DataPtr dataptr(env_, begin_of_data, 0);
+	unit_datatype->GenParseCode(out_cc, env_, dataptr, 0);
+
+	GenDeleteDataUnit(out_cc);
+	}

tools/binpac/src/pac_flow.h

@@ -0,0 +1,47 @@
+#ifndef pac_flow_h
+#define pac_flow_h
+
+#include "pac_analyzer.h"
+
+class FlowDecl : public AnalyzerDecl
+{
+public:
+	FlowDecl(ID *flow_id, ParamList *params, AnalyzerElementList *elemlist);
+	~FlowDecl();
+
+	void Prepare();
+
+	void set_conn_decl(ConnDecl *c)	{ conn_decl_ = c; }
+
+	static ParameterizedType *flow_buffer_type();
+
+protected:
+	void AddBaseClass(vector<string> *base_classes) const;
+
+	void GenInitCode(Output *out_cc);
+	void GenCleanUpCode(Output *out_cc);
+	void GenProcessFunc(Output *out_h, Output *out_cc);
+	void GenEOFFunc(Output *out_h, Output *out_cc);
+	void GenGapFunc(Output *out_h, Output *out_cc);
+
+	void GenPubDecls(Output *out_h, Output *out_cc);
+	void GenPrivDecls(Output *out_h, Output *out_cc);
+
+	void ProcessFlowElement(AnalyzerFlow *flow_elem);
+	void ProcessDataUnitElement(AnalyzerDataUnit *dataunit_elem);
+
+private:
+	void GenNewDataUnit(Output *out_cc);
+	void GenDeleteDataUnit(Output *out_cc);
+	void GenCodeFlowUnit(Output *out_cc);
+	void GenCodeDatagram(Output *out_cc);
+
+	AnalyzerDataUnit *dataunit_;
+	ConnDecl *conn_decl_;
+
+	Field *flow_buffer_var_field_;
+
+	static ParameterizedType *flow_buffer_type_;
+};
+
+#endif  // pac_flow_h

tools/binpac/src/pac_func.cc

@@ -0,0 +1,123 @@
+#include "pac_embedded.h"
+#include "pac_expr.h"
+#include "pac_func.h"
+#include "pac_output.h"
+#include "pac_param.h"
+#include "pac_type.h"
+
+Function::Function(ID *id, Type *type, ParamList *params)
+ 	: id_(id), type_(type), params_(params), expr_(0), code_(0)
+	{
+	analyzer_decl_ = 0;
+	env_ = 0;
+	}
+
+Function::~Function()
+	{
+	delete id_;
+	delete type_;
+	delete_list(ParamList, params_);
+	delete env_;
+	delete expr_;
+	delete code_;
+	}
+
+void Function::Prepare(Env *env)
+	{
+	env->AddID(id_, FUNC_ID, type_);
+	env->SetEvaluated(id_);
+
+	env_ = new Env(env, this);
+
+	foreach(i, ParamList, params_)
+		{
+		Param *p = *i;
+		env_->AddID(p->id(), FUNC_PARAM, p->type());
+		env_->SetEvaluated(p->id());
+		}
+	}
+
+void Function::GenForwardDeclaration(Output* out_h)
+	{
+	// Do nothing
+	}
+
+void Function::GenCode(Output* out_h, Output* out_cc)
+	{
+	out_h->println("%s %s(%s);", 
+		type_->DataTypeStr().c_str(),
+		id_->Name(),
+		ParamDecls(params_).c_str());
+
+	string class_str = "";
+	if ( analyzer_decl_ )
+		class_str = strfmt("%s::", analyzer_decl_->id()->Name());
+
+	string proto_str = strfmt("%s %s%s(%s)", 
+				type_->DataTypeStr().c_str(),
+				class_str.c_str(),
+				id_->Name(),
+				ParamDecls(params_).c_str());
+
+	ASSERT(!(expr_ && code_));
+
+	if ( expr_ )
+		{
+		out_cc->println("%s", proto_str.c_str());
+
+		out_cc->inc_indent();
+		out_cc->println("{");
+
+		out_cc->println("return static_cast<%s>(%s);", 
+			type_->DataTypeStr().c_str(),
+			expr_->EvalExpr(out_cc, env_));
+
+		out_cc->println("}");
+		out_cc->dec_indent();
+		}
+
+	else if ( code_ )
+		{
+		out_cc->println("%s", proto_str.c_str());
+
+		out_cc->inc_indent();
+		out_cc->println("{");
+
+		code_->GenCode(out_cc, env_);
+
+		out_cc->println("}");
+		out_cc->dec_indent();
+		}
+
+	out_cc->println("");
+	}
+
+FuncDecl::FuncDecl(Function *function)
+	: Decl(function->id()->clone(), FUNC), function_(function)
+	{
+	function_->Prepare(global_env());
+	}
+
+FuncDecl::~FuncDecl()
+	{
+	delete function_;
+	}
+
+void FuncDecl::Prepare()
+	{
+	}
+
+void FuncDecl::GenForwardDeclaration(Output *out_h)
+	{
+	function_->GenForwardDeclaration(out_h);
+	}
+
+void FuncDecl::GenCode(Output *out_h, Output *out_cc)
+	{
+	function_->GenCode(out_h, out_cc);
+	}
+
+AnalyzerFunction::AnalyzerFunction(Function *function)
+	: AnalyzerElement(FUNCTION), function_(function)
+	{
+	}

tools/binpac/src/pac_func.h

@@ -0,0 +1,68 @@
+#ifndef pac_func_h
+#define pac_func_h
+
+#include "pac_decl.h"
+#include "pac_analyzer.h"
+
+class Function : public Object
+{
+public:
+	Function(ID *id, Type *type, ParamList *params);
+	~Function();
+
+	ID *id() const			{ return id_; }
+
+	AnalyzerDecl *analyzer_decl() const { return analyzer_decl_; }
+	void set_analyzer_decl(AnalyzerDecl *decl) { analyzer_decl_ = decl; }
+
+	Expr *expr() const		{ return expr_; }
+	void set_expr(Expr *expr) 	{ expr_ = expr; }
+
+	EmbeddedCode *code() const	{ return code_; }
+	void set_code(EmbeddedCode *code) { code_ = code; }
+
+	void Prepare(Env *env);
+	void GenForwardDeclaration(Output *out_h);
+	void GenCode(Output *out_h, Output *out_cc);
+
+private:
+	Env *env_;
+
+	ID *id_;
+	Type *type_;
+	ParamList *params_;
+
+	AnalyzerDecl *analyzer_decl_;
+
+	Expr *expr_;
+	EmbeddedCode *code_;
+};
+
+class FuncDecl : public Decl
+{
+public:
+	FuncDecl(Function *function);
+	~FuncDecl();
+
+	Function *function() const	{ return function_; }
+
+	void Prepare();
+	void GenForwardDeclaration(Output *out_h);
+	void GenCode(Output *out_h, Output *out_cc);
+
+private:
+	Function *function_;
+};
+
+class AnalyzerFunction : public AnalyzerElement
+{
+public:
+	AnalyzerFunction(Function *function);
+
+	Function *function() const	{ return function_; }
+
+private:
+	Function *function_;
+};
+
+#endif // pac_func_h

tools/binpac/src/pac_id.cc

@@ -0,0 +1,440 @@
+#include "pac_exception.h"
+#include "pac_expr.h"
+#include "pac_exttype.h"
+#include "pac_field.h"
+#include "pac_id.h"
+#include "pac_type.h"
+#include "pac_utils.h"
+
+const ID *default_value_var = 0;
+const ID *null_id = 0;
+const ID *null_byteseg_id = 0;
+const ID *null_decl_id = 0;
+const ID *begin_of_data = 0;
+const ID *end_of_data = 0;
+const ID *len_of_data = 0;
+const ID *byteorder_id = 0;
+const ID *bigendian_id = 0;
+const ID *littleendian_id = 0;
+const ID *unspecified_byteorder_id = 0;
+const ID *const_true_id = 0;
+const ID *const_false_id = 0;
+const ID *analyzer_context_id = 0;
+const ID *context_macro_id = 0;
+const ID *this_id = 0;
+const ID *sourcedata_id = 0;
+const ID *connection_id = 0;
+const ID *upflow_id = 0;
+const ID *downflow_id = 0;
+const ID *dataunit_id = 0;
+const ID *flow_buffer_id = 0;
+const ID *element_macro_id = 0;
+const ID *input_macro_id = 0;
+const ID *cxt_connection_id = 0;
+const ID *cxt_flow_id = 0;
+const ID *parsing_state_id = 0;
+const ID *buffering_state_id = 0;
+
+int ID::anonymous_id_seq = 0;
+
+ID *ID::NewAnonymousID(const string &prefix)
+	{
+	ID *id = new ID(fmt("%s%03d", prefix.c_str(), ++anonymous_id_seq));
+	id->anonymous_id_ = true;
+	return id;
+	}
+
+IDRecord::IDRecord(Env *arg_env, const ID* arg_id, IDType arg_id_type)
+	: env(arg_env), id(arg_id), id_type(arg_id_type)
+	{
+	eval = 0;
+	evaluated = in_evaluation = false;
+	setfunc = "";  // except for STATE_VAR
+	switch (id_type)
+		{
+		case MEMBER_VAR:
+			rvalue = strfmt("%s()", id->Name());
+			lvalue = strfmt("%s_", id->Name());
+			break;
+		case PRIV_MEMBER_VAR:
+			rvalue = strfmt("%s_", id->Name());
+			lvalue = strfmt("%s_", id->Name());
+			break;
+		case UNION_VAR:
+			rvalue = strfmt("%s()", id->Name());
+			lvalue = strfmt("%s_", id->Name());
+			break;
+		case CONST:
+		case GLOBAL_VAR:
+			rvalue = strfmt("%s", id->Name());
+			lvalue = strfmt("%s", id->Name());
+			break;
+		case TEMP_VAR:
+			rvalue = strfmt("t_%s", id->Name());
+			lvalue = strfmt("t_%s", id->Name());
+			break;
+		case STATE_VAR:
+			rvalue = strfmt("%s()", id->Name());
+			lvalue = strfmt("%s_", id->Name());
+			break;
+		case MACRO:
+			rvalue = "@MACRO@";
+			lvalue = "@MACRO@";
+			break;
+		case FUNC_ID:
+			rvalue = strfmt("%s", id->Name());
+			lvalue = "@FUNC_ID@";
+			break;
+		case FUNC_PARAM:
+			rvalue = strfmt("%s", id->Name());
+			lvalue = "@FUNC_PARAM@";
+			break;
+		}
+	field = 0;
+	constant_set = false;
+	}
+
+IDRecord::~IDRecord()
+	{
+	}
+
+void IDRecord::SetConstant(int c)			
+	{ 
+	ASSERT(id_type == CONST);
+	constant_set = true;
+	constant = c; 
+	}
+
+bool IDRecord::GetConstant(int *pc) const
+	{ 
+	if ( constant_set ) 
+		*pc = constant;
+	return constant_set;
+	}
+
+void IDRecord::SetMacro(Expr *e)
+	{
+	ASSERT(id_type == MACRO);
+	macro = e;
+	}
+
+Expr *IDRecord::GetMacro() const
+	{
+	ASSERT(id_type == MACRO);
+	return macro;
+	}
+
+void IDRecord::SetEvaluated(bool v)
+	{
+	if ( v )
+		ASSERT(! evaluated);
+	evaluated = v;
+	}
+
+void IDRecord::Evaluate(Output* out, Env* env)	
+	{ 
+	if ( evaluated )
+		return;
+
+	if ( ! out )
+		throw ExceptionIDNotEvaluated(id);
+
+	if ( ! eval )
+		throw Exception(id, "no evaluation method");
+		
+	if ( in_evaluation )
+		throw ExceptionCyclicDependence(id);
+
+	in_evaluation = true;
+	eval->GenEval(out, env); 
+	in_evaluation = false;
+
+	evaluated = true;
+	}
+
+const char* IDRecord::RValue() const
+	{
+	if ( id_type == MACRO )
+		return macro->EvalExpr(0, env);
+
+	if ( id_type == TEMP_VAR && ! evaluated )
+		throw ExceptionIDNotEvaluated(id);
+
+	return rvalue.c_str();
+	}
+
+const char* IDRecord::LValue() const
+	{
+	ASSERT(id_type != MACRO && id_type != FUNC_ID);
+	return lvalue.c_str();
+	}
+
+Env::Env(Env* parent_env, Object* context_object)
+	: parent(parent_env), context_object_(context_object)
+	{
+	allow_undefined_id_ = false;
+	in_branch_ = false;
+	}
+
+Env::~Env()
+	{
+	for ( id_map_t::iterator it = id_map.begin(); it != id_map.end(); ++it )
+		{
+		delete it->second;
+		it->second = 0;
+		}
+	}
+
+void Env::AddID(const ID* id, IDType id_type, Type *data_type)
+	{
+	DEBUG_MSG("To add ID `%s'...\n", id->Name());
+	id_map_t::iterator it = id_map.find(id);
+	if ( it != id_map.end() )
+		{
+		DEBUG_MSG("Duplicate definition: `%s'\n", it->first->Name());
+		throw ExceptionIDRedefinition(id);
+		}
+	id_map[id] = new IDRecord(this, id, id_type);
+	// TODO: figure out when data_type must be non-NULL
+	// ASSERT(data_type);
+	SetDataType(id, data_type);
+	}
+
+void Env::AddConstID(const ID* id, const int c, Type *type)
+	{
+	if ( ! type )
+		type = extern_type_int;
+	AddID(id, CONST, type);
+	SetConstant(id, c);
+	SetEvaluated(id);  // a constant is always evaluated
+	}
+
+void Env::AddMacro(const ID *id, Expr *macro)
+	{
+	AddID(id, MACRO, macro->DataType(this));
+	SetMacro(id, macro);
+	SetEvaluated(id);
+	}
+
+ID *Env::AddTempID(Type *type)
+	{
+	ID *id = ID::NewAnonymousID("t_var_");
+	AddID(id, TEMP_VAR, type);
+	return id;
+	}
+
+IDRecord* Env::lookup(const ID* id, bool recursive, bool raise_exception) const
+	{
+	ASSERT(id);
+
+	id_map_t::const_iterator it = id_map.find(id);
+	if ( it != id_map.end() )
+		return it->second;
+
+	if ( recursive && parent )
+		return parent->lookup(id, recursive, raise_exception);
+
+	if ( raise_exception )
+		throw ExceptionIDNotFound(id);
+	else
+		return 0;
+	}
+
+IDType Env::GetIDType(const ID* id) const
+	{
+	return lookup(id, true, true)->GetType();
+	}
+
+const char* Env::RValue(const ID* id) const
+	{
+	IDRecord *r = lookup(id, true, false);
+	if ( r )
+		return r->RValue();
+	else
+		{
+		if ( allow_undefined_id() )
+			return id->Name();
+		else
+			throw ExceptionIDNotFound(id);
+		}
+	}
+
+const char* Env::LValue(const ID* id) const
+	{
+	return lookup(id, true, true)->LValue();
+	}
+
+void Env::SetEvalMethod(const ID* id, Evaluatable* eval)
+	{
+	lookup(id, true, true)->SetEvalMethod(eval);
+	}
+
+void Env::Evaluate(Output* out, const ID* id)
+	{
+	IDRecord *r = lookup(id, true, !allow_undefined_id());
+	if ( r )
+		r->Evaluate(out, this);
+	}
+
+bool Env::Evaluated(const ID* id) const
+	{
+	IDRecord *r = lookup(id, true, !allow_undefined_id());
+	if ( r )
+		return r->Evaluated();
+	else
+		// Assume undefined variables are already evaluated
+		return true;  
+	}
+
+void Env::SetEvaluated(const ID* id, bool v)
+	{
+	if ( in_branch() )
+		{
+		Field *f = GetField(id);
+		if (f && f->tof() == LET_FIELD)
+			{
+			throw Exception(
+				context_object_,
+				fmt("INTERNAL ERROR: "
+				"evaluating let field '%s' in a branch! "
+				"To work around this problem, "
+				"add '&requires(%s)' to the case type. "
+				"Sorry for the inconvenience.\n",
+				id->Name(),
+				id->Name()));
+			ASSERT(0);
+			}
+		}
+
+	IDRecord *r = lookup(id, false, false);
+	if ( r )
+		r->SetEvaluated(v);
+	else if ( parent )
+		parent->SetEvaluated(id, v);
+	else
+		throw ExceptionIDNotFound(id);
+	}
+
+void Env::SetField(const ID* id, Field* field)
+	{
+	lookup(id, false, true)->SetField(field);
+	}
+
+Field* Env::GetField(const ID* id) const
+	{
+	return lookup(id, true, true)->GetField();
+	}
+
+void Env::SetDataType(const ID* id, Type* type)
+	{
+	lookup(id, true, true)->SetDataType(type);
+	}
+
+Type* Env::GetDataType(const ID* id) const
+	{
+	IDRecord *r = lookup(id, true, false);
+	if ( r )
+		return r->GetDataType();
+	else
+		return 0;  
+	}
+
+string Env::DataTypeStr(const ID *id) const
+	{
+	Type *type = GetDataType(id);
+	if ( ! type )
+		throw Exception(id, "data type not defined");
+	return type->DataTypeStr();
+	}
+
+void Env::SetConstant(const ID* id, int constant)
+	{
+	lookup(id, false, true)->SetConstant(constant);
+	}
+
+bool Env::GetConstant(const ID* id, int* pc) const
+	{
+	ASSERT(pc);
+	// lookup without raising exception
+	IDRecord* r = lookup(id, true, false);
+	if ( r )
+		return r->GetConstant(pc);
+	else
+		return false;
+	}
+
+void Env::SetMacro(const ID* id, Expr *macro)
+	{
+	lookup(id, true, true)->SetMacro(macro);
+	}
+
+Expr* Env::GetMacro(const ID* id) const
+	{
+	return lookup(id, true, true)->GetMacro();
+	}
+
+void init_builtin_identifiers()
+	{
+	default_value_var = new ID("val");
+	null_id = new ID("NULL");
+	null_byteseg_id = new ID("null_byteseg");
+	begin_of_data = new ID("begin_of_data");
+	end_of_data = new ID("end_of_data");
+	len_of_data = new ID("length_of_data");
+	byteorder_id = new ID("byteorder");
+	bigendian_id = new ID("bigendian");
+	littleendian_id = new ID("littleendian");
+	unspecified_byteorder_id = new ID("unspecified_byteorder");
+	const_true_id = new ID("true");
+	const_false_id = new ID("false");
+	analyzer_context_id = new ID("context");
+	this_id = new ID("this");
+	sourcedata_id = new ID("sourcedata");
+	connection_id = new ID("connection");
+	upflow_id = new ID("upflow");
+	downflow_id = new ID("downflow");
+	dataunit_id = new ID("dataunit");
+	flow_buffer_id = new ID("flow_buffer");
+	element_macro_id = new ID("$element");
+	input_macro_id = new ID("$input"); 
+	context_macro_id = new ID("$context"); 
+	parsing_state_id = new ID("parsing_state");
+	buffering_state_id = new ID("buffering_state");
+
+	null_decl_id = new ID("<null-decl>");
+	current_decl_id = null_decl_id;
+	}
+
+Env* global_env()
+	{
+	static Env *the_global_env = 0;
+
+	if ( ! the_global_env )
+		{
+		the_global_env = new Env(0, 0);
+
+		// These two are defined in binpac.h, so we do not need to
+		// generate code for them.
+		the_global_env->AddConstID(bigendian_id, 0);
+		the_global_env->AddConstID(littleendian_id, 1);
+		the_global_env->AddConstID(unspecified_byteorder_id, -1);
+		the_global_env->AddConstID(const_false_id, 0);
+		the_global_env->AddConstID(const_true_id, 1);
+		// A hack for ID "this"
+		the_global_env->AddConstID(this_id, 0);
+		the_global_env->AddConstID(null_id, 0, extern_type_nullptr);
+
+#if 0
+		the_global_env->AddID(null_byteseg_id, 
+			GLOBAL_VAR, 
+			extern_type_const_byteseg);
+#endif
+		}
+
+	return the_global_env;
+	}
+
+string set_function(const ID *id)
+	{
+	return strfmt("set_%s", id->Name());
+	}

tools/binpac/src/pac_id.h

@@ -0,0 +1,246 @@
+#ifndef pac_id_h
+#define pac_id_h
+
+#include <map>
+#include <string>
+using namespace std;
+
+#include "pac_common.h"
+#include "pac_dbg.h"
+#include "pac_utils.h"
+
+// Classes handling identifiers.
+// 
+// ID -- name and location of definition of an ID
+//
+// IDRecord -- association of an ID, its definition type (const, global, temp, 
+//   member, or union member), and its evaluation method. 
+//
+// Evaluatable -- interface for a variable or a field that needs be evaluated
+//   before referenced.
+//
+// Env -- a mapping from ID names to their L/R-value expressions and evaluation
+//   methods.
+
+enum IDType {
+	CONST,
+	GLOBAL_VAR,
+	TEMP_VAR,
+	MEMBER_VAR,
+	PRIV_MEMBER_VAR,
+	UNION_VAR,
+	STATE_VAR,
+	MACRO,
+	FUNC_ID,
+	FUNC_PARAM,
+};
+
+class ID;
+class IDRecord;
+class Env;
+class Evaluatable;
+
+class ID : public Object
+{
+public:
+	ID(const char *arg_name)
+		: name(arg_name), anonymous_id_(false)
+		{
+		locname = nfmt("%s:%s", Location(), Name());
+		}
+	~ID()
+		{
+		delete locname;
+		}
+
+	bool operator==(ID const &x) const { return name == x.Name(); }
+
+	const char *Name() const 	{ return name.c_str(); }
+	const char *LocName() const 	{ return locname; }
+	bool is_anonymous() const	{ return anonymous_id_; }
+
+	ID *clone() const		{ return new ID(Name()); }
+
+protected:
+	string name;
+	bool anonymous_id_;
+	char *locname;
+	friend class ID_ptr_cmp;
+
+public:
+	static ID *NewAnonymousID(const string &prefix);
+private:
+	static int anonymous_id_seq;
+};
+
+// A comparison operator for pointers to ID's.
+class ID_ptr_cmp
+{
+public:
+	bool operator()(const ID *const & id1, const ID *const & id2) const
+		{
+		ASSERT(id1);
+		ASSERT(id2);
+		return id1->name < id2->name;
+		}
+};
+
+class IDRecord
+{
+public:
+	IDRecord(Env *env, const ID *id, IDType id_type);
+	~IDRecord();
+
+	IDType GetType() const			{ return id_type; }
+
+	void SetDataType(Type *type) 		{ data_type = type; }
+	Type *GetDataType() const		{ return data_type; }
+
+	void SetEvalMethod(Evaluatable *arg_eval) { eval = arg_eval; }
+	void Evaluate(Output *out, Env *env);
+	void SetEvaluated(bool v);
+	bool Evaluated() const 			{ return evaluated; }
+
+	void SetField(Field *f)			{ field = f; }
+	Field *GetField() const			{ return field; }
+
+	void SetConstant(int c);			
+	bool GetConstant(int *pc) const;
+
+	void SetMacro(Expr *expr);
+	Expr *GetMacro() const;
+
+	const char * RValue() const;
+	const char * LValue() const;
+
+protected:
+	Env *env;
+	const ID *id;
+	IDType id_type;
+
+	string rvalue;
+	string lvalue;
+	string setfunc;
+
+	Type *data_type;
+
+	Field *field;
+
+	int constant;
+	bool constant_set;
+
+	Expr *macro;
+
+	bool evaluated;
+	bool in_evaluation;	// to detect cyclic dependence
+	Evaluatable *eval;
+};
+
+class Evaluatable
+{
+public:
+	virtual ~Evaluatable() {}
+	virtual void GenEval(Output *out, Env *env) = 0;
+};
+
+class Env
+{
+public:
+	Env(Env *parent_env, Object *context_object);
+	~Env();
+
+	bool allow_undefined_id() const		{ return allow_undefined_id_; }
+	void set_allow_undefined_id(bool x)	{ allow_undefined_id_ = x; }
+
+	bool in_branch() const			{ return in_branch_; }
+	void set_in_branch(bool x)		{ in_branch_ = x; }
+
+	void AddID(const ID *id, IDType id_type, Type *type);
+	void AddConstID(const ID *id, const int c, Type *type = 0);
+	void AddMacro(const ID *id, Expr *expr);
+
+	// Generate a temp ID with a unique name
+	ID *AddTempID(Type *type);
+
+	IDType GetIDType(const ID *id) const;
+	const char * RValue(const ID *id) const;
+	const char * LValue(const ID *id) const;
+	// const char *SetFunc(const ID *id) const;
+
+	// Set evaluation method for the ID
+	void SetEvalMethod(const ID *id, Evaluatable *eval);
+
+	// Evaluate the ID according to the evaluation method. It
+	// assumes the ID has an evaluation emthod. It does nothing
+	// if the ID has already been evaluated.
+	void Evaluate(Output *out, const ID *id);
+
+	// Whether the ID has already been evaluated.
+	bool Evaluated(const ID *id) const;
+
+	// Set the ID as evaluated (or not).
+	void SetEvaluated(const ID *id, bool v = true);
+
+	void SetField(const ID *id, Field *field);
+	Field *GetField(const ID *id) const;
+
+	bool GetConstant(const ID *id, int *pc) const;
+
+	Expr *GetMacro(const ID *id) const;
+
+	Type *GetDataType(const ID *id) const;
+
+	string DataTypeStr(const ID *id) const;
+
+protected:
+	IDRecord *lookup(const ID *id, 
+	                 bool recursive, 
+	                 bool raise_exception) const;
+
+	void SetDataType(const ID *id, Type *type);
+	void SetConstant(const ID *id, int constant);
+	void SetMacro(const ID *id, Expr *macro);
+
+private:
+	Env *parent;
+	Object *context_object_;
+	typedef map<const ID*, IDRecord*, ID_ptr_cmp> id_map_t;
+	id_map_t id_map;
+	bool allow_undefined_id_;
+	bool in_branch_;
+};
+
+extern const ID *default_value_var;
+extern const ID *null_id;
+extern const ID *null_byteseg_id;
+extern const ID *begin_of_data;
+extern const ID *end_of_data;
+extern const ID *len_of_data;
+extern const ID *byteorder_id;
+extern const ID *bigendian_id;
+extern const ID *littleendian_id;
+extern const ID *unspecified_byteorder_id;
+extern const ID *analyzer_context_id;
+extern const ID *context_macro_id;
+extern const ID *this_id;
+extern const ID *sourcedata_id;
+// extern const ID *sourcedata_begin_id;
+// extern const ID *sourcedata_end_id;
+extern const ID *connection_id;
+extern const ID *upflow_id;
+extern const ID *downflow_id;
+extern const ID *dataunit_id;
+extern const ID *flow_buffer_id;
+extern const ID *element_macro_id;
+extern const ID *cxt_connection_id;
+extern const ID *cxt_flow_id;
+extern const ID *input_macro_id;
+extern const ID *parsing_state_id;
+extern const ID *buffering_state_id;
+
+extern void init_builtin_identifiers();
+extern Env *global_env();
+
+extern string set_function(const ID *id);
+
+#endif // pac_id_h

tools/binpac/src/pac_inputbuf.cc

@@ -0,0 +1,44 @@
+#include "pac_expr.h"
+#include "pac_exttype.h"
+#include "pac_id.h"
+#include "pac_inputbuf.h"
+#include "pac_output.h"
+#include "pac_type.h"
+
+InputBuffer::InputBuffer(Expr *expr)
+	: DataDepElement(INPUT_BUFFER), expr_(expr)
+	{
+	}
+
+bool InputBuffer::DoTraverse(DataDepVisitor *visitor)
+	{
+	if ( expr_ && ! expr_->Traverse(visitor) )
+		return false;
+	return true;
+	}
+
+bool InputBuffer::RequiresAnalyzerContext() const
+	{
+	return expr_->RequiresAnalyzerContext();
+	}
+
+DataPtr InputBuffer::GenDataBeginEnd(Output *out_cc, Env *env)
+	{
+	env->AddID(begin_of_data, TEMP_VAR, extern_type_const_byteptr);
+	env->AddID(end_of_data, TEMP_VAR, extern_type_const_byteptr);
+
+	out_cc->println("%s %s, %s;",
+		extern_type_const_byteptr->DataTypeStr().c_str(),
+		env->LValue(begin_of_data),
+		env->LValue(end_of_data));
+
+	out_cc->println("get_pointers(%s, &%s, &%s);",
+		expr_->EvalExpr(out_cc, env),
+		env->LValue(begin_of_data),
+		env->LValue(end_of_data));
+
+	env->SetEvaluated(begin_of_data);
+	env->SetEvaluated(end_of_data);
+
+	return DataPtr(env, begin_of_data, 0);
+	}

tools/binpac/src/pac_inputbuf.h

@@ -0,0 +1,24 @@
+#ifndef pac_inputbuf_h
+#define pac_inputbuf_h
+
+#include "pac_datadep.h"
+#include "pac_dataptr.h"
+
+class Expr;
+
+class InputBuffer : public Object, public DataDepElement
+{
+public:
+	InputBuffer(Expr *expr);
+
+	bool RequiresAnalyzerContext() const;
+	DataPtr GenDataBeginEnd(Output *out_cc, Env *env);
+
+protected:
+	bool DoTraverse(DataDepVisitor *visitor);
+
+private:
+	Expr *expr_;
+};
+
+#endif // pac_inputbuf_h

tools/binpac/src/pac_let.cc

@@ -0,0 +1,167 @@
+#include "pac_expr.h"
+#include "pac_exttype.h"
+#include "pac_let.h"
+#include "pac_output.h"
+#include "pac_type.h"
+
+namespace {
+
+void GenLetEval(const ID *id, Expr *expr, string prefix, Output* out, Env* env)
+	{
+	if ( expr )
+		{
+		}
+	}
+
+} // private namespace
+
+LetField::LetField(ID* id, Type *type, Expr* expr)
+	: Field(LET_FIELD, 
+		TYPE_NOT_TO_BE_PARSED | CLASS_MEMBER | PUBLIC_READABLE, 
+		id, type), 
+	  expr_(expr)
+	{
+	ASSERT(expr_);
+	}
+
+LetField::~LetField()
+	{
+	delete expr_;
+	}
+
+bool LetField::DoTraverse(DataDepVisitor *visitor)
+	{ 
+	return Field::DoTraverse(visitor) &&
+	       expr()->Traverse(visitor); 
+	}
+
+bool LetField::RequiresAnalyzerContext() const 
+	{ 
+	return Field::RequiresAnalyzerContext() ||
+	       (expr() && expr()->RequiresAnalyzerContext()); 
+	}
+
+void LetField::Prepare(Env* env)
+	{
+	if ( ! type_ )
+		{
+		ASSERT(expr_);
+		type_ = expr_->DataType(env);
+		if ( type_ )
+			type_ = type_->Clone();		
+		else
+			type_ = extern_type_int->Clone();
+
+		foreach(i, AttrList, attrs_)
+			ProcessAttr(*i);
+		}
+
+	Field::Prepare(env);
+	env->SetEvalMethod(id_, this);
+	}
+
+void LetField::GenInitCode(Output* out_cc, Env* env)
+	{
+	int v;
+	if ( expr_ && expr_->ConstFold(env, &v) )
+		{
+		DEBUG_MSG("Folding const for `%s'\n", id_->Name());
+		GenEval(out_cc, env);
+		}
+	else
+		type_->GenInitCode(out_cc, env);
+	}
+
+void LetField::GenParseCode(Output* out_cc, Env* env)
+	{
+	if ( env->Evaluated(id_) )
+		return;
+
+	if ( type_->attr_if_expr() )
+		{
+		// A conditional field
+
+		env->Evaluate(out_cc, type_->has_value_var());
+
+		// force evaluation of IDs contained in this expr
+		expr()->ForceIDEval(out_cc, env);
+
+		out_cc->println("if ( %s )", 
+			env->RValue(type_->has_value_var()));
+		out_cc->inc_indent();
+		out_cc->println("{");
+		}
+
+	out_cc->println("%s = %s;", 
+		env->LValue(id_), 
+	       	expr()->EvalExpr(out_cc, env));
+	if ( ! env->Evaluated(id_) )
+		env->SetEvaluated(id_);
+
+	if ( type_->attr_if_expr() )
+		{
+		out_cc->println("}");
+		out_cc->dec_indent();
+		}
+	}
+
+void LetField::GenEval(Output* out_cc, Env* env)
+	{
+	GenParseCode(out_cc, env);
+	}
+
+LetDecl::LetDecl(ID *id, Type *type, Expr *expr)
+ 	: Decl(id, LET), type_(type), expr_(expr)
+	{
+	if ( ! type_ )
+		{
+		ASSERT(expr_);
+	        type_ = expr_->DataType(global_env());
+		if ( type_ )
+			type_ = type_->Clone();		
+		else
+			type_ = extern_type_int->Clone();
+		}
+
+	Env *env = global_env();
+	int c;
+	if ( expr_ && expr_->ConstFold(env, &c) )
+		env->AddConstID(id_, c);
+	else
+		env->AddID(id_, GLOBAL_VAR, type_);
+	}
+
+LetDecl::~LetDecl()
+	{
+	delete id_;
+	delete type_;
+	delete expr_;
+	}
+
+void LetDecl::Prepare()
+	{
+	}
+
+void LetDecl::GenForwardDeclaration(Output* out_h)
+	{
+	}
+
+void LetDecl::GenCode(Output * out_h, Output *out_cc)
+	{
+	out_h->println("extern %s const %s;",
+		type_->DataTypeStr().c_str(),
+		global_env()->RValue(id_));
+	GenEval(out_cc, global_env());
+	}
+
+void LetDecl::GenEval(Output *out_cc, Env * /* env */)
+	{
+	Env *env = global_env();
+	out_cc->println("%s %s = %s;", 
+		fmt("%s const", type_->DataTypeStr().c_str()),
+		env->LValue(id_), 
+	       	expr_->EvalExpr(out_cc, env));
+
+	if ( ! env->Evaluated(id_) )
+		env->SetEvaluated(id_);
+	}

tools/binpac/src/pac_let.h

@@ -0,0 +1,48 @@
+#ifndef pac_let_h
+#define pac_let_h
+
+#include "pac_decl.h"
+#include "pac_field.h"
+
+class LetField : public Field, Evaluatable
+{
+public:
+	LetField(ID* arg_id, Type *type, Expr* arg_expr);
+	~LetField();
+
+	Expr *expr() const			{ return expr_; }
+
+	void Prepare(Env* env);
+
+	void GenInitCode(Output* out, Env* env);
+	void GenParseCode(Output* out, Env* env);
+	void GenEval(Output* out, Env* env);
+
+	bool RequiresAnalyzerContext() const;
+
+protected:
+	bool DoTraverse(DataDepVisitor *visitor);
+
+protected:
+	Expr* expr_;
+};
+
+class LetDecl : public Decl, Evaluatable
+{
+public:
+	LetDecl(ID *id, Type *type, Expr *expr);
+	~LetDecl();
+
+	Expr *expr() const	{ return expr_; }
+
+	void Prepare();
+	void GenForwardDeclaration(Output *out_h);
+	void GenCode(Output *out_h, Output *out_cc);
+	void GenEval(Output* out, Env* env);
+
+private:
+	Type *type_;
+	Expr *expr_;
+};
+
+#endif  // pac_let_h

tools/binpac/src/pac_main.cc

@@ -0,0 +1,259 @@
+// $Id: pac_main.cc 3320 2006-06-20 21:19:33Z rpang $
+
+#include <unistd.h>
+#include <ctype.h>
+
+#include "pac_common.h"
+#include "pac_decl.h"
+#include "pac_exttype.h"
+#include "pac_id.h"
+#include "pac_output.h"
+#include "pac_parse.h"
+#include "pac_type.h"
+#include "pac_utils.h"
+#include "pac_exception.h"
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+extern int yydebug;
+extern int yyparse();
+extern void switch_to_file(FILE* fp_input);
+string input_filename;
+
+bool FLAGS_pac_debug = false;
+string FLAGS_output_directory;
+vector<string> FLAGS_include_directories;
+
+Output* header_output = 0;
+Output* source_output = 0;
+
+void add_to_include_directories(string dirs)
+	{
+	unsigned int dir_begin = 0, dir_end;
+	while ( dir_begin < dirs.length() )
+		{
+		for ( dir_end = dir_begin; dir_end < dirs.length(); ++dir_end)
+			if ( dirs[dir_end] == ':' )
+				break;
+
+		string dir = dirs.substr(dir_begin, dir_end - dir_begin);
+
+		// Add a trailing '/' if necessary
+		if ( dir.length() > 0 && *(dir.end() - 1) != '/' )
+			dir += '/';
+			
+		FLAGS_include_directories.push_back(dir);
+		dir_begin = dir_end + 1;
+		}
+	}
+
+void pac_init()
+	{
+	init_builtin_identifiers();
+	Type::init();
+	}
+
+void insert_comments(Output* out, const char* source_filename)
+	{
+	out->println("// This file is automatically generated from %s.\n",
+		source_filename);
+	}
+
+void insert_basictype_defs(Output* out)
+	{
+	out->println("#ifndef pac_type_defs");
+	out->println("#define pac_type_defs");
+	out->println("");
+	out->println("typedef char int8;");
+	out->println("typedef short int16;");
+	out->println("typedef long int32;");
+	out->println("typedef unsigned char uint8;");
+	out->println("typedef unsigned short uint16;");
+	out->println("typedef unsigned long uint32;");
+	out->println("");
+	out->println("#endif /* pac_type_defs */");
+	out->println("");
+	}
+
+void insert_byteorder_macros(Output* out)
+	{
+	out->println("#define FixByteOrder16(x)	(byteorder == HOST_BYTEORDER ? (x) : pac_swap16(x))");
+	out->println("#define FixByteOrder32(x)	(byteorder == HOST_BYTEORDER ? (x) : pac_swap32(x))");
+	out->println("");
+	}
+
+const char* to_id(const char* s)
+	{
+	static char t[1024];
+	int i;
+	for ( i = 0; s[i] && i < (int) sizeof(t) - 1; ++i )
+		t[i] = isalnum(s[i]) ? s[i] : '_';
+	if ( isdigit(t[0]) )
+		t[0] = '_'; 
+	t[i] = '\0';
+	return t;
+	}
+
+int compile(const char* filename)
+	{
+	FILE* fp_input = fopen(filename, "r");
+	if ( ! fp_input )
+		{
+		perror(fmt("Error in opening %s", filename));
+		return -1;
+		}
+	input_filename = filename;
+
+	string basename;
+
+	if ( ! FLAGS_output_directory.empty() )
+		{
+		// Strip leading directories of filename
+		const char *last_slash = strrchr(filename, '/');
+		if ( last_slash )
+			basename = last_slash + 1;
+		else
+			basename = filename;
+		basename = FLAGS_output_directory + "/" + basename;
+		}
+	else
+		basename = filename;
+
+	// If the file name ends with ".pac"
+	if ( basename.length() > 4 &&
+	     basename.substr(basename.length() - 4) == ".pac" )
+		{
+		basename = basename.substr(0, basename.length() - 4);
+		}
+
+	basename += "_pac";
+
+	DEBUG_MSG("Output file: %s.{h,cc}\n", basename.c_str());
+
+	int ret = 0;
+
+	try 
+		{
+		switch_to_file(fp_input);
+		if ( yyparse() )
+			return 1;
+
+		Output out_h(fmt("%s.h", basename.c_str()));
+		Output out_cc(fmt("%s.cc", basename.c_str()));
+
+		header_output = &out_h;
+		source_output = &out_cc;
+
+		insert_comments(&out_h, filename);
+		insert_comments(&out_cc, filename);
+
+		const char* filename_id = to_id(filename);
+
+		out_h.println("#ifndef %s_h", filename_id);
+		out_h.println("#define %s_h", filename_id);
+		out_h.println("");
+		out_h.println("#include <vector>");
+		out_h.println("");
+		out_h.println("#include \"binpac.h\"");
+		out_h.println("");
+
+		out_cc.println("#include \"%s.h\"\n", basename.c_str());
+
+		Decl::ProcessDecls(&out_h, &out_cc);
+
+		out_h.println("#endif /* %s_h */", filename_id);
+		} 
+	catch ( OutputException& e ) 
+		{
+		fprintf(stderr, "Error in compiling %s: %s\n",
+			filename, e.errmsg());
+		ret = 1;
+		}
+	catch ( Exception& e )
+		{
+		fprintf(stderr, "%s\n", e.msg());
+		exit(1);
+		}
+
+	header_output = 0;
+	source_output = 0;
+	input_filename = "";
+	fclose(fp_input);
+
+	return ret;
+	}
+
+void usage()
+	{
+#ifdef VERSION
+	fprintf(stderr, "binpac version %s\n", VERSION);
+#endif
+	fprintf(stderr, "usage: binpac [options] <pac files>\n");
+	fprintf(stderr, "     <pac files>           | pac-language input files\n");
+	fprintf(stderr, "     -d <dir>              | use given directory for compiler output\n");
+	fprintf(stderr, "     -D                    | enable debugging output\n");
+	fprintf(stderr, "     -h                    | show command line help\n");
+	fprintf(stderr, "     -I <dir>              | include <dir> in input file search path\n");
+	exit(1);
+	}
+
+int main(int argc, char* argv[])
+	{
+#ifdef HAVE_MALLOC_OPTIONS
+	extern char *malloc_options;
+#endif
+	int o;
+	while ( (o = getopt(argc, argv, "DI:d:h")) != -1 )
+		{
+		switch(o)
+			{
+			case 'D': 
+				yydebug = 1;
+				FLAGS_pac_debug = true;
+#ifdef HAVE_MALLOC_OPTIONS
+				malloc_options = "A";
+#endif
+				break;
+
+			case 'I':
+				// Add to FLAGS_include_directories
+				add_to_include_directories(optarg);
+				break;
+
+			case 'd': 
+				FLAGS_output_directory = optarg;
+				break;
+
+			case 'h':
+				usage();
+				break;
+			}
+		}
+
+	// Strip the trailing '/'s
+	while ( ! FLAGS_output_directory.empty() &&
+	        *(FLAGS_output_directory.end() - 1) == '/' )
+		{
+		FLAGS_output_directory.erase(FLAGS_output_directory.end()-1);
+		}
+
+	// Add the current directory to FLAGS_include_directories
+	add_to_include_directories(".");
+
+	pac_init();
+
+	argc -= optind;
+	argv += optind;
+	if ( argc == 0 )
+		compile("-");
+
+	int ret = 0;
+	for ( int i = 0; i < argc; ++i )
+		if ( compile(argv[i]) )
+			ret = 1;
+
+	return ret;
+	}
+

tools/binpac/src/pac_number.h

@@ -0,0 +1,21 @@
+#ifndef pac_number_h
+#define pac_number_h
+
+#include "pac_common.h"
+
+class Number : public Object
+{
+public:
+	Number(int arg_n)
+		: s(fmt("%d", arg_n)), n(arg_n) {}
+	Number(const char* arg_s, int arg_n)
+		: s(arg_s), n(arg_n) {}
+	const char* Str() const 	{ return s.c_str(); }
+	int Num() const 		{ return n; }
+
+protected:
+	const string s;
+	const int n;
+};
+
+#endif  // pac_number_h

tools/binpac/src/pac_output.cc

@@ -0,0 +1,61 @@
+// $Id: pac_output.cc 3225 2006-06-08 00:00:01Z vern $
+
+#include <string.h>
+#include <errno.h>
+#include <stdarg.h>
+#include <stdio.h>
+
+#include "pac_utils.h"
+#include "pac_output.h"
+
+OutputException::OutputException(const char* arg_msg)
+	{
+	msg = arg_msg;
+	}
+
+OutputException::~OutputException()
+	{
+	}
+
+Output::Output(const char* filename)
+	{
+	fp = fopen(filename, "w");
+	if ( ! fp )
+		throw OutputException(strerror(errno));
+	indent_ = 0;
+	}
+
+Output::~Output()
+	{
+	if ( fp )
+		fclose(fp);
+	}
+
+int Output::print(const char* fmt, va_list ap)
+	{
+	int r = vfprintf(fp, fmt, ap);
+	if ( r == -1 )
+		throw OutputException(strerror(errno));
+	return r;
+	}
+
+int Output::print(const char* fmt, ...)
+	{
+	va_list ap;
+	va_start(ap, fmt);
+	return print(fmt, ap);
+	}
+
+int Output::println(const char* fmt, ...)
+	{
+	for ( int i = 0; i < indent(); ++i )
+		fprintf(fp, "\t");
+
+	int r;
+	va_list ap;
+	va_start(ap, fmt);
+	r = print(fmt, ap);
+
+	fprintf(fp, "\n");
+	return r;
+	}

tools/binpac/src/pac_output.h

@@ -0,0 +1,42 @@
+// $Id: pac_output.h 3225 2006-06-08 00:00:01Z vern $
+
+#ifndef pac_output_h
+#define pac_output_h
+
+#include <stdio.h>
+#include <stdarg.h>
+#include <string>
+
+using namespace std;
+
+class OutputException {
+public:
+	OutputException(const char* arg_msg);
+	~OutputException();
+	const char* errmsg() const { return msg.c_str(); }
+
+protected:
+	string msg;
+};
+
+class Output {
+public:
+	Output(const char *filename);
+	~Output();
+
+	int println(const char* fmt, ...);
+	int print(const char* fmt, ...);
+
+	int indent() const { return indent_; }
+
+	void inc_indent() { ++indent_; }
+	void dec_indent() { --indent_; }
+
+protected:
+	int print(const char* fmt, va_list ap);
+
+	FILE* fp;
+	int indent_;
+};
+
+#endif /* pac_output_h */

tools/binpac/src/pac_param.cc

@@ -0,0 +1,70 @@
+#include "pac_decl.h"
+#include "pac_exttype.h"
+#include "pac_field.h"
+#include "pac_id.h"
+#include "pac_output.h"
+#include "pac_type.h"
+#include "pac_utils.h"
+
+#include "pac_param.h"
+
+Param::Param(ID* id, Type *type)
+	: id_(id), type_(type)
+	{
+	if ( ! type_ )
+		type_ = extern_type_int->Clone();
+
+	decl_str_ = strfmt("%s %s", 
+	                   type_->DataTypeConstRefStr().c_str(), 
+	                   id_->Name());
+
+	param_field_ = new ParamField(this);
+	}
+
+Param::~Param()
+	{
+	}
+
+const string &Param::decl_str() const
+	{ 
+	ASSERT(!decl_str_.empty()); 
+	return decl_str_; 
+	}
+
+string ParamDecls(ParamList *params)
+	{
+	string param_decls;
+
+	int first = 1;
+	foreach (i, ParamList, params)
+			{
+			Param* p = *i;
+			const char* decl_str = p->decl_str().c_str();
+			if ( first )
+				first = 0;
+			else
+				param_decls += ", ";
+			param_decls += decl_str;
+			}
+	return param_decls;
+	}
+
+ParamField::ParamField(const Param *param)
+	: Field(PARAM_FIELD, 
+		TYPE_NOT_TO_BE_PARSED | CLASS_MEMBER | PUBLIC_READABLE,
+		param->id(), 
+		param->type())
+	{
+	}
+
+void ParamField::GenInitCode(Output *out_cc, Env *env)
+	{
+	out_cc->println("%s = %s;", 
+		env->LValue(id()), id()->Name());
+	env->SetEvaluated(id());
+	}
+
+void ParamField::GenCleanUpCode(Output* out_cc, Env* env)
+	{
+	// Do nothing
+	}

tools/binpac/src/pac_param.h

@@ -0,0 +1,48 @@
+#ifndef pac_param_h
+#define pac_param_h
+
+#include "pac_common.h"
+#include "pac_field.h"
+
+class Param : public Object
+{
+public:
+	Param(ID* id, Type* type); 
+	~Param();
+
+	ID *id() const			{ return id_; }
+	Type *type() const		{ return type_; }
+	const string & decl_str() const;
+	Field *param_field() const	{ return param_field_; }
+
+private:
+	ID* id_;
+	Type* type_;
+	string decl_str_;
+	Field *param_field_;
+};
+
+class ParamField : public Field
+{
+public:
+	ParamField(const Param *param);
+
+	void GenInitCode(Output *out, Env *env);
+	void GenCleanUpCode(Output* out, Env* env);
+};
+
+// Returns the string with a list of param declarations separated by ','.
+string ParamDecls(ParamList *params);
+
+#if 0
+// Generate assignments to parameters, in the form of "%s_ = %s;" % (id, id).
+void GenParamAssignments(ParamList *params, Output *out_cc, Env *env);
+
+// Generate public access methods to parameter members.
+void GenParamPubDecls(ParamList *params, Output *out_h, Env *env);
+
+// Generate private definitions of parameter members.
+void GenParamPrivDecls(ParamList *params, Output *out_h, Env *env);
+#endif
+
+#endif  // pac_param_h

tools/binpac/src/pac_paramtype.cc

@@ -0,0 +1,289 @@
+#include "pac_context.h"
+#include "pac_dataptr.h"
+#include "pac_exception.h"
+#include "pac_expr.h"
+#include "pac_output.h"
+#include "pac_paramtype.h"
+#include "pac_typedecl.h"
+
+ParameterizedType::ParameterizedType(ID* type_id, ExprList* args)
+	: Type(PARAMETERIZED), type_id_(type_id), args_(args)
+	{
+	checking_requires_analyzer_context_ = false;
+	}
+
+ParameterizedType::~ParameterizedType()
+	{
+	}
+
+string ParameterizedType::EvalMember(const ID *member_id) const
+	{
+	Type *ty = ReferredDataType(true);
+	return strfmt("->%s", ty->env()->RValue(member_id));
+	}
+
+string ParameterizedType::class_name() const
+	{ 
+	return type_id_->Name(); 
+	}
+
+Type *ParameterizedType::DoClone() const
+	{
+	return new ParameterizedType(type_id_->clone(), args_);
+	}
+
+void ParameterizedType::AddParamArg(Expr *arg)
+	{
+	args_->push_back(arg);
+	}
+
+bool ParameterizedType::DefineValueVar() const
+	{
+	return true;
+	}
+
+string ParameterizedType::DataTypeStr() const
+	{
+	return strfmt("%s *", type_id_->Name());
+	}
+
+Type *ParameterizedType::MemberDataType(const ID *member_id) const
+	{
+	Type *ref_type = TypeDecl::LookUpType(type_id_);
+	if ( ! ref_type )
+		return 0;
+	return ref_type->MemberDataType(member_id);
+	}
+
+Type *ParameterizedType::ReferredDataType(bool throw_exception) const
+	{
+	Type* type = TypeDecl::LookUpType(type_id_);
+	if ( ! type )
+		{
+		DEBUG_MSG("WARNING: cannot find referenced type for %s\n",
+			type_id_->Name());
+		if ( throw_exception )
+			throw ExceptionIDNotFound(type_id_);
+		}
+	return type;
+	}
+
+int ParameterizedType::StaticSize(Env* env) const
+	{
+	return ReferredDataType(true)->StaticSize(env);
+	}
+
+void ParameterizedType::DoMarkIncrementalInput()
+	{
+	Type *ty = ReferredDataType(true);
+
+	ty->MarkIncrementalInput();
+
+	buffer_input_ = ty->buffer_input();
+	incremental_parsing_ = ty->incremental_parsing();
+	}
+
+Type::BufferMode ParameterizedType::buffer_mode() const
+	{
+	// Note that the precedence is on attributes (&oneline or &length) 
+	// specified on the parameterized type directly than on the type
+	// declaration. 
+	//
+	// If both &oneline and &length are specified at the same place, 
+	// use &length.
+	//
+	BufferMode mode = Type::buffer_mode();
+	Type *ty = ReferredDataType(true);
+
+	if ( mode != NOT_BUFFERABLE )
+		return mode;
+	else if ( ty->BufferableByLength() )
+		return BUFFER_BY_LENGTH;
+	else if ( ty->BufferableByLine() )
+		return BUFFER_BY_LINE;
+
+	return NOT_BUFFERABLE;
+	}
+
+bool ParameterizedType::ByteOrderSensitive() const
+	{
+	return ReferredDataType(true)->RequiresByteOrder();
+	}
+
+bool ParameterizedType::DoTraverse(DataDepVisitor *visitor)
+	{
+	if ( ! Type::DoTraverse(visitor) )
+		return false;
+
+	foreach(i, ExprList, args_)
+		if ( ! (*i)->Traverse(visitor) )
+			return false;
+
+	Type *ty = ReferredDataType(false);
+	if ( ty && ! ty->Traverse(visitor) )
+		return false;
+
+	return true;
+	}
+
+bool ParameterizedType::RequiresAnalyzerContext()
+	{
+	if ( checking_requires_analyzer_context_ )
+		return false;
+	checking_requires_analyzer_context_ = true;
+
+	bool ret = false;
+	// If any argument expression refers to analyzer context
+	foreach(i, ExprList, args_)
+		if ( (*i)->RequiresAnalyzerContext() )
+			{
+			ret = true;
+			break;
+			}
+	ret = ret ||
+	      Type::RequiresAnalyzerContext();
+
+	if ( ! ret )
+		{
+		Type *ty = ReferredDataType(false);
+		if ( ty )
+	      		ret = ty->RequiresAnalyzerContext();
+		}
+
+	checking_requires_analyzer_context_ = false;
+	return ret;
+	}
+
+void ParameterizedType::GenInitCode(Output* out_cc, Env* env)
+	{
+	ASSERT(persistent());
+	out_cc->println("%s = 0;", env->LValue(value_var()));
+	Type::GenInitCode(out_cc, env);
+	}
+
+void ParameterizedType::GenCleanUpCode(Output* out_cc, Env* env)
+	{
+	Type *ty = ReferredDataType(false);
+	if ( ty && ty->attr_refcount() )
+		out_cc->println("Unref(%s);", lvalue());
+	else
+		out_cc->println("delete %s;", lvalue());
+	out_cc->println("%s = 0;", lvalue());
+	Type::GenCleanUpCode(out_cc, env);
+	}
+
+string ParameterizedType::EvalParameters(Output* out_cc, Env *env) const
+	{
+	string arg_str;
+
+	int first = 1;
+	foreach (i, ExprList, args_)
+		{
+		Expr* e = *i;
+		if ( first )
+			first = 0;
+		else
+			arg_str += ", ";
+		arg_str += e->EvalExpr(out_cc, env);
+		}
+
+	return arg_str;
+	}
+
+void ParameterizedType::GenNewInstance(Output *out_cc, Env *env)
+	{
+	out_cc->println("%s = new %s(%s);", 
+		lvalue(), 
+		type_id_->Name(), 
+		EvalParameters(out_cc, env).c_str());
+	}
+
+void ParameterizedType::DoGenParseCode(Output* out_cc, Env* env,
+		const DataPtr& data, int flags)
+	{
+	DEBUG_MSG("DoGenParseCode for %s\n", type_id_->Name());
+
+	Type *ref_type = ReferredDataType(true);
+
+	const char *parse_func;
+	string parse_params;
+
+	if ( buffer_mode() == BUFFER_NOTHING )
+	        {
+		ASSERT(!ref_type->incremental_input());
+		parse_func = kParseFuncWithoutBuffer;
+		parse_params = "0, 0";
+		}
+	else if ( ref_type->incremental_input() )
+		{
+		parse_func = kParseFuncWithBuffer;
+		parse_params = env->RValue(flow_buffer_id);
+		}
+	else
+		{
+		parse_func = kParseFuncWithoutBuffer;
+		parse_params = strfmt("%s, %s",
+			data.ptr_expr(), 
+			env->RValue(end_of_data));
+		}
+
+	if ( RequiresAnalyzerContext::compute(ref_type) )
+		{
+		parse_params += strfmt(", %s", env->RValue(analyzer_context_id));
+		}
+
+	if ( ref_type->RequiresByteOrder() )
+		{
+		env->Evaluate(out_cc, byteorder_id);
+		parse_params += strfmt(", %s", env->RValue(byteorder_id));
+		}
+
+	string call_parse_func = strfmt("%s->%s(%s)",
+			lvalue(), // parse() needs an LValue
+			parse_func,
+			parse_params.c_str());
+
+	if ( incremental_input() )
+		{
+		if ( buffer_mode() == BUFFER_NOTHING )
+		        {
+		        out_cc->println("%s;", call_parse_func.c_str());
+			out_cc->println("%s = true;", 
+				env->LValue(parsing_complete_var()));
+			}
+		else
+		        {
+			ASSERT(parsing_complete_var());
+			out_cc->println("%s = %s;",
+				env->LValue(parsing_complete_var()),
+				call_parse_func.c_str());
+
+			// parsing_complete_var might have been already
+			// evaluated when set to false
+			if ( ! env->Evaluated(parsing_complete_var()) )
+			        env->SetEvaluated(parsing_complete_var());
+			}
+		}
+	else
+		{
+		if ( AddSizeVar(out_cc, env) )
+			{
+			out_cc->println("%s = %s;", 
+				env->LValue(size_var()),
+				call_parse_func.c_str());
+			env->SetEvaluated(size_var());
+			}
+		else
+			{
+			out_cc->println("%s;", 
+				call_parse_func.c_str());
+			}
+		}
+	}
+
+void ParameterizedType::GenDynamicSize(Output* out_cc, Env* env,
+		const DataPtr& data)
+	{
+	GenParseCode(out_cc, env, data, 0);
+	}
+

tools/binpac/src/pac_paramtype.h

@@ -0,0 +1,62 @@
+#ifndef pac_paramtype_h
+#define pac_paramtype_h
+
+#include "pac_type.h"
+
+// An instantiated type: ID + expression list
+class ParameterizedType : public Type
+{
+public:
+	ParameterizedType(ID *type_id, ExprList *args);
+	~ParameterizedType();
+
+	Type *clone() const;
+
+	string EvalMember(const ID *member_id) const;
+	// Env *member_env() const;
+
+	void AddParamArg(Expr *arg);
+
+	bool DefineValueVar() const;
+	string DataTypeStr() const;
+	string DefaultValue() const	{ return "0"; }
+	Type *MemberDataType(const ID *member_id) const;
+
+	// "throw_exception" specifies whether to throw an exception
+	// if the referred data type is not found
+	Type *ReferredDataType(bool throw_exception) const;
+
+	void GenCleanUpCode(Output *out, Env *env);
+
+	int StaticSize(Env *env) const;
+
+	bool IsPointerType() const	{ return true; }
+
+	bool ByteOrderSensitive() const;
+	bool RequiresAnalyzerContext();
+
+	void GenInitCode(Output *out_cc, Env *env);
+
+	string class_name() const;
+	string EvalParameters(Output *out_cc, Env *env) const;
+
+	BufferMode buffer_mode() const;
+
+protected:
+	void GenNewInstance(Output *out, Env *env);
+
+	bool DoTraverse(DataDepVisitor *visitor);
+	Type *DoClone() const;
+	void DoMarkIncrementalInput();
+
+private:
+	ID *type_id_;
+	ExprList *args_;
+	char *data_type_;
+	bool checking_requires_analyzer_context_;
+
+	void DoGenParseCode(Output *out, Env *env, const DataPtr& data, int flags);
+	void GenDynamicSize(Output *out, Env *env, const DataPtr& data);
+};
+
+#endif  // pac_paramtype_h

tools/binpac/src/pac_primitive.cc

@@ -0,0 +1,39 @@
+#include "pac_dbg.h"
+#include "pac_expr.h"
+#include "pac_id.h"
+#include "pac_primitive.h"
+#include "pac_type.h"
+
+string PPVal::ToCode(Env *env)
+	{
+	ASSERT(expr_);
+	return string(expr_->EvalExpr(0, env));
+	}
+
+string PPSet::ToCode(Env *env)
+	{
+	ASSERT(expr_);
+	return expr_->SetFunc(0, env);
+	}
+
+string PPType::ToCode(Env *env)
+	{
+	Type *type = expr_->DataType(env);
+	if ( ! type )
+		{
+		}
+	return type->DataTypeStr();
+	}
+
+string PPConstDef::ToCode(Env *env)
+	{
+	Type *type = expr_->DataType(env);
+	env->AddID(id_, TEMP_VAR, type);
+	env->SetEvaluated(id_);
+
+	string type_str = type->DataTypeStr();
+	return strfmt("%s %s = %s", 
+	              type_str.c_str(),
+	              env->LValue(id_),
+	              expr_->EvalExpr(0, env));
+	}

tools/binpac/src/pac_primitive.h

@@ -0,0 +1,75 @@
+#ifndef pac_primitive_h
+#define pac_primitive_h
+
+#include "pac_common.h"
+
+class PacPrimitive
+{
+public:
+	enum PrimitiveType { VAL, SET, TYPE, CONST_DEF };
+
+	explicit PacPrimitive(PrimitiveType type) : type_(type) {}
+	virtual ~PacPrimitive() {}
+
+	PrimitiveType type() const	{ return type(); }
+
+	virtual string ToCode(Env *env) = 0;
+
+private:
+	PrimitiveType type_;
+};
+
+class PPVal : public PacPrimitive
+{
+public:
+	PPVal(Expr *expr) : PacPrimitive(VAL), expr_(expr) {}
+	Expr *expr() const	{ return expr_; }
+
+	string ToCode(Env *env);
+
+private:
+	Expr *expr_;
+};
+
+class PPSet : public PacPrimitive
+{
+public:
+	PPSet(Expr *expr) : PacPrimitive(SET), expr_(expr) {}
+	Expr *expr() const	{ return expr_; }
+
+	string ToCode(Env *env);
+
+private:
+	Expr *expr_;
+};
+
+class PPType : public PacPrimitive
+{
+public:
+	PPType(Expr *expr) : PacPrimitive(TYPE), expr_(expr) {}
+	Expr *expr() const	{ return expr_; }
+
+	string ToCode(Env *env);
+
+private:
+	Expr *expr_;
+};
+
+class PPConstDef : public PacPrimitive
+{
+public:
+	PPConstDef(const ID *id, Expr *expr) 
+		: PacPrimitive(CONST_DEF), 
+		  id_(id),
+		  expr_(expr) {}
+	const ID *id() const	{ return id_; }
+	Expr *expr() const	{ return expr_; }
+
+	string ToCode(Env *env);
+
+private:
+	const ID *id_;
+	Expr *expr_;
+};
+
+#endif  // pac_primitive_h

tools/binpac/src/pac_record.cc

@@ -0,0 +1,680 @@
+#include "pac_attr.h"
+#include "pac_dataptr.h"
+#include "pac_exception.h"
+#include "pac_expr.h"
+#include "pac_exttype.h"
+#include "pac_field.h"
+#include "pac_output.h"
+#include "pac_record.h"
+#include "pac_type.h"
+#include "pac_typedecl.h"
+#include "pac_utils.h"
+#include "pac_varfield.h"
+
+
+RecordType::RecordType(RecordFieldList* record_fields)
+	: Type(RECORD)
+	{
+	// Here we assume that the type is a standalone type.
+	value_var_ = 0;
+
+	// Put all fields in fields_
+	foreach (i, RecordFieldList, record_fields)
+		AddField(*i);
+
+	// Put RecordField's in record_fields_
+	record_fields_ = record_fields;
+
+	parsing_dataptr_var_field_ = 0;
+	}
+
+RecordType::~RecordType()
+	{
+	// Do not delete_list(RecordFieldList, record_fields_)
+	// because the fields are also in fields_.
+	delete record_fields_;
+	delete parsing_dataptr_var_field_;
+	}
+
+const ID *RecordType::parsing_dataptr_var() const
+	{ 
+	return parsing_dataptr_var_field_ ?
+		parsing_dataptr_var_field_->id() : 0;
+	}
+
+bool RecordType::DefineValueVar() const
+	{
+	return false;
+	}
+
+string RecordType::DataTypeStr() const
+	{
+	ASSERT(type_decl());
+	return strfmt("%s *", type_decl()->class_name().c_str());
+	}
+
+void RecordType::Prepare(Env* env, int flags)
+	{
+	ASSERT(flags & TO_BE_PARSED);
+
+	RecordField *prev = 0;
+	int offset = 0;
+	int seq = 0;
+	foreach (i, RecordFieldList, record_fields_)
+		{
+		RecordField *f = *i;
+		f->set_record_type(this);
+		f->set_prev(prev);
+		if ( prev )
+			prev->set_next(f);
+		prev = f;
+		if ( offset >= 0 )
+			{
+			f->set_static_offset(offset);
+			int w = f->StaticSize(env, offset);
+			if ( w < 0 )
+				offset = -1;
+			else
+				offset += w;
+			}
+		++seq;
+		f->set_parsing_state_seq(seq);
+		}
+
+	if ( incremental_parsing() )
+		{
+#if 0
+		ASSERT(! parsing_state_var_field_);
+		ID *parsing_state_var_id = new ID("parsing_state");
+		parsing_state_var_field_ = new PrivVarField(
+			parsing_state_var_id, extern_type_int->Clone());
+		AddField(parsing_state_var_field_);
+
+		ID *parsing_dataptr_var_id = new ID("parsing_dataptr");
+		parsing_dataptr_var_field_ = new TempVarField(
+			parsing_dataptr_var_id, extern_type_const_byteptr->Clone());
+		parsing_dataptr_var_field_->Prepare(env);
+#endif
+		}
+
+	Type::Prepare(env, flags);
+	}
+
+void RecordType::GenPubDecls(Output* out_h, Env* env)
+	{
+	Type::GenPubDecls(out_h, env);
+	}
+
+void RecordType::GenPrivDecls(Output* out_h, Env* env)
+	{
+	Type::GenPrivDecls(out_h, env);
+	}
+
+void RecordType::GenInitCode(Output* out_cc, Env* env)
+	{
+	Type::GenInitCode(out_cc, env);
+	}
+
+void RecordType::GenCleanUpCode(Output* out_cc, Env* env)
+	{
+	Type::GenCleanUpCode(out_cc, env);
+	}
+
+void RecordType::DoGenParseCode(Output* out_cc, Env* env, 
+		const DataPtr& data, int flags)
+	{
+	  if ( !incremental_input() && StaticSize(env) >= 0 )
+		GenBoundaryCheck(out_cc, env, data);
+
+	if ( incremental_parsing() )
+		{
+		out_cc->println("switch ( %s ) {",
+			env->LValue(parsing_state_id));
+
+		out_cc->println("case 0:");
+		out_cc->inc_indent();
+		foreach (i, RecordFieldList, record_fields_)
+			{
+			RecordField *f = *i;
+			f->GenParseCode(out_cc, env);
+			out_cc->println("");
+			}
+		out_cc->println("");
+		out_cc->println("%s = true;", 
+			env->LValue(parsing_complete_var()));
+		out_cc->dec_indent();
+		out_cc->println("}");
+		}
+	else
+		{
+		ASSERT(	data.id() == begin_of_data && 
+			data.offset() == 0 );
+		foreach (i, RecordFieldList, record_fields_)
+			{
+			RecordField *f = *i;
+			f->GenParseCode(out_cc, env);
+			out_cc->println("");
+			}
+		if ( incremental_input() )
+			{
+			ASSERT(parsing_complete_var());
+			out_cc->println("%s = true;", 
+				env->LValue(parsing_complete_var()));
+			}
+		}
+
+	if ( ! incremental_input() && AddSizeVar(out_cc, env) )
+		{
+		const DataPtr& end_of_record_dataptr = 
+			record_fields_->back()->getFieldEnd(out_cc, env);
+
+		out_cc->println("%s = %s - %s;", 
+			env->LValue(size_var()), 
+			end_of_record_dataptr.ptr_expr(), 
+			env->RValue(begin_of_data));
+		env->SetEvaluated(size_var());
+		}
+
+	if ( ! boundary_checked() )
+		{
+		RecordField *last_field = record_fields_->back();
+		if ( ! last_field->BoundaryChecked() )
+			GenBoundaryCheck(out_cc, env, data);
+		}
+	}
+
+void RecordType::GenDynamicSize(Output* out_cc, Env* env,
+		const DataPtr& data)
+	{
+	GenParseCode(out_cc, env, data, 0);
+	}
+
+int RecordType::StaticSize(Env* env) const
+	{
+	int tot_w = 0;
+	foreach (i, RecordFieldList, record_fields_)
+		{
+		RecordField *f = *i;
+		int w = f->StaticSize(env, tot_w);
+		if ( w < 0 )
+			return -1;
+		tot_w += w;
+		}
+	return tot_w;
+	}
+
+void RecordType::SetBoundaryChecked()
+	{
+	Type::SetBoundaryChecked();
+	foreach (i, RecordFieldList, record_fields_)
+		{
+		RecordField *f = *i;
+		f->SetBoundaryChecked();
+		}
+	}
+
+void RecordType::DoMarkIncrementalInput()
+	{
+	foreach (i, RecordFieldList, record_fields_)
+		{
+		RecordField *f = *i;
+		f->type()->MarkIncrementalInput();
+		}
+	}
+
+bool RecordType::DoTraverse(DataDepVisitor *visitor)
+	{
+	return Type::DoTraverse(visitor);
+	}
+
+bool RecordType::ByteOrderSensitive() const
+	{
+	foreach (i, RecordFieldList, record_fields_)
+		{
+		RecordField *f = *i;
+		if ( f->RequiresByteOrder() )
+			return true;
+		}
+	return false;
+	}
+
+RecordField::RecordField(FieldType tof, ID *id, Type *type)
+	: Field(tof, 
+		TYPE_TO_BE_PARSED | CLASS_MEMBER | PUBLIC_READABLE,
+		id, type)
+	{
+	begin_of_field_dataptr = 0;
+	end_of_field_dataptr = 0;
+	field_size_expr = 0;
+	field_offset_expr = 0;
+	end_of_field_dataptr_var = 0;
+	record_type_ = 0;
+	prev_ = 0;
+	next_ = 0;
+	static_offset_ = -1;
+	boundary_checked_ = false;
+	}
+
+RecordField::~RecordField()
+	{
+	delete begin_of_field_dataptr;
+	delete end_of_field_dataptr;
+	delete [] field_size_expr;
+	delete [] field_offset_expr;
+	delete end_of_field_dataptr_var;
+	}
+
+const DataPtr& RecordField::getFieldBegin(Output* out_cc, Env* env)
+	{
+	if ( prev() )
+		return prev()->getFieldEnd(out_cc, env);
+	else
+		{
+		// The first field
+		if ( ! begin_of_field_dataptr )
+			{
+			begin_of_field_dataptr = 
+				new DataPtr(env, begin_of_data, 0);
+			}
+		return *begin_of_field_dataptr;
+		}	
+	}
+
+const DataPtr& RecordField::getFieldEnd(Output* out_cc, Env* env)
+	{
+	if ( end_of_field_dataptr )
+		return *end_of_field_dataptr;
+
+	const DataPtr& begin_ptr = getFieldBegin(out_cc, env);
+
+	if ( record_type()->incremental_parsing() )
+		{
+		ASSERT(0);
+		if ( ! end_of_field_dataptr )
+			{
+			const ID *dataptr_var = 
+				record_type()->parsing_dataptr_var();
+			ASSERT(dataptr_var);
+
+			end_of_field_dataptr = 
+				new DataPtr(env, dataptr_var, 0);
+			}
+		}
+	else
+		{
+		int field_offset;
+		if ( begin_ptr.id() == begin_of_data )
+			field_offset = begin_ptr.offset();
+		else
+			field_offset = -1;	// unknown
+			
+		int field_size = StaticSize(env, field_offset);
+		if ( field_size >= 0 ) // can be statically determinted 
+			{
+			end_of_field_dataptr = new DataPtr(
+				env,
+				begin_ptr.id(), 
+				begin_ptr.offset() + field_size);
+			}
+		else
+			{
+			// If not, we add a variable for the offset after the field
+			end_of_field_dataptr_var = new ID(
+				fmt("dataptr_after_%s", id()->Name()));
+			env->AddID(end_of_field_dataptr_var, 
+		           	TEMP_VAR, 
+		           	extern_type_const_byteptr);
+
+			GenFieldEnd(out_cc, env, begin_ptr);
+
+			end_of_field_dataptr = new DataPtr(
+				env, 
+				end_of_field_dataptr_var, 
+				0);
+			}
+		}
+
+	return *end_of_field_dataptr;
+	}
+
+const char* RecordField::FieldSize(Output* out_cc, Env* env)
+	{
+	if ( field_size_expr )
+		return field_size_expr;
+
+	const DataPtr& begin = getFieldBegin(out_cc, env);
+	const DataPtr& end = getFieldEnd(out_cc, env);
+	if ( begin.id() == end.id() )
+		field_size_expr = nfmt("%d", end.offset() - begin.offset());
+	else
+		field_size_expr = nfmt("(%s - %s)", end.ptr_expr(), begin.ptr_expr());
+	return field_size_expr;
+	}
+
+const char* RecordField::FieldOffset(Output* out_cc, Env* env)
+	{
+	if ( field_offset_expr )
+		return field_offset_expr;
+
+	const DataPtr& begin = getFieldBegin(out_cc, env);
+	if ( begin.id() == begin_of_data )
+		field_offset_expr = nfmt("%d", begin.offset());
+	else
+		field_offset_expr = nfmt("(%s - %s)", 
+			begin.ptr_expr(), env->RValue(begin_of_data));
+	return field_offset_expr;
+	}
+
+// The reasoning behind AttemptBoundaryCheck is: "If my next field
+// can check its boundary, then I don't have to check mine, and it
+// will save me a boundary-check."
+bool RecordField::AttemptBoundaryCheck(Output* out_cc, Env* env)
+	{
+	if ( boundary_checked_ )
+		return true;
+
+	// If I do not even know my size till I parse the data, my
+	// next field won't be able to check its boundary now.  
+
+	const DataPtr& begin = getFieldBegin(out_cc, env);
+	if ( StaticSize(env, begin.AbsOffset(begin_of_data)) < 0 )
+		return false;
+
+	// Now we ask the next field to check its boundary. 
+	if ( next() && next()->AttemptBoundaryCheck(out_cc, env) ) 
+		{
+		// If it works, we are all set
+		SetBoundaryChecked();
+		return true;
+		}
+	else
+		// If it fails, then I can still try to do it by myself
+		return GenBoundaryCheck(out_cc, env);
+	}
+
+RecordDataField::RecordDataField(ID* id, Type* type)
+	: RecordField(RECORD_FIELD, id, type)
+	{
+	ASSERT(type_);
+	}
+
+RecordDataField::~RecordDataField()
+	{
+	}
+
+void RecordDataField::Prepare(Env* env)
+	{
+	Field::Prepare(env);
+	env->SetEvalMethod(id_, this);
+	env->SetField(id_, this);
+	}
+
+void RecordDataField::GenParseCode(Output* out_cc, Env* env)
+	{
+	if ( env->Evaluated(id()) )
+		return;
+
+	// Always evaluate record fields in order if parsing
+	// is incremental.
+	if ( record_type()->incremental_parsing() && prev() )
+		prev()->GenParseCode(out_cc, env);
+
+	DataPtr data(env, 0, 0);
+	if ( ! record_type()->incremental_parsing() )
+		{
+		data = getFieldBegin(out_cc, env);	
+		AttemptBoundaryCheck(out_cc, env);
+		}
+
+	out_cc->println("// Parse \"%s\"", id_->Name());
+#if 0
+	out_cc->println("DEBUG_MSG(\"%%.6f Parse %s\\n\", network_time());", 
+		id_->Name());
+#endif
+	type_->GenPreParsing(out_cc, env);
+	if ( type_->incremental_input() )
+		{
+		// The enclosing record type must be incrementally parsed
+		out_cc->println("%s = %d;", 
+			env->LValue(parsing_state_id),
+			parsing_state_seq());
+		out_cc->dec_indent();
+		out_cc->println("case %d:", parsing_state_seq());
+		out_cc->inc_indent();
+		out_cc->println("{");
+		}
+
+	type_->GenParseCode(out_cc, env, data, 0);
+
+	if ( record_type()->incremental_parsing() )
+		{
+		ASSERT(type_->incremental_input());
+
+		out_cc->println("if ( ! (%s) )", 
+			type_->parsing_complete(env).c_str());
+		out_cc->inc_indent();
+		out_cc->println("goto %s;", kNeedMoreData);
+		out_cc->dec_indent();
+		}
+
+	if ( record_type()->incremental_parsing() )
+		{
+#if 0
+		const ID *dataptr_var = 
+			record_type()->parsing_dataptr_var();
+		ASSERT(dataptr_var);
+		out_cc->println("%s += (%s);",
+			env->LValue(dataptr_var),
+			type_->DataSize(out_cc, env, data).c_str());
+#endif
+		out_cc->println("}");
+		}
+
+	SetBoundaryChecked();
+	}
+
+void RecordDataField::GenEval(Output* out_cc, Env* env)
+	{
+	GenParseCode(out_cc, env);
+	}
+
+void RecordDataField::GenFieldEnd(Output* out_cc, Env* env, 
+		const DataPtr& field_begin)
+	{
+	out_cc->println("const_byteptr const %s = %s + (%s);", 
+		env->LValue(end_of_field_dataptr_var),
+		field_begin.ptr_expr(),
+		type_->DataSize(out_cc, env, field_begin).c_str());
+	env->SetEvaluated(end_of_field_dataptr_var);
+
+	out_cc->println("BINPAC_ASSERT(%s <= %s);",
+		env->RValue(end_of_field_dataptr_var),
+		env->RValue(end_of_data));
+	}
+
+void RecordDataField::SetBoundaryChecked()
+	{
+	RecordField::SetBoundaryChecked();
+	type_->SetBoundaryChecked();
+	}
+
+bool RecordDataField::GenBoundaryCheck(Output* out_cc, Env* env)
+	{
+	if ( boundary_checked_ )
+		return true;
+
+	type_->GenBoundaryCheck(out_cc, env, getFieldBegin(out_cc, env));
+
+	SetBoundaryChecked();
+	return true;
+	}
+
+bool RecordDataField::DoTraverse(DataDepVisitor *visitor)
+	{ 
+	return Field::DoTraverse(visitor);
+	}
+
+bool RecordDataField::RequiresAnalyzerContext() const 
+	{ 
+	return Field::RequiresAnalyzerContext() ||
+	       type()->RequiresAnalyzerContext(); 
+	}
+
+RecordPaddingField::RecordPaddingField(ID* id, PaddingType ptype, Expr* expr)
+	: RecordField(PADDING_FIELD, id, 0), ptype_(ptype), expr_(expr)
+	{
+	wordsize_ = -1;
+	}
+
+RecordPaddingField::~RecordPaddingField()
+	{
+	}
+
+void RecordPaddingField::Prepare(Env* env)
+	{
+	Field::Prepare(env);
+	if ( ptype_ == PAD_TO_NEXT_WORD )
+		{
+		if ( ! expr_->ConstFold(env, &wordsize_) )
+			throw ExceptionPaddingError(this, 
+				fmt("padding word size not a constant"));
+		}
+	}
+
+void RecordPaddingField::GenParseCode(Output* out_cc, Env* env)
+	{
+	// Always evaluate record fields in order if parsing
+	// is incremental.
+	if ( record_type()->incremental_parsing() && prev() )
+		prev()->GenParseCode(out_cc, env);
+	}
+
+int RecordPaddingField::StaticSize(Env* env, int offset) const
+	{
+	int length;
+	int target_offset;
+	int offset_in_word;
+
+	switch ( ptype_ )
+		{
+		case PAD_BY_LENGTH:
+			return expr_->ConstFold(env, &length) ? length : -1;
+
+		case PAD_TO_OFFSET:
+			// If the current offset cannot be statically
+			// determined, we need to Generate code to
+			// check the offset
+			if ( offset == -1 )
+				return -1;
+
+			if ( ! expr_->ConstFold(env, &target_offset) )
+				return -1;
+
+			// If both the current and target offsets
+			// can be statically computed, we can get its
+			// static size
+			if ( offset > target_offset )
+				throw ExceptionPaddingError(
+					this,
+					fmt("current offset = %d, "
+					    "target offset = %d", 
+					    offset, target_offset));
+			return target_offset - offset;
+
+		case PAD_TO_NEXT_WORD:
+			if ( offset == -1 || wordsize_ == -1 )
+				return -1;
+
+			offset_in_word = offset % wordsize_;
+			return ( offset_in_word == 0 ) ? 
+				0 : wordsize_ - offset_in_word;
+		}
+
+	return -1;
+	}
+
+void RecordPaddingField::GenFieldEnd(Output* out_cc, Env* env, const DataPtr& field_begin)
+	{
+	ASSERT(! env->Evaluated(end_of_field_dataptr_var));
+
+	char* padding_var;
+	switch ( ptype_ )
+		{
+		case PAD_BY_LENGTH:
+			out_cc->println("const_byteptr const %s = %s + (%s);",
+				env->LValue(end_of_field_dataptr_var),
+				field_begin.ptr_expr(),
+				expr_->EvalExpr(out_cc, env));
+			break;
+
+		case PAD_TO_OFFSET:
+			out_cc->println("const_byteptr %s = %s + (%s);",
+				env->LValue(end_of_field_dataptr_var),
+				env->RValue(begin_of_data),
+				expr_->EvalExpr(out_cc, env));
+			out_cc->println("if ( %s < %s )",
+				env->LValue(end_of_field_dataptr_var),
+				field_begin.ptr_expr());
+			out_cc->inc_indent();
+			out_cc->println("{");
+			out_cc->println("// throw ExceptionInvalidOffset(\"%s\", %s - %s, %s);",
+				id_->LocName(), 
+				field_begin.ptr_expr(),
+				env->RValue(begin_of_data),
+				expr_->EvalExpr(out_cc, env));
+			out_cc->println("%s = %s;", 
+				env->LValue(end_of_field_dataptr_var),
+				field_begin.ptr_expr());
+			out_cc->println("}");
+			out_cc->dec_indent();
+			break;
+
+		case PAD_TO_NEXT_WORD:
+			padding_var = nfmt("%s__size", id()->Name());
+			out_cc->println("int %s = (%s - %s) %% %d;",
+				padding_var, 
+				field_begin.ptr_expr(),
+				env->RValue(begin_of_data),
+				wordsize_);
+			out_cc->println("%s = (%s == 0) ? 0 : %d - %s;",
+				padding_var,
+				padding_var,
+				wordsize_,
+				padding_var);
+			out_cc->println("const_byteptr const %s = %s + %s;",
+				env->LValue(end_of_field_dataptr_var), 
+				field_begin.ptr_expr(),
+				padding_var);
+			delete [] padding_var;
+			break;
+		}
+
+	env->SetEvaluated(end_of_field_dataptr_var);
+	}
+
+bool RecordPaddingField::GenBoundaryCheck(Output* out_cc, Env* env)
+	{
+	if ( boundary_checked_ )
+		return true;
+
+	const DataPtr& begin = getFieldBegin(out_cc, env);
+
+	char* size;
+	int ss = StaticSize(env, begin.AbsOffset(begin_of_data));
+	ASSERT ( ss >= 0 );
+ 	size = nfmt("%d", ss);
+	
+	begin.GenBoundaryCheck(out_cc, env, size, field_id_str_.c_str());
+
+	delete [] size;
+
+	SetBoundaryChecked();
+	return true;
+	}
+
+bool RecordPaddingField::DoTraverse(DataDepVisitor *visitor)
+	{ 
+	return Field::DoTraverse(visitor) && 
+	       (! expr_ || expr_->Traverse(visitor));
+	}
+

tools/binpac/src/pac_record.h

@@ -0,0 +1,169 @@
+#ifndef pac_record_h
+#define pac_record_h
+
+#include "pac_common.h"
+#include "pac_field.h"
+#include "pac_id.h"
+#include "pac_let.h"
+#include "pac_type.h"
+
+class RecordType : public Type
+{
+public:
+	RecordType(RecordFieldList* fields);
+	~RecordType();
+
+	bool DefineValueVar() const;
+	string DataTypeStr() const;
+
+	void Prepare(Env* env, int flags);
+
+	void GenPubDecls(Output* out, Env* env);
+	void GenPrivDecls(Output* out, Env* env);
+
+	void GenInitCode(Output* out, Env* env);
+	void GenCleanUpCode(Output* out, Env* env);
+
+	int StaticSize(Env* env) const;
+
+	void SetBoundaryChecked();
+
+	const ID *parsing_dataptr_var() const;
+
+	bool IsPointerType() const	{ ASSERT(0); return false; }
+
+protected:
+	void DoGenParseCode(Output* out, Env* env, const DataPtr& data, int flags);
+	void GenDynamicSize(Output* out, Env* env, const DataPtr& data);
+
+	Type *DoClone() const 	{ return 0; }
+
+	void DoMarkIncrementalInput();
+
+	bool DoTraverse(DataDepVisitor *visitor);
+	bool ByteOrderSensitive() const;
+
+private:
+	Field *parsing_dataptr_var_field_;
+	RecordFieldList* record_fields_;
+};
+
+// A data field of a record type. A RecordField corresponds to a
+// segment of input data, and therefore RecordField's are ordered---each
+// of them has a known previous and next field.
+
+class RecordField : public Field
+{
+public:
+	RecordField(FieldType tof, ID* id, Type *type);
+	~RecordField();
+
+	RecordType *record_type() const		{ return record_type_; }
+	void set_record_type(RecordType* ty) 	{ record_type_ = ty; }
+
+	virtual void GenParseCode(Output* out, Env* env) = 0;
+
+	RecordField* prev() const		{ return prev_; }
+	RecordField* next() const		{ return next_; }
+	void set_prev(RecordField* f) 		{ prev_ = f; }
+	void set_next(RecordField* f) 		{ next_ = f; }
+
+	int static_offset() const 		{ return static_offset_; }
+	void set_static_offset(int offset)	{ static_offset_ = offset; }
+
+	int parsing_state_seq() const		{ return parsing_state_seq_; }
+	void set_parsing_state_seq(int x) 	{ parsing_state_seq_ = x; }
+
+	virtual int StaticSize(Env* env, int offset) const = 0;
+	const char* FieldSize(Output* out, Env* env);
+	const char* FieldOffset(Output* out, Env* env);
+
+	virtual bool BoundaryChecked() const	{ return boundary_checked_; }
+	virtual void SetBoundaryChecked()	{ boundary_checked_ = true; }
+
+	virtual bool RequiresByteOrder() const = 0;
+
+	friend class RecordType;
+
+protected:
+	RecordType* record_type_;
+	RecordField* prev_;
+	RecordField* next_;
+	bool boundary_checked_;
+	int static_offset_;
+	int parsing_state_seq_;
+
+	DataPtr* begin_of_field_dataptr;
+	DataPtr* end_of_field_dataptr;
+	char* field_size_expr;
+	char* field_offset_expr;
+	ID* end_of_field_dataptr_var;
+
+	const DataPtr& getFieldBegin(Output* out_cc, Env* env);
+	const DataPtr& getFieldEnd(Output* out_cc, Env* env);
+	virtual void GenFieldEnd(Output* out, Env* env, const DataPtr& begin) = 0; 
+
+	bool AttemptBoundaryCheck(Output* out_cc, Env* env);
+	virtual bool GenBoundaryCheck(Output* out_cc, Env* env) = 0;
+};
+
+class RecordDataField : public RecordField, public Evaluatable
+{
+public:
+	RecordDataField(ID* arg_id, Type* arg_type);
+	~RecordDataField();
+
+	// Instantiates abstract class Field
+	void Prepare(Env* env);
+	void GenParseCode(Output* out, Env* env);
+
+	// Instantiates abstract class Evaluatable
+	void GenEval(Output* out, Env* env);
+
+	int StaticSize(Env* env, int) const 	{ return type()->StaticSize(env); }
+
+	void SetBoundaryChecked();
+
+	bool RequiresByteOrder() const
+		{ return type()->RequiresByteOrder(); }
+	bool RequiresAnalyzerContext() const;
+
+protected:
+	void GenFieldEnd(Output* out, Env* env, const DataPtr& begin); 
+	bool GenBoundaryCheck(Output* out_cc, Env* env);
+	bool DoTraverse(DataDepVisitor *visitor);
+};
+
+enum PaddingType { PAD_BY_LENGTH, PAD_TO_OFFSET, PAD_TO_NEXT_WORD };
+
+class RecordPaddingField : public RecordField
+{
+public:
+	RecordPaddingField(ID* id, PaddingType ptype, Expr* expr);
+	~RecordPaddingField();
+
+	void Prepare(Env* env);
+
+	void GenPubDecls(Output* out, Env* env) 	{ /* nothing */ }
+	void GenPrivDecls(Output* out, Env* env) 	{ /* nothing */ }
+
+	void GenInitCode(Output* out, Env* env) 	{ /* nothing */ }
+	void GenCleanUpCode(Output* out, Env* env)	{ /* nothing */ }
+	void GenParseCode(Output* out, Env* env);
+
+	int StaticSize(Env* env, int offset) const;
+
+	bool RequiresByteOrder() const		{ return false; }
+
+protected:
+	void GenFieldEnd(Output* out, Env* env, const DataPtr& begin); 
+	bool GenBoundaryCheck(Output* out_cc, Env* env);
+	bool DoTraverse(DataDepVisitor *visitor);
+
+private:
+	PaddingType ptype_;
+	Expr* expr_;
+	int wordsize_;
+};
+
+#endif  // pac_record_h

tools/binpac/src/pac_redef.cc

@@ -0,0 +1,173 @@
+#include "pac_analyzer.h"
+#include "pac_case.h"
+#include "pac_exception.h"
+#include "pac_expr.h"
+#include "pac_func.h"
+#include "pac_record.h"
+#include "pac_redef.h"
+#include "pac_type.h"
+#include "pac_typedecl.h"
+
+namespace {
+
+Decl *find_decl(const ID *id)
+	{
+	Decl *decl = Decl::LookUpDecl(id);
+	if ( ! decl )
+		{
+		throw Exception(id, 
+		                fmt("cannot find declaration for %s", 
+		                    id->Name()));
+		}
+
+	return decl;
+	}
+
+}
+
+Decl *ProcessTypeRedef(const ID *id, FieldList *fieldlist)
+	{
+	Decl *decl = find_decl(id);
+
+	if ( decl->decl_type() != Decl::TYPE )
+		{
+		throw Exception(id, 
+		                fmt("not a type declaration: %s", 
+		                    id->Name()));
+		}
+
+	TypeDecl *type_decl = static_cast<TypeDecl *>(decl);
+	ASSERT(type_decl);
+	Type *type = type_decl->type();
+
+	foreach(i, FieldList, fieldlist)
+		{
+		Field *f = *i;
+
+		// One cannot change data layout in 'redef'.
+		// Only 'let' or 'action' can be added
+		if ( f->tof() == LET_FIELD ||
+		     f->tof() == WITHINPUT_FIELD )
+			{
+			type->AddField(f);
+			}
+		else if ( f->tof() == RECORD_FIELD || 
+		          f->tof() == PADDING_FIELD )
+			{
+			throw Exception(f, 
+				"cannot change data layout in redef");
+			}
+		else if ( f->tof() == CASE_FIELD )
+			{
+			throw Exception(f, 
+				"use 'redef case' adding cases");
+			}
+		}
+
+	return decl;
+	}
+
+Decl *ProcessCaseTypeRedef(const ID *id, CaseFieldList *casefieldlist)
+	{
+	Decl *decl = find_decl(id);
+
+	if ( decl->decl_type() != Decl::TYPE )
+		{
+		throw Exception(id, 
+		                fmt("not a type declaration: %s", 
+		                    id->Name()));
+		}
+
+	TypeDecl *type_decl = static_cast<TypeDecl *>(decl);
+	ASSERT(type_decl);
+
+	Type *type = type_decl->type();
+	if ( type->tot() != Type::CASE )
+		{
+		throw Exception(id, 
+		                fmt("not a case type: %s", 
+		                    id->Name()));
+		}
+
+	CaseType *casetype = static_cast<CaseType*>(type);
+	ASSERT(casetype);
+
+	foreach(i, CaseFieldList, casefieldlist)
+		{
+		CaseField *f = *i;
+		casetype->AddCaseField(f);
+		}
+
+	return decl;
+	}
+
+Decl *ProcessCaseExprRedef(const ID *id, CaseExprList *caseexprlist)
+	{
+	Decl *decl = find_decl(id);
+
+	if ( decl->decl_type() != Decl::FUNC )
+		{
+		throw Exception(id, 
+		                fmt("not a function declaration: %s", 
+		                    id->Name()));
+		}
+
+	FuncDecl *func_decl = static_cast<FuncDecl *>(decl);
+	ASSERT(func_decl);
+
+	Expr *expr = func_decl->function()->expr();
+	if ( ! expr ||expr->expr_type() != Expr::EXPR_CASE )
+		{
+		throw Exception(id, 
+		                fmt("function not defined by a case expression: %s", 
+		                    id->Name()));
+		}
+
+	foreach(i, CaseExprList, caseexprlist)
+		{
+		CaseExpr *e = *i;
+		expr->AddCaseExpr(e);
+		}
+	
+	return decl;
+	}
+
+Decl *ProcessAnalyzerRedef(const ID *id, 
+		Decl::DeclType decl_type, 
+		AnalyzerElementList *elements)
+	{
+	Decl *decl = find_decl(id);
+
+	if ( decl->decl_type() != decl_type )
+		{
+		throw Exception(id, 
+		                fmt("not a connection/flow declaration: %s", 
+		                    id->Name()));
+		}
+
+	AnalyzerDecl *analyzer_decl = static_cast<AnalyzerDecl *>(decl);
+	ASSERT(analyzer_decl);
+
+	analyzer_decl->AddElements(elements);
+
+	return decl;
+	}
+
+Decl *ProcessTypeAttrRedef(const ID *id, AttrList *attrlist)
+	{
+	Decl *decl = find_decl(id);
+
+	if ( decl->decl_type() != Decl::TYPE )
+		{
+		throw Exception(id, 
+		                fmt("not a type declaration: %s", 
+		                    id->Name()));
+		}
+
+	TypeDecl *type_decl = static_cast<TypeDecl *>(decl);
+	ASSERT(type_decl);
+
+	type_decl->AddAttrs(attrlist);
+
+	return decl;
+	}

tools/binpac/src/pac_redef.h

@@ -0,0 +1,13 @@
+#ifndef pac_redef_h
+#define pac_redef_h
+
+#include "pac_decl.h"
+
+Decl *ProcessCaseTypeRedef(const ID *id, CaseFieldList *casefieldlist);
+Decl *ProcessCaseExprRedef(const ID *id, CaseExprList *caseexprlist);
+Decl *ProcessAnalyzerRedef(const ID *id, 
+			Decl::DeclType decl_type, 
+			AnalyzerElementList *elements);
+Decl *ProcessTypeAttrRedef(const ID *id, AttrList *attrlist);
+
+#endif  // pac_redef_h

tools/binpac/src/pac_regex.cc

@@ -0,0 +1,82 @@
+#include "pac_exttype.h"
+#include "pac_id.h"
+#include "pac_output.h"
+#include "pac_regex.h"
+#include "pac_type.h"
+
+// Depends on the regular expression library we are using
+const char *RegEx::kREMatcherType = "RegExMatcher";
+const char *RegEx::kMatchPrefix = "MatchPrefix";
+
+string escape_char(const string &s)
+	{
+	char buf[s.length() * 2 + 1];
+	int j = 0;
+	for ( int i = 0; i < (int) s.length(); ++i )
+		{
+		if ( s[i] == '\\' )
+			{
+			if ( i + 1 < (int) s.length() ) 
+				{
+				buf[j++] = '\\';
+				if ( s[i+1] == '/' )
+					buf[j-1] = s[++i];
+				else if ( s[i+1] == '/' || s[i+1] == '\\' || s[i+1] == '"' )
+					buf[j++] = s[++i];
+				else
+					buf[j++] = '\\';
+				}
+			}
+		else if ( s[i] == '"' )
+			{
+			buf[j++] = '\\';
+			buf[j++] = '"';
+			}
+		else
+			{
+			buf[j++] = s[i];
+			}
+		}
+
+	buf[j++] = '\0';
+
+	return string(buf);
+	}
+
+RegEx::RegEx(const string &s)
+	{
+	str_ = escape_char(s);
+	string prefix = strfmt("%s_re_", current_decl_id->Name());
+	matcher_id_ = ID::NewAnonymousID(prefix);
+	decl_ = new RegExDecl(this);
+	}
+
+RegEx::~RegEx()
+	{
+	}
+
+RegExDecl::RegExDecl(RegEx *regex)
+	: Decl(regex->matcher_id(), REGEX)
+	{
+	regex_ = regex;
+	}
+
+void RegExDecl::Prepare()
+	{
+	global_env()->AddID(id(), GLOBAL_VAR, extern_type_re_matcher);
+	}
+
+void RegExDecl::GenForwardDeclaration(Output *out_h)
+	{
+	out_h->println("extern %s %s;\n", 
+		RegEx::kREMatcherType,
+		global_env()->LValue(regex_->matcher_id()));
+	}
+
+void RegExDecl::GenCode(Output *out_h, Output *out_cc)
+	{
+	out_cc->println("%s %s(\"%s\");\n", 
+		RegEx::kREMatcherType,
+		global_env()->LValue(regex_->matcher_id()),
+		regex_->str().c_str());
+	}

tools/binpac/src/pac_regex.h

@@ -0,0 +1,41 @@
+#ifndef pac_regex_h
+#define pac_regex_h
+
+#include "pac_common.h"
+#include "pac_decl.h"
+
+class RegExDecl;
+
+class RegEx : public Object
+{
+public:
+	RegEx(const string &str);
+	~RegEx();
+
+	const string &str() const	{ return str_; }
+	ID *matcher_id() const		{ return matcher_id_; }
+
+private:
+	string str_;
+	ID *matcher_id_;
+	RegExDecl *decl_;
+
+public:
+	static const char *kREMatcherType;
+	static const char *kMatchPrefix;
+};
+
+class RegExDecl : public Decl
+{
+public:
+	RegExDecl(RegEx *regex);
+
+	void Prepare();
+	void GenForwardDeclaration(Output *out_h); 
+	void GenCode(Output *out_h, Output *out_cc);
+
+private:
+	RegEx *regex_;
+};
+
+#endif  // pac_regex_h

tools/binpac/src/pac_scan.ll

@@ -0,0 +1,356 @@
+%{
+// $Id: pac_scan.ll 3361 2006-07-06 18:06:49Z rpang $
+
+#include "pac_action.h"
+#include "pac_array.h"
+#include "pac_attr.h"
+#include "pac_case.h"
+#include "pac_common.h"
+#include "pac_conn.h"
+#include "pac_dataptr.h"
+#include "pac_dataunit.h"
+#include "pac_dbg.h"
+#include "pac_decl.h"
+#include "pac_exception.h"
+#include "pac_expr.h"
+#include "pac_flow.h"
+#include "pac_id.h"
+#include "pac_number.h"
+#include "pac_output.h"
+#include "pac_param.h"
+#include "pac_parse.h"
+#include "pac_record.h"
+#include "pac_type.h"
+#include "pac_utils.h"
+
+int line_number = 1;
+
+int begin_pac_primitive(int tok);
+int end_pac_primitive();
+
+int string_token(int tok)
+	{
+	yylval.str = copy_string(yytext);
+	return tok;
+	}
+
+int char_token(int tok)
+	{
+	yylval.val = yytext[0];
+	return tok;
+	}
+
+void include_file(const char *filename);
+
+%}
+
+/* EC -- embedded code state */
+/* PP -- PAC primitive state */
+/* INCL -- @include line */
+
+%s EC INCL PP RE
+
+WS	[ \t]+
+ID	[A-Za-z_][A-Za-z_0-9]*
+D	[0-9]+
+HEX	[0-9a-fA-F]+
+FILE	[A-Za-z._0-9\-]+
+ESCSEQ	(\\([^\n]|[0-7]{3}|x[[:xdigit:]]{2}))
+
+%option nounput
+
+%%
+
+<INITIAL>"%include"		{
+				BEGIN(INCL);
+				}
+
+<INCL>{WS}			/* skip whitespace */
+
+<INCL>{FILE}			{
+				BEGIN(INITIAL);
+				include_file(yytext);
+				}
+
+<INITIAL>"%extern{"		{
+				BEGIN(EC);
+				return TOK_LPB_EXTERN;
+				}
+<INITIAL>"%header{"		{
+				BEGIN(EC);
+				return TOK_LPB_HEADER;
+				}
+<INITIAL>"%code{"		{
+				BEGIN(EC);
+				return TOK_LPB_CODE;
+				}
+<INITIAL>"%init{"		{
+				BEGIN(EC);
+				return TOK_LPB_INIT;
+				}
+<INITIAL>"%cleanup{"		{
+				BEGIN(EC);
+				return TOK_LPB_CLEANUP;
+				}
+<INITIAL>"%member{"		{
+				BEGIN(EC);
+				return TOK_LPB_MEMBER;
+				}
+<INITIAL>"%eof{"		{
+				BEGIN(EC);
+				return TOK_LPB_EOF;
+				}
+<INITIAL>"%{"			{
+				BEGIN(EC);
+				return TOK_LPB;
+				}
+<EC>"%}"			{
+				BEGIN(INITIAL);
+				return TOK_RPB;
+				}
+
+<EC>"${"			return begin_pac_primitive(TOK_PAC_VAL);
+<EC>"$set{"			return begin_pac_primitive(TOK_PAC_SET); 
+<EC>"$type{"			return begin_pac_primitive(TOK_PAC_TYPE);
+<EC>"$typeof{"			return begin_pac_primitive(TOK_PAC_TYPEOF);
+<EC>"$const_def{"		return begin_pac_primitive(TOK_PAC_CONST_DEF);
+
+<EC>"//".*			return string_token(TOK_EMBEDDED_STRING);
+<EC>.				return char_token(TOK_EMBEDDED_ATOM);
+<EC>\n				{ ++line_number; return char_token(TOK_EMBEDDED_ATOM); }
+
+<PP>"}"				return end_pac_primitive();
+
+<INITIAL,PP>\n			++line_number;
+<INITIAL>#.*			/* eat comments */
+<INITIAL,PP>{WS}		/* eat whitespace */
+
+<INITIAL>"RE/"			{
+				BEGIN(RE);
+				return TOK_BEGIN_RE;
+				}
+
+<RE>([^/\\\n]|{ESCSEQ})+	return string_token(TOK_REGEX);
+
+<RE>"/"				{
+				BEGIN(INITIAL);
+				return TOK_END_RE;
+				}
+
+<RE>[\\\n]			return yytext[0];
+
+<INITIAL>analyzer		return TOK_ANALYZER;
+<INITIAL>enum			return TOK_ENUM;
+<INITIAL>extern			return TOK_EXTERN;
+<INITIAL>flow			return TOK_FLOW;
+<INITIAL>function		return TOK_FUNCTION;
+<INITIAL>let			return TOK_LET;
+<INITIAL>refine			return TOK_REFINE;
+<INITIAL>type			return TOK_TYPE;
+
+<INITIAL>align			return TOK_ALIGN;
+<INITIAL>case			return TOK_CASE;
+<INITIAL>casefunc		return TOK_CASEFUNC;
+<INITIAL>casetype		return TOK_CASETYPE;
+<INITIAL>connection		return TOK_CONNECTION;
+<INITIAL>datagram		{
+				yylval.val = AnalyzerDataUnit::DATAGRAM;
+				return TOK_DATAUNIT;
+				}
+<INITIAL>default 		return TOK_DEFAULT;
+<INITIAL>downflow		{
+				yylval.val = AnalyzerFlow::DOWN;
+				return TOK_FLOWDIR;
+				}
+<INITIAL>flowunit		{
+				yylval.val = AnalyzerDataUnit::FLOWUNIT;
+				return TOK_DATAUNIT;
+				}
+<INITIAL>of			return TOK_OF;
+<INITIAL>offsetof 		return TOK_OFFSETOF;
+<INITIAL>padding		return TOK_PADDING;
+<INITIAL>record			return TOK_RECORD;
+<INITIAL>sizeof			return TOK_SIZEOF;
+<INITIAL>to			return TOK_TO;
+<INITIAL>typeattr		return TOK_TYPEATTR;
+<INITIAL>upflow			{
+				yylval.val = AnalyzerFlow::UP;
+				return TOK_FLOWDIR;
+				}
+<INITIAL>withcontext		return TOK_WITHCONTEXT;
+<INITIAL>withinput		return TOK_WITHINPUT;
+
+<INITIAL>&also			return TOK_ATTR_ALSO;
+<INITIAL>&byteorder		return TOK_ATTR_BYTEORDER;
+<INITIAL>&check			return TOK_ATTR_CHECK;
+<INITIAL>&chunked		return TOK_ATTR_CHUNKED;
+<INITIAL>&exportsourcedata	return TOK_ATTR_EXPORTSOURCEDATA;
+<INITIAL>&if			return TOK_ATTR_IF;
+<INITIAL>&length		return TOK_ATTR_LENGTH;
+<INITIAL>&let			return TOK_ATTR_LET;
+<INITIAL>&linebreaker		return TOK_ATTR_LINEBREAKER;
+<INITIAL>&oneline		return TOK_ATTR_ONELINE;
+<INITIAL>&refcount		return TOK_ATTR_REFCOUNT;
+<INITIAL>&requires		return TOK_ATTR_REQUIRES;
+<INITIAL>&restofdata		return TOK_ATTR_RESTOFDATA;
+<INITIAL>&restofflow		return TOK_ATTR_RESTOFFLOW;
+<INITIAL>&transient		return TOK_ATTR_TRANSIENT;
+<INITIAL>&until			return TOK_ATTR_UNTIL;
+
+<INITIAL,PP>"0x"{HEX}		{
+				int n;
+				sscanf(yytext + 2, "%x", &n);
+				yylval.num = new Number(yytext, n);
+				return TOK_NUMBER;
+				}
+
+<INITIAL,PP>{D}			{
+				int n;
+				sscanf(yytext, "%d", &n);
+				yylval.num = new Number(yytext, n);
+				return TOK_NUMBER;
+				}
+
+<INITIAL,PP>{ID}		{
+				yylval.id = new ID(yytext);
+				return TOK_ID;
+				}
+
+<INITIAL>"$"{ID}		{
+				yylval.id = new ID(yytext);
+				return TOK_ID;
+				}
+
+<INITIAL>\"([^\\\n\"]|{ESCSEQ})*\" return string_token(TOK_STRING);
+
+<INITIAL,PP>"=="		return TOK_EQUAL;
+<INITIAL,PP>"!="		return TOK_NEQ;
+<INITIAL,PP>">="		return TOK_GE;
+<INITIAL,PP>"<="		return TOK_LE;
+<INITIAL,PP>"<<"		return TOK_LSHIFT;
+<INITIAL,PP>">>"		return TOK_RSHIFT;
+<INITIAL,PP>"&&"		return TOK_AND;
+<INITIAL,PP>"||"		return TOK_OR;
+<INITIAL,PP>"+="		return TOK_PLUSEQ;
+<INITIAL>"->"			return TOK_RIGHTARROW;
+
+<INITIAL,PP>[\.!%*/+\-&|\^,:;<=>?()\[\]{}~]	return yytext[0];
+
+%%
+
+void begin_RE()
+	{
+	BEGIN(RE);
+	}
+
+void end_RE()
+	{
+	BEGIN(INITIAL);
+	}
+
+// The DECL state is deprecated
+void begin_decl()
+	{
+	// BEGIN(DECL);
+	}
+
+void end_decl()
+	{
+	// BEGIN(INITIAL);
+	}
+
+int begin_pac_primitive(int tok)
+	{
+	BEGIN(PP);
+	return tok;
+	}
+
+int end_pac_primitive()
+	{
+	BEGIN(EC);
+	return TOK_END_PAC;
+	}
+
+const int MAX_INCLUDE_DEPTH = 100;
+
+struct IncludeState {
+	YY_BUFFER_STATE yystate;
+	string input_filename;
+	int line_number;
+};
+
+IncludeState include_stack[MAX_INCLUDE_DEPTH];
+int include_stack_ptr = 0;
+
+void switch_to_file(FILE *fp)
+	{
+	yy_switch_to_buffer(yy_create_buffer(fp, YY_BUF_SIZE));
+	}
+
+void switch_to_file(const char *filename)
+	{
+	if ( include_stack_ptr >= MAX_INCLUDE_DEPTH )
+		{
+		fprintf( stderr, "Includes nested too deeply" );
+		exit( 1 );
+		}
+
+	IncludeState state = 
+		{ YY_CURRENT_BUFFER, input_filename, line_number };
+	include_stack[include_stack_ptr++] = state;
+
+	FILE *fp = fopen(filename, "r");
+
+	if ( ! fp )
+		{
+		fprintf(stderr, "%s:%d: error: cannot include file \"%s\"\n", 
+			input_filename.c_str(), line_number,filename);
+		--include_stack_ptr;
+		return;
+		}
+
+	yyin = fp;
+	input_filename = string(filename);
+	line_number = 1;
+	switch_to_file(yyin);
+	fprintf(stderr, "switching to file %s\n", input_filename.c_str());
+	}
+
+void include_file(const char *filename)
+	{
+	ASSERT(filename);
+
+	string full_filename;
+	if ( filename[0] == '/' || filename[0] == '.' )
+		full_filename = filename;
+	else
+		{
+		int i;
+		for ( i = 0; i < (int) FLAGS_include_directories.size(); ++i )
+			{
+			full_filename = FLAGS_include_directories[i] + filename;
+			DEBUG_MSG("Try include file: \"%s\"\n", 
+				full_filename.c_str());
+			if ( access(full_filename.c_str(), R_OK) == 0 )
+				break;
+			}
+		if ( i >= (int) FLAGS_include_directories.size() )
+			full_filename = filename;
+		}
+
+	switch_to_file(full_filename.c_str());
+	}
+
+int yywrap()
+	{
+	yy_delete_buffer(YY_CURRENT_BUFFER);
+	--include_stack_ptr;
+	if ( include_stack_ptr < 0 )
+		return 1;
+
+	IncludeState state = include_stack[include_stack_ptr];
+	yy_switch_to_buffer(state.yystate);
+	input_filename = state.input_filename;
+	line_number = state.line_number;
+	return 0;
+	}

tools/binpac/src/pac_state.cc

@@ -0,0 +1,36 @@
+#include "pac_id.h"
+#include "pac_output.h"
+#include "pac_type.h"
+
+#include "pac_state.h"
+
+void StateVar::GenDecl(Output *out_h, Env *env)
+	{
+	out_h->println("%s %s;", 
+		type_->DataTypeStr().c_str(), 
+		env->LValue(id_));
+	}
+
+void StateVar::GenAccessFunction(Output *out_h, Env *env)
+	{
+	out_h->println("%s %s const	{ return %s; }", 
+		type_->DataTypeConstRefStr().c_str(), 
+		env->RValue(id_), 
+		env->LValue(id_));
+	}
+
+void StateVar::GenSetFunction(Output *out_h, Env *env)
+	{
+	out_h->println("void %s(%s x) 	{ %s = x; }", 
+		set_function(id_).c_str(),
+		type_->DataTypeConstRefStr().c_str(), 
+		env->LValue(id_));
+	}
+
+void StateVar::GenInitCode(Output *out_cc, Env *env)
+	{
+	}
+
+void StateVar::GenCleanUpCode(Output *out_cc, Env *env)
+	{
+	}

tools/binpac/src/pac_state.h

@@ -0,0 +1,28 @@
+#ifndef pac_state_h
+#define pac_state_h
+
+// Classes representing analyzer states.
+
+#include "pac_common.h"
+
+class StateVar
+{
+public:
+	StateVar(ID *id, Type *type)
+		: id_(id), type_(type) {}
+
+	const ID *id() const	{ return id_; }
+	Type *type() const	{ return type_; }
+
+	void GenDecl(Output *out_h, Env *env);
+	void GenAccessFunction(Output *out_h, Env *env);
+	void GenSetFunction(Output *out_h, Env *env);
+	void GenInitCode(Output *out_cc, Env *env);
+	void GenCleanUpCode(Output *out_cc, Env *env);
+
+private:
+	ID *id_;
+	Type *type_;
+};
+
+#endif  // pac_state_h