Move binpac code into the main Zeek repository

This is based on commit 48f75b5f6415fe9d597e3e991cec635b1bc400dc from
the binpac repository.

.gitmodules

@@ -1,9 +1,6 @@
 [submodule "auxil/zeek-aux"]
 	path = auxil/zeek-aux
 	url = https://github.com/zeek/zeek-aux
-[submodule "auxil/binpac"]
-	path = auxil/binpac
-	url = https://github.com/zeek/binpac
 [submodule "auxil/zeekctl"]
 	path = auxil/zeekctl
 	url = https://github.com/zeek/zeekctl

CMakeLists.txt

@@ -396,14 +396,14 @@ endfunction ()
 
 add_zeek_dynamic_plugin_build_interface_include_directories(
     ${PROJECT_SOURCE_DIR}/src/include
-    ${PROJECT_SOURCE_DIR}/auxil/binpac/lib
+    ${PROJECT_SOURCE_DIR}/tools/binpac/lib
     ${PROJECT_SOURCE_DIR}/auxil/broker/libbroker
     ${PROJECT_SOURCE_DIR}/auxil/paraglob/include
     ${PROJECT_SOURCE_DIR}/auxil/prometheus-cpp/core/include
     ${PROJECT_SOURCE_DIR}/auxil/expected-lite/include
     ${CMAKE_BINARY_DIR}/src
     ${CMAKE_BINARY_DIR}/src/include
-    ${CMAKE_BINARY_DIR}/auxil/binpac/lib
+    ${CMAKE_BINARY_DIR}/tools/binpac/lib
     ${CMAKE_BINARY_DIR}/auxil/broker/libbroker
     ${CMAKE_BINARY_DIR}/auxil/prometheus-cpp/core/include)
 
@@ -892,12 +892,12 @@ if (BUILD_STATIC_BINPAC)
     set(ENABLE_STATIC_ONLY true)
 endif ()
 
-add_subdirectory(auxil/binpac)
+add_subdirectory(tools/binpac)
 set(ENABLE_STATIC_ONLY ${ENABLE_STATIC_ONLY_SAVED})
 
 # FIXME: avoid hard-coding a path for multi-config generator support. See the
 # TODO in ZeekPluginConfig.cmake.in.
-set(BINPAC_EXE_PATH "${CMAKE_BINARY_DIR}/auxil/binpac/src/binpac${CMAKE_EXECUTABLE_SUFFIX}")
+set(BINPAC_EXE_PATH "${CMAKE_BINARY_DIR}/tools/binpac/src/binpac${CMAKE_EXECUTABLE_SUFFIX}")
 set(_binpac_exe_path "included")
 
 # Need to call find_package so it sets up the include paths used by plugin builds.

auxil/binpac

@@ -1 +0,0 @@
-Subproject commit 48f75b5f6415fe9d597e3e991cec635b1bc400dc

tools/binpac/CMakeLists.txt

@@ -0,0 +1,5 @@
+# ##############################################################################
+# Recurse on sub-directories
+
+add_subdirectory(lib)
+add_subdirectory(src)

tools/binpac/TODO

@@ -0,0 +1,34 @@
+Big features
+* Variable context (xid, call in RPC)? -- no variable context
+* Helpers
+* Connection states and actions
+* Case and analyzer redef
+* &also withinput
+* Explicit analyzer context (interface + instantiation) "withcontext"
++ Interface with C++ and Zeek (events, extern, weird)
++ Incremental input
++ ASCII protocols
++ Reassembly
+- Dealing with exceptions
+- Dependency analysis to save parsing time on unused fields
+- Performance measurement
+
+Small features
+* Restructure the code: break up pac.{h,cc}
+* ref counting (to keep certain structures)
+* analyzer context as a parameter of class
+* &autolength
+* find a better name for "analyzer_context" ("analcxt", "context", "analyzer") $context
+* &if
+* &autolength (now &restofdata)
+* Use vector<> instead of array<>?
+* set end_of_data when &length = ...
+- make the `default' case mandatory?
+- &inline
+- &warn and &check? (follow &if)
+- typedef?
+
+Binpac 1
+- create a namespace for each .pac file
+- type equivalence
+- byteorder() for every type?

tools/binpac/lib/CMakeLists.txt

@@ -0,0 +1,44 @@
+include(TestBigEndian)
+test_big_endian(HOST_BIGENDIAN)
+
+include(CheckTypeSize)
+check_type_size("unsigned int" SIZEOF_UNSIGNED_INT)
+
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/binpac.h.in ${CMAKE_CURRENT_BINARY_DIR}/binpac.h)
+
+include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
+set(binpac_headers ${CMAKE_CURRENT_BINARY_DIR}/binpac.h binpac_analyzer.h binpac_buffer.h
+                   binpac_bytestring.h binpac_exception.h binpac_regex.h)
+
+set(binpac_lib_SRCS binpac_buffer.cc binpac_bytestring.cc binpac_regex.cc)
+
+if (BUILD_STATIC_BINPAC)
+    add_library(binpac_static STATIC)
+    target_sources(binpac_static PRIVATE ${binpac_lib_SRCS})
+    set_target_properties(binpac_static PROPERTIES OUTPUT_NAME binpac)
+    install(TARGETS binpac_static DESTINATION ${CMAKE_INSTALL_LIBDIR})
+    if (MSVC)
+        target_compile_options(binpac_static PRIVATE "/J")
+    endif ()
+    set(BinPAC_LIBRARY binpac_static CACHE STRING "BinPAC library" FORCE)
+else ()
+    add_library(binpac_lib SHARED)
+    target_sources(binpac_lib PRIVATE ${binpac_lib_SRCS})
+    target_sources(binpac_lib INTERFACE ${binpac_headers})
+    set_target_properties(binpac_lib PROPERTIES MACOSX_RPATH true OUTPUT_NAME binpac)
+    if (MSVC)
+        target_compile_options(binpac_lib PRIVATE "/J")
+    endif ()
+    install(TARGETS binpac_lib DESTINATION ${CMAKE_INSTALL_LIBDIR})
+    set(BinPAC_LIBRARY binpac_lib CACHE STRING "BinPAC library" FORCE)
+endif ()
+
+if (ZEEK_ROOT_DIR)
+    # Installed in binpac subdir just for organization purposes.
+    install(FILES ${binpac_headers} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/binpac)
+else ()
+    install(FILES ${binpac_headers} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+endif ()
+
+set(BinPAC_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}
+    CACHE STRING "BinPAC header directories" FORCE)

tools/binpac/lib/README

@@ -0,0 +1,3 @@
+This directory contains a library needed by generated C++ code from
+binpac. Note that the library is not needed by the binpac compiler
+itself.

tools/binpac/lib/binpac.h.in

@@ -0,0 +1,160 @@
+// Do not edit binpac.h, edit binpac.h.in instead!
+
+#ifndef binpac_h
+#define binpac_h
+
+#ifndef _MSC_VER
+#include <sys/param.h>
+#endif
+
+#cmakedefine HOST_BIGENDIAN
+#ifdef HOST_BIGENDIAN
+#define HOST_BYTEORDER bigendian
+#else
+#define HOST_BYTEORDER littleendian
+#endif
+
+#include <cassert>
+#include <cstdarg>
+#include <cstdint>
+#include <cstdio>
+#include <string>
+
+// Expose C99 functionality from inttypes.h, which would otherwise not be
+// available in C++.
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+static constexpr void BINPAC_ASSERT(bool val) { assert(val); }
+
+using namespace std;
+
+namespace binpac {
+
+const int bigendian = 0;
+const int littleendian = 1;
+const int unspecified_byteorder = -1;
+
+#ifndef pac_type_defs
+#define pac_type_defs
+
+using int8 = int8_t;
+using int16 = int16_t;
+using int32 = int32_t;
+using int64 = int64_t;
+using uint8 = uint8_t;
+using uint16 = uint16_t;
+using uint32 = uint32_t;
+using uint64 = uint64_t;
+using nulptr = void*;
+using voidptr = void*;
+using byteptr = uint8*;
+using const_byteptr = const uint8*;
+using const_charptr = const char*;
+
+static_assert(sizeof(unsigned int) == 4, "Unexpected size of unsigned int");
+
+#endif /* pac_type_defs */
+
+/* Handling byte order */
+
+namespace {
+
+inline uint16 pac_swap(const uint16 x) { return (x >> 8) | ((x & 0xff) << 8); }
+
+inline int16 pac_swap(const int16 x) {
+    // Forward to unsigned version with argument/result casted
+    // appropriately.
+    uint16 (*p)(const uint16) = &pac_swap;
+    return (*p)(x);
+}
+
+inline uint32 pac_swap(const uint32 x) {
+    return (x >> 24) | ((x & 0xff0000) >> 8) | ((x & 0xff00) << 8) | ((x & 0xff) << 24);
+}
+
+inline int32 pac_swap(const int32 x) {
+    // Forward to unsigned version with argument/result casted
+    // appropriately.
+    uint32 (*p)(const uint32) = &pac_swap;
+    return (*p)(x);
+}
+
+inline uint64 pac_swap(const uint64 x) {
+    return x >> 56 | (x & 0xff000000000000) >> 40 | (x & 0xff0000000000) >> 24 | (x & 0xff00000000) >> 8 |
+           (x & 0xff000000) << 8 | (x & 0xff0000) << 24 | (x & 0xff00) << 40 | (x & 0xff) << 56;
+}
+
+inline int64 pac_swap(const int64 x) {
+    // Forward to unsigned version with argument/result casted
+    // appropriately.
+    uint64 (*p)(const uint64) = &pac_swap;
+    return (*p)(x);
+}
+
+template<typename T>
+static constexpr T FixByteOrder(int byteorder, T x) {
+    if ( byteorder == HOST_BYTEORDER )
+        return x;
+
+    return static_cast<T>(pac_swap(x));
+}
+
+template<class T>
+inline T UnMarshall(const unsigned char* data, int byteorder) {
+    T result = 0;
+    for ( int i = 0; i < (int)sizeof(T); ++i )
+        result = (result << 8) | data[byteorder == bigendian ? i : sizeof(T) - 1 - i];
+    return result;
+}
+
+inline const char* do_fmt(const char* format, va_list ap) {
+    static char buf[1024];
+    vsnprintf(buf, sizeof(buf), format, ap);
+    return buf;
+}
+
+inline string strfmt(const char* format, ...) {
+    va_list ap;
+    va_start(ap, format);
+    const char* r = do_fmt(format, ap);
+    va_end(ap);
+    return {r};
+}
+
+} // anonymous namespace
+
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define binpac_fmt(...) strfmt(__VA_ARGS__).c_str()
+
+class RefCount {
+public:
+    RefCount() { count = 1; }
+    virtual ~RefCount() {}
+    void Ref() { ++count; }
+    int Unref() {
+        BINPAC_ASSERT(count > 0);
+        return --count;
+    }
+
+private:
+    int count;
+};
+
+namespace {
+inline void Unref(RefCount* x) {
+    if ( x && x->Unref() <= 0 )
+        delete x;
+}
+} // anonymous namespace
+
+} // namespace binpac
+
+#include "binpac_analyzer.h"
+#include "binpac_buffer.h"
+#include "binpac_bytestring.h"
+#include "binpac_exception.h"
+#include "binpac_regex.h"
+
+#endif /* binpac_h */

tools/binpac/lib/binpac_analyzer.h

@@ -0,0 +1,24 @@
+#ifndef binpac_an_h
+#define binpac_an_h
+
+namespace binpac {
+
+// TODO: Add the Done() function
+
+// The interface for a connection analyzer
+class ConnectionAnalyzer {
+public:
+    virtual ~ConnectionAnalyzer() = default;
+    virtual void NewData(bool is_orig, const unsigned char* begin_of_data, const unsigned char* end_of_data) = 0;
+};
+
+// The interface for a flow analyzer
+class FlowAnalyzer {
+public:
+    virtual ~FlowAnalyzer() = default;
+    virtual void NewData(const unsigned char* begin_of_data, const unsigned char* end_of_data) = 0;
+};
+
+} // namespace binpac
+
+#endif // binpac_an_h

tools/binpac/lib/binpac_buffer.cc

@@ -0,0 +1,457 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h> // for memcpy
+
+#define binpac_regex_h
+
+#include "binpac.h"
+#include "binpac_buffer.h"
+
+namespace binpac {
+
+extern double network_time();
+
+namespace {
+const unsigned char CR = '\r';
+const unsigned char LF = '\n';
+} // namespace
+
+binpac::FlowBuffer::Policy binpac::FlowBuffer::policy = {
+    // max_capacity
+    10 * 1024 * 1024,
+    // min_capacity
+    512,
+    // contract_threshold
+    2 * 1024 * 1024,
+};
+
+FlowBuffer::FlowBuffer(LineBreakStyle linebreak_style) {
+    buffer_length_ = 0;
+    buffer_ = nullptr;
+
+    orig_data_begin_ = nullptr;
+    orig_data_end_ = nullptr;
+
+    linebreak_style_ = linebreak_style;
+    linebreak_style_default = linebreak_style;
+    linebreaker_ = 0;
+    ResetLineState();
+
+    mode_ = UNKNOWN_MODE;
+    frame_length_ = 0;
+    chunked_ = false;
+
+    data_seq_at_orig_data_end_ = 0;
+    eof_ = false;
+    have_pending_request_ = false;
+
+    buffer_n_ = 0;
+
+    NewMessage();
+}
+
+FlowBuffer::~FlowBuffer() {
+    if ( buffer_ )
+        free(buffer_);
+}
+
+void FlowBuffer::NewMessage() {
+    BINPAC_ASSERT(frame_length_ >= 0);
+
+    int bytes_to_advance = 0;
+    if ( buffer_n_ == 0 ) {
+        switch ( mode_ ) {
+            case LINE_MODE: bytes_to_advance = (frame_length_ + (linebreak_style_ == STRICT_CRLF ? 2 : 1)); break;
+            case FRAME_MODE: bytes_to_advance = frame_length_; break;
+            case UNKNOWN_MODE: break;
+        }
+    }
+
+    orig_data_begin_ += bytes_to_advance;
+    BINPAC_ASSERT(orig_data_begin_ <= orig_data_end_);
+
+    buffer_n_ = 0;
+    message_complete_ = false;
+    ContractBuffer();
+}
+
+void FlowBuffer::ResetLineState() {
+    switch ( linebreak_style_ ) {
+        case CR_OR_LF: state_ = CR_OR_LF_0; break;
+        case STRICT_CRLF: state_ = STRICT_CRLF_0; break;
+        case LINE_BREAKER: break; // Nothing to reset
+        default: BINPAC_ASSERT(0); break;
+    }
+}
+
+void FlowBuffer::ExpandBuffer(int length) {
+    if ( buffer_length_ >= length )
+        return;
+
+    if ( length < policy.min_capacity )
+        length = policy.min_capacity;
+
+    if ( length < buffer_length_ * 2 )
+        length = buffer_length_ * 2;
+
+    if ( length > policy.max_capacity ) {
+        std::string reason = strfmt("expand past max capacity %d/%d", length, policy.max_capacity);
+        throw ExceptionFlowBufferAlloc(reason.c_str());
+    }
+
+    // Allocate a new buffer and copy the existing contents
+    buffer_length_ = length;
+    unsigned char* new_buf = (unsigned char*)realloc(buffer_, buffer_length_);
+
+    if ( ! new_buf )
+        throw ExceptionFlowBufferAlloc("expand realloc OOM");
+
+    buffer_ = new_buf;
+}
+
+void FlowBuffer::ContractBuffer() {
+    if ( buffer_length_ < policy.contract_threshold )
+        return;
+
+    buffer_length_ = policy.min_capacity;
+    unsigned char* new_buf = (unsigned char*)realloc(buffer_, buffer_length_);
+
+    if ( ! new_buf )
+        throw ExceptionFlowBufferAlloc("contract realloc OOM");
+
+    buffer_ = new_buf;
+}
+
+void FlowBuffer::SetLineBreaker(unsigned char* lbreaker) {
+    linebreaker_ = *lbreaker;
+    linebreak_style_default = linebreak_style_;
+    linebreak_style_ = LINE_BREAKER;
+}
+
+void FlowBuffer::UnsetLineBreaker() { linebreak_style_ = linebreak_style_default; }
+
+void FlowBuffer::NewLine() {
+    FlowBuffer::NewMessage();
+    mode_ = LINE_MODE;
+    frame_length_ = 0;
+    chunked_ = false;
+    have_pending_request_ = true;
+    if ( state_ == FRAME_0 )
+        ResetLineState();
+    MarkOrCopyLine();
+}
+
+void FlowBuffer::NewFrame(int frame_length, bool chunked) {
+    FlowBuffer::NewMessage();
+    mode_ = FRAME_MODE;
+    frame_length_ = frame_length;
+    chunked_ = chunked;
+    have_pending_request_ = true;
+    MarkOrCopyFrame();
+}
+
+void FlowBuffer::BufferData(const_byteptr data, const_byteptr end) {
+    mode_ = FRAME_MODE;
+    frame_length_ += (end - data);
+    MarkOrCopyFrame();
+    NewData(data, end);
+}
+
+void FlowBuffer::FinishBuffer() { message_complete_ = true; }
+
+void FlowBuffer::GrowFrame(int length) {
+    BINPAC_ASSERT(frame_length_ >= 0);
+    if ( length <= frame_length_ )
+        return;
+    BINPAC_ASSERT(! chunked_ || frame_length_ == 0);
+    mode_ = FRAME_MODE;
+    frame_length_ = length;
+    MarkOrCopyFrame();
+}
+
+void FlowBuffer::DiscardData() {
+    mode_ = UNKNOWN_MODE;
+    message_complete_ = false;
+    have_pending_request_ = false;
+    orig_data_begin_ = orig_data_end_ = nullptr;
+
+    buffer_n_ = 0;
+    frame_length_ = 0;
+    ContractBuffer();
+}
+
+void FlowBuffer::set_eof() {
+    // fprintf(stderr, "EOF\n");
+    eof_ = true;
+    if ( chunked_ )
+        frame_length_ = orig_data_end_ - orig_data_begin_;
+    if ( frame_length_ < 0 )
+        frame_length_ = 0;
+}
+
+void FlowBuffer::NewData(const_byteptr begin, const_byteptr end) {
+    BINPAC_ASSERT(begin <= end);
+
+    ClearPreviousData();
+
+    BINPAC_ASSERT((buffer_n_ == 0 && message_complete_) || orig_data_begin_ == orig_data_end_);
+
+    orig_data_begin_ = begin;
+    orig_data_end_ = end;
+    data_seq_at_orig_data_end_ += (end - begin);
+
+    MarkOrCopy();
+}
+
+void FlowBuffer::MarkOrCopy() {
+    if ( ! message_complete_ ) {
+        switch ( mode_ ) {
+            case LINE_MODE: MarkOrCopyLine(); break;
+
+            case FRAME_MODE: MarkOrCopyFrame(); break;
+
+            default: break;
+        }
+    }
+}
+
+void FlowBuffer::ClearPreviousData() {
+    // All previous data must have been processed or buffered already
+    if ( orig_data_begin_ < orig_data_end_ ) {
+        BINPAC_ASSERT(buffer_n_ == 0);
+        if ( chunked_ ) {
+            if ( frame_length_ > 0 ) {
+                frame_length_ -= (orig_data_end_ - orig_data_begin_);
+            }
+            orig_data_begin_ = orig_data_end_;
+        }
+    }
+}
+
+void FlowBuffer::NewGap(int length) {
+    ClearPreviousData();
+
+    if ( chunked_ && frame_length_ >= 0 ) {
+        frame_length_ -= length;
+        if ( frame_length_ < 0 )
+            frame_length_ = 0;
+    }
+
+    orig_data_begin_ = orig_data_end_ = nullptr;
+    MarkOrCopy();
+}
+
+void FlowBuffer::MarkOrCopyLine() {
+    switch ( linebreak_style_ ) {
+        case CR_OR_LF: MarkOrCopyLine_CR_OR_LF(); break;
+        case STRICT_CRLF: MarkOrCopyLine_STRICT_CRLF(); break;
+        case LINE_BREAKER: MarkOrCopyLine_LINEBREAK(); break;
+        default: BINPAC_ASSERT(0); break;
+    }
+}
+
+/*
+Finite state automaton for CR_OR_LF:
+(!--line is complete, *--add to buffer)
+
+CR_OR_LF_0:
+    CR:	CR_OR_LF_1 !
+    LF:	CR_OR_LF_0 !
+    .:	CR_OR_LF_0 *
+
+CR_OR_LF_1:
+    CR:	CR_OR_LF_1 !
+    LF:	CR_OR_LF_0
+    .:	CR_OR_LF_0 *
+*/
+
+void FlowBuffer::MarkOrCopyLine_CR_OR_LF() {
+    if ( ! (orig_data_begin_ && orig_data_end_) )
+        return;
+
+    if ( state_ == CR_OR_LF_1 && orig_data_begin_ < orig_data_end_ && *orig_data_begin_ == LF ) {
+        state_ = CR_OR_LF_0;
+        ++orig_data_begin_;
+    }
+
+    const_byteptr data;
+    for ( data = orig_data_begin_; data < orig_data_end_; ++data ) {
+        switch ( *data ) {
+            case CR: state_ = CR_OR_LF_1; goto found_end_of_line;
+
+            case LF:
+                // state_ = CR_OR_LF_0;
+                goto found_end_of_line;
+
+            default:
+                // state_ = CR_OR_LF_0;
+                break;
+        }
+    }
+
+    AppendToBuffer(orig_data_begin_, orig_data_end_ - orig_data_begin_);
+    return;
+
+found_end_of_line:
+    if ( buffer_n_ == 0 ) {
+        frame_length_ = data - orig_data_begin_;
+    }
+    else {
+        AppendToBuffer(orig_data_begin_, data + 1 - orig_data_begin_);
+        // But eliminate the last CR or LF
+        --buffer_n_;
+    }
+    message_complete_ = true;
+
+#if DEBUG_FLOW_BUFFER
+    fprintf(stderr, "%.6f Line complete: [%s]\n", network_time(),
+            string((const char*)begin(), (const char*)end()).c_str());
+#endif
+}
+
+/*
+Finite state automaton and STRICT_CRLF:
+(!--line is complete, *--add to buffer)
+
+STRICT_CRLF_0:
+    CR:	STRICT_CRLF_1 *
+    LF:	STRICT_CRLF_0 *
+    .:	STRICT_CRLF_0 *
+
+STRICT_CRLF_1:
+    CR:	STRICT_CRLF_1 *
+    LF:	STRICT_CRLF_0 ! (--buffer_n_)
+    .:	STRICT_CRLF_0 *
+*/
+
+void FlowBuffer::MarkOrCopyLine_STRICT_CRLF() {
+    const_byteptr data;
+    for ( data = orig_data_begin_; data < orig_data_end_; ++data ) {
+        switch ( *data ) {
+            case CR: state_ = STRICT_CRLF_1; break;
+
+            case LF:
+                if ( state_ == STRICT_CRLF_1 ) {
+                    state_ = STRICT_CRLF_0;
+                    goto found_end_of_line;
+                }
+                break;
+
+            default: state_ = STRICT_CRLF_0; break;
+        }
+    }
+
+    AppendToBuffer(orig_data_begin_, orig_data_end_ - orig_data_begin_);
+    return;
+
+found_end_of_line:
+    if ( buffer_n_ == 0 ) {
+        frame_length_ = data - 1 - orig_data_begin_;
+    }
+    else {
+        AppendToBuffer(orig_data_begin_, data + 1 - orig_data_begin_);
+        // Pop the preceding CR and LF from the buffer
+        buffer_n_ -= 2;
+    }
+
+    message_complete_ = true;
+
+#if DEBUG_FLOW_BUFFER
+    fprintf(stderr, "%.6f Line complete: [%s]\n", network_time(),
+            string((const char*)begin(), (const char*)end()).c_str());
+#endif
+}
+
+void FlowBuffer::MarkOrCopyLine_LINEBREAK() {
+    if ( ! (orig_data_begin_ && orig_data_end_) )
+        return;
+
+    const_byteptr data;
+    for ( data = orig_data_begin_; data < orig_data_end_; ++data ) {
+        if ( *data == linebreaker_ )
+            goto found_end_of_line;
+    }
+
+    AppendToBuffer(orig_data_begin_, orig_data_end_ - orig_data_begin_);
+    return;
+
+found_end_of_line:
+    if ( buffer_n_ == 0 ) {
+        frame_length_ = data - orig_data_begin_;
+    }
+    else {
+        AppendToBuffer(orig_data_begin_, data + 1 - orig_data_begin_);
+        // But eliminate the last 'linebreaker' character
+        --buffer_n_;
+    }
+    message_complete_ = true;
+
+#if DEBUG_FLOW_BUFFER
+    fprintf(stderr, "%.6f Line complete: [%s]\n", network_time(),
+            string((const char*)begin(), (const char*)end()).c_str());
+#endif
+}
+
+// Invariants:
+//
+// When buffer_n_ == 0:
+// Frame = [orig_data_begin_..(orig_data_begin_ + frame_length_)]
+//
+// When buffer_n_ > 0:
+// Frame = [0..buffer_n_][orig_data_begin_..]
+
+void FlowBuffer::MarkOrCopyFrame() {
+    if ( mode_ == FRAME_MODE && state_ == CR_OR_LF_1 && orig_data_begin_ < orig_data_end_ ) {
+        // Skip the lingering LF
+        if ( *orig_data_begin_ == LF ) {
+            ++orig_data_begin_;
+        }
+        state_ = FRAME_0;
+    }
+
+    if ( buffer_n_ == 0 ) {
+        // If there is enough data
+        if ( frame_length_ >= 0 && orig_data_end_ - orig_data_begin_ >= frame_length_ ) {
+            // Do nothing except setting the message complete flag
+            message_complete_ = true;
+        }
+        else {
+            if ( ! chunked_ ) {
+                AppendToBuffer(orig_data_begin_, orig_data_end_ - orig_data_begin_);
+            }
+            message_complete_ = false;
+        }
+    }
+    else {
+        BINPAC_ASSERT(! chunked_);
+        int bytes_to_copy = orig_data_end_ - orig_data_begin_;
+        message_complete_ = false;
+        if ( frame_length_ >= 0 && buffer_n_ + bytes_to_copy >= frame_length_ ) {
+            bytes_to_copy = frame_length_ - buffer_n_;
+            message_complete_ = true;
+        }
+        AppendToBuffer(orig_data_begin_, bytes_to_copy);
+    }
+
+#if DEBUG_FLOW_BUFFER
+    if ( message_complete_ ) {
+        fprintf(stderr, "%.6f frame complete: [%s]\n", network_time(),
+                string((const char*)begin(), (const char*)end()).c_str());
+    }
+#endif
+}
+
+void FlowBuffer::AppendToBuffer(const_byteptr data, int len) {
+    if ( len <= 0 )
+        return;
+
+    BINPAC_ASSERT(! chunked_);
+    ExpandBuffer(buffer_n_ + len);
+    memcpy(buffer_ + buffer_n_, data, len);
+    buffer_n_ += len;
+
+    orig_data_begin_ += len;
+    BINPAC_ASSERT(orig_data_begin_ <= orig_data_end_);
+}
+
+} // namespace binpac

tools/binpac/lib/binpac_buffer.h

@@ -0,0 +1,168 @@
+#ifndef binpac_buffer_h
+#define binpac_buffer_h
+
+#include <sys/types.h>
+
+#include "binpac.h"
+
+namespace binpac {
+
+class FlowBuffer {
+public:
+    struct Policy {
+        int max_capacity;
+        int min_capacity;
+        int contract_threshold;
+    };
+
+    enum LineBreakStyle {
+        CR_OR_LF,     // CR or LF or CRLF
+        STRICT_CRLF,  // CR followed by LF
+        CR_LF_NUL,    // CR or LF or CR-LF or CR-NUL
+        LINE_BREAKER, // User specified linebreaker
+    };
+
+    FlowBuffer(LineBreakStyle linebreak_style = CR_OR_LF);
+    virtual ~FlowBuffer();
+
+    void NewData(const_byteptr begin, const_byteptr end);
+    void NewGap(int length);
+
+    // Interface for delayed parsing. Sometimes BinPAC doesn't get the
+    // buffering right and then one can use these to feed parts
+    // individually and assemble them internally. After calling
+    // FinishBuffer(), one can send the upper-layer flow an FlowEOF() to
+    // trigger parsing.
+    void BufferData(const_byteptr data, const_byteptr end);
+    void FinishBuffer();
+
+    // Discard unprocessed data
+    void DiscardData();
+
+    // Whether there is enough data for the frame
+    bool ready() const { return message_complete_ || mode_ == UNKNOWN_MODE; }
+
+    inline const_byteptr begin() const {
+        BINPAC_ASSERT(ready());
+        return (buffer_n_ == 0) ? orig_data_begin_ : buffer_;
+    }
+
+    inline const_byteptr end() const {
+        BINPAC_ASSERT(ready());
+        if ( buffer_n_ == 0 ) {
+            BINPAC_ASSERT(frame_length_ >= 0);
+            const_byteptr end = orig_data_begin_ + frame_length_;
+            BINPAC_ASSERT(end <= orig_data_end_);
+            return end;
+        }
+        else
+            return buffer_ + buffer_n_;
+    }
+
+    inline int data_length() const {
+        if ( buffer_n_ > 0 )
+            return buffer_n_;
+
+        if ( frame_length_ < 0 || orig_data_begin_ + frame_length_ > orig_data_end_ )
+            return orig_data_end_ - orig_data_begin_;
+        else
+            return frame_length_;
+    }
+
+    inline bool data_available() const { return buffer_n_ > 0 || orig_data_end_ > orig_data_begin_; }
+
+    void SetLineBreaker(unsigned char* lbreaker);
+    void UnsetLineBreaker();
+    void NewLine();
+    // A negative frame_length represents a frame till EOF
+    void NewFrame(int frame_length, bool chunked_);
+    void GrowFrame(int new_frame_length);
+
+    int data_seq() const {
+        int data_seq_at_orig_data_begin = data_seq_at_orig_data_end_ - (orig_data_end_ - orig_data_begin_);
+        if ( buffer_n_ > 0 )
+            return data_seq_at_orig_data_begin;
+        else
+            return data_seq_at_orig_data_begin + data_length();
+    }
+    bool eof() const { return eof_; }
+    void set_eof();
+
+    bool have_pending_request() const { return have_pending_request_; }
+
+    static void init(Policy p) { policy = p; }
+
+protected:
+    // Reset the buffer for a new message
+    void NewMessage();
+
+    void ClearPreviousData();
+
+    // Expand the buffer to at least <length> bytes. If there
+    // are contents in the existing buffer, copy them to the new
+    // buffer.
+    void ExpandBuffer(int length);
+
+    // Contract the buffer to some minimum capacity.
+    // Existing contents in the buffer are preserved (but only usage
+    // at the time of creation this function is when the contents
+    // are being discarded due to parsing exception or have already been
+    // copied out after parsing a complete unit).
+    void ContractBuffer();
+
+    // Reset line state when transit from frame mode to line mode.
+    void ResetLineState();
+
+    void AppendToBuffer(const_byteptr data, int len);
+
+    // MarkOrCopy{Line,Frame} sets message_complete_ and
+    // marks begin/end pointers if a line/frame is complete,
+    // otherwise it clears message_complete_ and copies all
+    // the original data to the buffer.
+    //
+    void MarkOrCopy();
+    void MarkOrCopyLine();
+    void MarkOrCopyFrame();
+
+    void MarkOrCopyLine_CR_OR_LF();
+    void MarkOrCopyLine_STRICT_CRLF();
+    void MarkOrCopyLine_LINEBREAK();
+
+    int buffer_n_;      // number of bytes in the buffer
+    int buffer_length_; // size of the buffer
+    unsigned char* buffer_;
+    bool message_complete_;
+    int frame_length_;
+    bool chunked_;
+    const_byteptr orig_data_begin_, orig_data_end_;
+
+    LineBreakStyle linebreak_style_;
+    LineBreakStyle linebreak_style_default;
+    unsigned char linebreaker_;
+
+    enum {
+        UNKNOWN_MODE,
+        LINE_MODE,
+        FRAME_MODE,
+    } mode_;
+
+    enum {
+        CR_OR_LF_0,
+        CR_OR_LF_1,
+        STRICT_CRLF_0,
+        STRICT_CRLF_1,
+        FRAME_0,
+    } state_;
+
+    int data_seq_at_orig_data_end_;
+    bool eof_;
+    bool have_pending_request_;
+
+    static Policy policy;
+};
+
+typedef FlowBuffer* flow_buffer_t;
+
+} // namespace binpac
+
+#endif // binpac_buffer_h

tools/binpac/lib/binpac_bytestring.cc

@@ -0,0 +1,15 @@
+#define binpac_regex_h
+
+#include "binpac_bytestring.h"
+
+#include <stdlib.h>
+
+namespace binpac {
+
+std::string std_string(bytestring const* s) { return std::string((const char*)s->begin(), (const char*)s->end()); }
+
+int bytestring_to_int(bytestring const* s) { return atoi((const char*)s->begin()); }
+
+double bytestring_to_double(bytestring const* s) { return atof((const char*)s->begin()); }
+
+} // namespace binpac

tools/binpac/lib/binpac_bytestring.h

@@ -0,0 +1,142 @@
+#ifndef binpac_bytestring_h
+#define binpac_bytestring_h
+
+#include <string.h>
+#include <string>
+
+#include "binpac.h"
+
+namespace binpac {
+
+template<class T>
+class datastring;
+
+template<class T>
+class const_datastring {
+public:
+    const_datastring() : begin_(0), end_(0) {}
+
+    const_datastring(T const* data, int length) : begin_(data), end_(data + length) {}
+
+    const_datastring(const T* begin, const T* end) : begin_(begin), end_(end) {}
+
+    const_datastring(datastring<T> const& s) : begin_(s.begin()), end_(s.end()) {}
+
+    void init(const T* data, int length) {
+        begin_ = data;
+        end_ = data + length;
+    }
+
+    T const* begin() const { return begin_; }
+    T const* end() const { return end_; }
+    int length() const { return end_ - begin_; }
+
+    T const& operator[](int index) const { return begin()[index]; }
+
+    bool operator==(const_datastring<T> const& s) {
+        if ( length() != s.length() )
+            return false;
+        return memcmp((const void*)begin(), (const void*)s.begin(), sizeof(T) * length()) == 0;
+    }
+
+    void set_begin(T const* begin) { begin_ = begin; }
+    void set_end(T const* end) { end_ = end; }
+
+private:
+    T const* begin_;
+    T const* end_;
+};
+
+typedef const_datastring<uint8> const_bytestring;
+
+template<class T>
+class datastring {
+public:
+    datastring() { clear(); }
+
+    datastring(T* data, int len) { set(data, len); }
+
+    datastring(T const* begin, T const* end) { set_const(begin, end - begin); }
+
+    datastring(datastring<T> const& x) : data_(x.data()), length_(x.length()) {}
+
+    explicit datastring(const_datastring<T> const& x) { set_const(x.begin(), x.length()); }
+
+    datastring const& operator=(datastring<T> const& x) {
+        BINPAC_ASSERT(! data_);
+        set(x.data(), x.length());
+        return *this;
+    }
+
+    void init(T const* begin, int length) {
+        BINPAC_ASSERT(! data_);
+        set_const(begin, length);
+    }
+
+    void clear() {
+        data_ = 0;
+        length_ = 0;
+    }
+
+    void free() {
+        if ( data_ )
+            delete[] data_;
+        clear();
+    }
+
+    void clone() { set_const(begin(), length()); }
+
+    datastring const& operator=(const_datastring<T> const& x) {
+        BINPAC_ASSERT(! data_);
+        set_const(x.begin(), x.length());
+        return *this;
+    }
+
+    T const& operator[](int index) const { return begin()[index]; }
+
+    T* data() const { return data_; }
+    int length() const { return length_; }
+
+    T const* begin() const { return data_; }
+    T const* end() const { return data_ + length_; }
+
+private:
+    void set(T* data, int len) {
+        data_ = data;
+        length_ = len;
+    }
+
+    void set_const(T const* data, int len) {
+        length_ = len;
+        data_ = new T[len + 1];
+        memcpy(data_, data, sizeof(T) * len);
+        data_[len] = 0;
+    }
+
+    T* data_;
+    int length_;
+};
+
+typedef datastring<uint8> bytestring;
+
+inline const char* c_str(bytestring const& s) { return (const char*)s.begin(); }
+
+inline std::string std_str(const_bytestring const& s) {
+    return std::string((const char*)s.begin(), (const char*)s.end());
+}
+
+inline bool operator==(bytestring const& s1, const char* s2) { return strcmp(c_str(s1), s2) == 0; }
+
+inline void get_pointers(const_bytestring const& s, uint8 const** pbegin, uint8 const** pend) {
+    *pbegin = s.begin();
+    *pend = s.end();
+}
+
+inline void get_pointers(bytestring const* s, uint8 const** pbegin, uint8 const** pend) {
+    *pbegin = s->begin();
+    *pend = s->end();
+}
+
+} // namespace binpac
+
+#endif // binpac_bytestring_h

tools/binpac/lib/binpac_exception.h

@@ -0,0 +1,95 @@
+#ifndef binpac_exception_h
+#define binpac_exception_h
+
+#include <inttypes.h>
+#include <stdint.h>
+
+namespace binpac {
+
+class Exception {
+public:
+    Exception(const char* m = 0) : msg_("binpac exception: ") {
+        if ( m )
+            append(m);
+        // abort();
+    }
+
+    void append(string m) { msg_ += m; }
+    string msg() const { return msg_; }
+    const char* c_msg() const { return msg_.c_str(); }
+
+protected:
+    string msg_;
+};
+
+class ExceptionEnforceViolation : public Exception {
+public:
+    ExceptionEnforceViolation(const char* where) { append(binpac_fmt("&enforce violation : %s", where)); }
+};
+
+class ExceptionOutOfBound : public Exception {
+public:
+    ExceptionOutOfBound(const char* where, int len_needed, int len_given) {
+        append(binpac_fmt("out_of_bound: %s: %d > %d", where, len_needed, len_given));
+    }
+};
+
+class ExceptionInvalidCase : public Exception {
+public:
+    ExceptionInvalidCase(const char* location, int64_t index, const char* expected)
+        : location_(location), index_(index), expected_(expected) {
+        append(binpac_fmt("invalid case: %s: %" PRIi64 " (%s)", location, index, expected));
+    }
+
+protected:
+    const char* location_;
+    int64_t index_;
+    string expected_;
+};
+
+class ExceptionInvalidCaseIndex : public Exception {
+public:
+    ExceptionInvalidCaseIndex(const char* location, int64_t index) : location_(location), index_(index) {
+        append(binpac_fmt("invalid index for case: %s: %" PRIi64, location, index));
+    }
+
+protected:
+    const char* location_;
+    int64_t index_;
+};
+
+class ExceptionInvalidOffset : public Exception {
+public:
+    ExceptionInvalidOffset(const char* location, int min_offset, int offset)
+        : location_(location), min_offset_(min_offset), offset_(offset) {
+        append(binpac_fmt("invalid offset: %s: min_offset = %d, offset = %d", location, min_offset, offset));
+    }
+
+protected:
+    const char* location_;
+    int min_offset_, offset_;
+};
+
+class ExceptionStringMismatch : public Exception {
+public:
+    ExceptionStringMismatch(const char* location, const char* expected, const char* actual_data) {
+        append(binpac_fmt("string mismatch at %s: \nexpected pattern: \"%s\"\nactual data: \"%s\"", location, expected,
+                          actual_data));
+    }
+};
+
+class ExceptionInvalidStringLength : public Exception {
+public:
+    ExceptionInvalidStringLength(const char* location, int len) {
+        append(binpac_fmt("invalid length string: %s: %d", location, len));
+    }
+};
+
+class ExceptionFlowBufferAlloc : public Exception {
+public:
+    ExceptionFlowBufferAlloc(const char* reason) { append(binpac_fmt("flowbuffer allocation failed: %s", reason)); }
+};
+
+} // namespace binpac
+
+#endif // binpac_exception_h

tools/binpac/lib/binpac_regex.cc

@@ -0,0 +1,12 @@
+
+#include <vector>
+
+namespace zeek {
+class RE_Matcher;
+}
+
+namespace binpac {
+
+std::vector<zeek::RE_Matcher*>* uncompiled_re_matchers = nullptr;
+
+}

tools/binpac/lib/binpac_regex.h

@@ -0,0 +1,72 @@
+#ifndef binpac_regex_h
+#define binpac_regex_h
+
+#include "zeek/RE.h"
+
+#include "binpac.h"
+
+namespace zeek {
+class RE_Matcher;
+}
+
+namespace binpac {
+
+// Must be called before any binpac functionality is used.
+//
+// Note, this must be declared/defined here, and inline, because the RE
+// functionality can only be used when compiling from inside Zeek.
+// A copy is made of any FlowBuffer policy struct data passed.
+inline void init(FlowBuffer::Policy* fbp = 0);
+
+// Internal vector recording not yet compiled matchers.
+extern std::vector<zeek::RE_Matcher*>* uncompiled_re_matchers;
+
+class RegExMatcher {
+public:
+    RegExMatcher(const char* pattern) : pattern_(pattern) {
+        if ( ! uncompiled_re_matchers )
+            uncompiled_re_matchers = new std::vector<zeek::RE_Matcher*>;
+
+        re_matcher_ = new zeek::RE_Matcher(pattern_.c_str());
+        uncompiled_re_matchers->push_back(re_matcher_);
+    }
+
+    ~RegExMatcher() { delete re_matcher_; }
+
+    // Returns the length of longest match, or -1 on mismatch.
+    int MatchPrefix(const_byteptr data, int len) { return re_matcher_->MatchPrefix(data, len); }
+
+private:
+    friend void ::binpac::init(FlowBuffer::Policy*);
+
+    // Function, and state, for compiling matchers.
+    static void init();
+
+    string pattern_;
+    zeek::RE_Matcher* re_matcher_;
+};
+
+inline void RegExMatcher::init() {
+    if ( ! uncompiled_re_matchers )
+        return;
+
+    for ( size_t i = 0; i < uncompiled_re_matchers->size(); ++i ) {
+        if ( ! (*uncompiled_re_matchers)[i]->Compile() ) {
+            fprintf(stderr, "binpac: cannot compile regular expression\n");
+            exit(1);
+        }
+    }
+
+    uncompiled_re_matchers->clear();
+}
+
+inline void init(FlowBuffer::Policy* fbp) {
+    RegExMatcher::init();
+
+    if ( fbp )
+        FlowBuffer::init(*fbp);
+}
+
+} // namespace binpac
+
+#endif // binpac_regex_h

tools/binpac/patches/README

@@ -0,0 +1,2 @@
+Note: It's unclear which of these patches have in fact already been
+applied. We should figure that out ...

tools/binpac/patches/binpac-5.patch

@@ -0,0 +1,66 @@
+diff -urN bro-1.2.1-orig/src/pac_paramtype.cc bro-1.2.1-ssl-binpac/src/pac_paramtype.cc
+--- bro-1.2.1-orig/src/pac_paramtype.cc	2006-07-26 15:02:40.000000000 -0700
++++ bro-1.2.1-ssl-binpac/src/pac_paramtype.cc	2007-05-10 15:09:47.470104000 -0700
+@@ -208,7 +208,13 @@
+ 	const char *parse_func;
+ 	string parse_params;
+
+-	if ( ref_type->incremental_input() )
++	if ( buffer_mode() == BUFFER_NOTHING )
++	        {
++		ASSERT(!ref_type->incremental_input());
++		parse_func = kParseFuncWithoutBuffer;
++		parse_params = "0, 0";
++		}
++	else if ( ref_type->incremental_input() )
+ 		{
+ 		parse_func = kParseFuncWithBuffer;
+ 		parse_params = env->RValue(flow_buffer_id);
+@@ -239,15 +245,24 @@
+
+ 	if ( incremental_input() )
+ 		{
+-		ASSERT(parsing_complete_var());
+-		out_cc->println("%s = %s;",
+-			env->LValue(parsing_complete_var()),
+-			call_parse_func.c_str());
+-
+-		// parsing_complete_var might have been already
+-		// evaluated when set to false
+-		if ( ! env->Evaluated(parsing_complete_var()) )
+-			env->SetEvaluated(parsing_complete_var());
++		if ( buffer_mode() == BUFFER_NOTHING )
++		        {
++		        out_cc->println("%s;", call_parse_func.c_str());
++			out_cc->println("%s = true;",
++				env->LValue(parsing_complete_var()));
++			}
++		else
++		        {
++			ASSERT(parsing_complete_var());
++			out_cc->println("%s = %s;",
++				env->LValue(parsing_complete_var()),
++				call_parse_func.c_str());
++
++			// parsing_complete_var might have been already
++			// evaluated when set to false
++			if ( ! env->Evaluated(parsing_complete_var()) )
++			        env->SetEvaluated(parsing_complete_var());
++			}
+ 		}
+ 	else
+ 		{
+diff -urN bro-1.2.1-orig/src/pac_type.cc bro-1.2.1-ssl-binpac/src/pac_type.cc
+--- bro-1.2.1-orig/src/pac_type.cc	2006-07-26 15:02:40.000000000 -0700
++++ bro-1.2.1-ssl-binpac/src/pac_type.cc	2007-05-24 10:56:42.140658000 -0700
+@@ -501,8 +501,8 @@
+
+ 		if ( buffer_mode() == BUFFER_NOTHING )
+ 			{
+-			out_cc->println("%s = true;",
+-				env->LValue(parsing_complete_var()));
++			// this is the empty type
++			DoGenParseCode(out_cc, env, data, flags);
+ 			}
+ 		else if ( buffer_input() )
+ 			{

tools/binpac/patches/binpac-7.patch

@@ -0,0 +1,21 @@
+diff -urN bro-1.2.1-orig/src/pac_type.cc bro-1.2.1-ssl-binpac/src/pac_type.cc
+--- bro-1.2.1-orig/src/pac_type.cc	2006-07-26 15:02:40.000000000 -0700
++++ bro-1.2.1-ssl-binpac/src/pac_type.cc	2007-05-24 10:56:42.140658000 -0700
+@@ -393,7 +393,7 @@
+ 			break;
+
+ 		case BUFFER_BY_LENGTH:
+-			if ( buffering_state_var_field_ )
++		        if ( env->GetDataType(buffering_state_id) )
+ 				{
+ 				out_cc->println("if ( %s == 0 )",
+ 					env->RValue(buffering_state_id));
+@@ -421,7 +421,7 @@
+ 				frame_buffer_arg.c_str(),
+ 				attr_chunked() ? "true" : "false");
+
+-			if ( buffering_state_var_field_ )
++			if ( env->GetDataType(buffering_state_id) )
+ 				{
+ 				out_cc->println("%s = 1;",
+ 					env->LValue(buffering_state_id));

tools/binpac/patches/binpac-patch-doc.txt

@@ -0,0 +1,87 @@
+binpac fixes
+----------------
+
+numbers of issues below correspond to the patch numbers
+
+(1) correct calculation of minimal header size in pac_expr.cc
+- problem: EXPR_CALLARGS and EXPR_CASE not considered for the calculation
+  of minimal header size
+- solution: added two cases in switch stmt of Expr::MinimalHeaderSize
+  for EXPR_CALLARGS and EXPR_CASE
+
+
+(2) ensure parsing of fields first referenced in a case expression or
+    let field with an &if attribute
+- problem: in cases where the if expression evaluates to false or the
+  proper case does not occur, fields get not parsed at all
+- solution: force evaluation of all IDs referenced in a let field with
+  if attribute or a case expression before the body of the corresponding
+  switch stmt or the if stmt
+- added public method Expr::ForceIDEval, properly called before
+  generating the code of a field with if attribute or the case expression
+
+
+(3) properly assert the use of fields with an if attribute
+- problem: the use of fields with an if attribute was not asserted in all
+  cases and asserted in the wrong way in some others due to the
+  corresponding BINPAC_ASSERT only called upon parsing the field
+- solution: perform BINPAC_ASSERT upon calling the fields accessor
+  function
+- moved BINPAC_ASSERT statement from LetField::GenEval to
+  Type::GenPubDecls
+
+
+(4) incremental input with records with a non-negative StaticSize
+- problem: incremental input with records with a StaticSize >= 0
+  cannot be performed due to necessary length attribute, leading to
+  an invalid call of GenBoundaryCheck in RecordType::DoGenParseCode
+- solution: added a check for incremental input in
+  RecordType::DoGenParseCode before calling GenBoundaryCheck
+
+
+(5) empty type with incremental input
+- problem: with an empty type and incremental input, although the
+  Parse function is created, it is never called, leading to problems,
+  if additional actions are to be performed when encountering that
+  empty type
+- solution: generate call to Parse of empty type in Type::GenParseBuffer
+
+
+(6) parsing loop in flow ParseBuffer (while(true))
+- problem: while(true) leads to problems after parsing of a type is
+  complete; at this time, it is unexpected that parsing continues, even
+  if no data is available in the flow buffer
+- solution: check if data is available before starting a new parsing
+  cycle
+- added a method data_available to FlowBuffer
+- changed while(true) in FlowDecl::GenCodeFlowUnit to
+  while(flow_buffer_->data_available())
+
+
+(7) initialization of flow buffer in CaseType with bufferable fields
+    in cases
+- problem: initialization of buffer occurs in every Parse call,
+  regardless if it was initialized before or not; initialization
+  is correct only on first such occurrence
+- solution: check to buffer_state is to be created always when
+  buffering_state_id is in environment in Type::GenBufferConfig
+- changed condition from buffering_state_var_field_ to
+  env->GetDataType(buffering_state_id)
+
+
+(8) allowing init and cleanup code to be redefined, as well as addition
+    of code to FlowEOF calls in analyzer and flow
+- problem 1: when refining an analyzer or flow definition, additional
+  init and cleanup code was not allowed, if these were already defined
+  before; this leads to problems when adding new members, as these
+  cannot be initialized and destroyed properly
+- solution: allow init and cleanup code to be specified more than once
+- changed deifnitions and usage of constructor_helper and
+  destructor_helper to allow for lists of constructor and destructor
+  helpers (similar to member declarations) in pac_analyzer.h and
+  pac_analyzer.cc
+- problem 2: in some cases, it is desirable to execute code when
+  encountering the end of the input stream, which is not possible in
+  binpac
+- solution: added a %eof binpac primitive similar to %init, which adds
+  code to the FlowEOF function of an analyzer or a flow

tools/binpac/src/CMakeLists.txt

@@ -0,0 +1,77 @@
+find_package(FLEX REQUIRED)
+find_package(BISON REQUIRED)
+
+bison_target(PACParser pac_parse.yy ${CMAKE_CURRENT_BINARY_DIR}/pac_parse.cc
+             DEFINES_FILE ${CMAKE_CURRENT_BINARY_DIR}/pac_parse.h COMPILE_FLAGS "--debug")
+flex_target(PACScanner pac_scan.ll ${CMAKE_CURRENT_BINARY_DIR}/pac_scan.cc)
+add_flex_bison_dependency(PACScanner PACParser)
+if (MSVC)
+    set_property(SOURCE pac_scan.cc APPEND_STRING PROPERTY COMPILE_FLAGS "/wd4018")
+else ()
+    set_property(SOURCE pac_scan.cc APPEND_STRING PROPERTY COMPILE_FLAGS "-Wno-sign-compare")
+endif ()
+
+set(binpac_SRCS
+    ${BISON_PACParser_INPUT}
+    ${FLEX_PACScanner_INPUT}
+    ${BISON_PACParser_OUTPUTS}
+    ${FLEX_PACScanner_OUTPUTS}
+    pac_action.cc
+    pac_analyzer.cc
+    pac_array.cc
+    pac_attr.cc
+    pac_btype.cc
+    pac_case.cc
+    pac_conn.cc
+    pac_context.cc
+    pac_cstr.cc
+    pac_datadep.cc
+    pac_dataptr.cc
+    pac_dataunit.cc
+    pac_decl.cc
+    pac_embedded.cc
+    pac_enum.cc
+    pac_expr.cc
+    pac_exttype.cc
+    pac_field.cc
+    pac_flow.cc
+    pac_func.cc
+    pac_id.cc
+    pac_inputbuf.cc
+    pac_let.cc
+    pac_param.cc
+    pac_paramtype.cc
+    pac_primitive.cc
+    pac_record.cc
+    pac_redef.cc
+    pac_regex.cc
+    pac_state.cc
+    pac_strtype.cc
+    pac_type.cc
+    pac_typedecl.cc
+    pac_withinput.cc
+    pac_output.cc
+    pac_utils.cc
+    pac_exception.cc
+    pac_main.cc)
+
+add_executable(binpac ${binpac_SRCS})
+
+target_include_directories(binpac BEFORE PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
+target_include_directories(binpac BEFORE PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
+
+if (MSVC)
+    target_compile_options(binpac PUBLIC "/J")
+    # If building separately from zeek, we need to add the libunistd subdirectory
+    # so that linking doesn't fail.
+    if ("${CMAKE_PROJECT_NAME}" STREQUAL "BinPAC")
+        add_subdirectory(${PROJECT_SOURCE_DIR}auxil/libunistd EXCLUDE_FROM_ALL)
+    endif ()
+    target_link_libraries(binpac PRIVATE libunistd)
+endif ()
+
+install(TARGETS binpac DESTINATION bin)
+
+# This is set to assist superprojects that want to build BinPac from source and
+# rely on it as a target
+set(BinPAC_EXE binpac CACHE STRING "BinPAC executable" FORCE)

tools/binpac/src/pac_action.cc

@@ -0,0 +1,79 @@
+#include "pac_action.h"
+
+#include "pac_embedded.h"
+#include "pac_exception.h"
+#include "pac_id.h"
+#include "pac_output.h"
+#include "pac_type.h"
+#include "pac_typedecl.h"
+#include "pac_utils.h"
+
+AnalyzerAction::AnalyzerAction(ID* action_id, When when, ActionParam* param, EmbeddedCode* code)
+    : AnalyzerElement(ACTION), action_id_(action_id), when_(when), param_(param), code_(code), analyzer_(nullptr) {}
+
+AnalyzerAction::~AnalyzerAction() {
+    delete action_id_;
+    delete param_;
+    delete code_;
+}
+
+string AnalyzerAction::action_function() const { return strfmt("Action_%s", action_id_->Name()); }
+
+void AnalyzerAction::InstallHook(AnalyzerDecl* analyzer) {
+    ASSERT(0);
+    analyzer_ = analyzer;
+    // param_->MainDataType()->InstallAction(this);
+}
+
+void AnalyzerAction::GenCode(Output* out_h, Output* out_cc, AnalyzerDecl* decl) {
+    Env action_func_env(decl->env(), this);
+    action_func_env.AddID(param_->id(), TEMP_VAR, param_->DataType());
+    action_func_env.SetEvaluated(param_->id());
+
+    string action_func_proto = strfmt("%s(%s)", action_function().c_str(), ParamDecls(&action_func_env).c_str());
+
+    out_h->println("void %s;", action_func_proto.c_str());
+
+    out_cc->println("void %s::%s {", decl->class_name().c_str(), action_func_proto.c_str());
+    out_cc->inc_indent();
+
+    code_->GenCode(out_cc, &action_func_env);
+
+    out_cc->println("");
+    out_cc->dec_indent();
+    out_cc->println("}");
+    out_cc->println("");
+}
+
+string AnalyzerAction::ParamDecls(Env* env) const { return param_->DeclStr(env); }
+
+Type* ActionParam::MainDataType() const {
+    // Note: this is not equal to DataType()
+    Type* main_type = TypeDecl::LookUpType(type()->type_id());
+
+    if ( ! main_type ) {
+        throw Exception(type()->type_id(), "type not defined");
+    }
+
+    return main_type;
+}
+
+Type* ActionParam::DataType() const {
+    Type* main_type = MainDataType();
+
+    if ( ! type()->field_id() ) {
+        return main_type;
+    }
+    else {
+        Type* member_type = main_type->MemberDataType(type()->field_id());
+        if ( ! member_type ) {
+            throw Exception(type()->field_id(), strfmt("cannot find member type for `%s.%s'", type()->type_id()->Name(),
+                                                       type()->field_id()->Name()));
+        }
+        return member_type;
+    }
+}
+
+string ActionParam::DeclStr(Env* env) const {
+    return strfmt("%s %s", DataType()->DataTypeStr().c_str(), env->LValue(id()));
+}

tools/binpac/src/pac_action.h

@@ -0,0 +1,66 @@
+#ifndef pac_action_h
+#define pac_action_h
+
+// Classes representing analyzer actions.
+
+#include "pac_analyzer.h"
+#include "pac_common.h"
+
+class AnalyzerAction : public AnalyzerElement {
+public:
+    enum When { BEFORE, AFTER };
+
+    AnalyzerAction(ID* action_id, When when, ActionParam* param, EmbeddedCode* code);
+
+    ~AnalyzerAction() override;
+
+    When when() const { return when_; }
+    ActionParam* param() const { return param_; }
+    AnalyzerDecl* analyzer() const { return analyzer_; }
+    string action_function() const;
+
+    // Generate function prototype and code for the action
+    void GenCode(Output* out_h, Output* out_cc, AnalyzerDecl* decl);
+
+    // Install the hook at the corresponding data type parsing
+    // function to invoke the action.
+    void InstallHook(AnalyzerDecl* analyzer);
+
+private:
+    string ParamDecls(Env* env) const;
+
+    ID* action_id_;
+    When when_;
+    ActionParam* param_;
+    EmbeddedCode* code_;
+    AnalyzerDecl* analyzer_;
+};
+
+class ActionParam {
+public:
+    ActionParam(const ID* id, ActionParamType* type) : id_(id), type_(type) {}
+
+    const ID* id() const { return id_; }
+    ActionParamType* type() const { return type_; }
+
+    Type* MainDataType() const;
+    Type* DataType() const;
+    string DeclStr(Env* env) const;
+
+private:
+    const ID* id_;
+    ActionParamType* type_;
+};
+
+class ActionParamType {
+public:
+    ActionParamType(const ID* type_id, const ID* field_id = 0) : type_id_(type_id), field_id_(field_id) {}
+
+    const ID* type_id() const { return type_id_; }
+    const ID* field_id() const { return field_id_; }
+
+protected:
+    const ID *type_id_, *field_id_;
+};
+
+#endif // pac_action_h

tools/binpac/src/pac_analyzer.cc

@@ -0,0 +1,263 @@
+#include "pac_analyzer.h"
+
+#include "pac_action.h"
+#include "pac_context.h"
+#include "pac_embedded.h"
+#include "pac_exception.h"
+#include "pac_expr.h"
+#include "pac_flow.h"
+#include "pac_func.h"
+#include "pac_output.h"
+#include "pac_param.h"
+#include "pac_paramtype.h"
+#include "pac_state.h"
+#include "pac_type.h"
+#include "pac_varfield.h"
+
+AnalyzerDecl::AnalyzerDecl(ID* id, DeclType decl_type, ParamList* params) : TypeDecl(id, params, new DummyType()) {
+    decl_type_ = decl_type;
+
+    statevars_ = new StateVarList();
+    actions_ = new AnalyzerActionList();
+    helpers_ = new AnalyzerHelperList();
+    functions_ = new FunctionList();
+
+    constructor_helpers_ = new AnalyzerHelperList();
+    destructor_helpers_ = new AnalyzerHelperList();
+    eof_helpers_ = new AnalyzerHelperList();
+
+    SetAnalyzerContext();
+
+    env_ = nullptr;
+}
+
+AnalyzerDecl::~AnalyzerDecl() {
+    delete_list(StateVarList, statevars_);
+    delete_list(AnalyzerActionList, actions_);
+    delete_list(AnalyzerHelperList, helpers_);
+    delete_list(FunctionList, functions_);
+    delete_list(ParamList, params_);
+    delete_list(AnalyzerHelperList, constructor_helpers_);
+    delete_list(AnalyzerHelperList, destructor_helpers_);
+    delete_list(AnalyzerHelperList, eof_helpers_);
+}
+
+void AnalyzerDecl::AddElements(AnalyzerElementList* elemlist) {
+    ASSERT(! env_);
+    foreach (i, AnalyzerElementList, elemlist) {
+        AnalyzerElement* elem = *i;
+        switch ( elem->type() ) {
+            case AnalyzerElement::STATE: {
+                ASSERT(0);
+                AnalyzerState* state_elem = (AnalyzerState*)elem;
+                statevars_->insert(statevars_->end(), state_elem->statevars()->begin(), state_elem->statevars()->end());
+            } break;
+            case AnalyzerElement::ACTION: {
+                ASSERT(0);
+                AnalyzerAction* action_elem = (AnalyzerAction*)elem;
+                actions_->push_back(action_elem);
+            } break;
+            case AnalyzerElement::HELPER: {
+                AnalyzerHelper* helper_elem = (AnalyzerHelper*)elem;
+
+                switch ( helper_elem->helper_type() ) {
+                    case AnalyzerHelper::INIT_CODE: constructor_helpers_->push_back(helper_elem); break;
+                    case AnalyzerHelper::CLEANUP_CODE: destructor_helpers_->push_back(helper_elem); break;
+                    case AnalyzerHelper::EOF_CODE: eof_helpers_->push_back(helper_elem); break;
+                    default: helpers_->push_back(helper_elem);
+                }
+            } break;
+            case AnalyzerElement::FUNCTION: {
+                AnalyzerFunction* func_elem = (AnalyzerFunction*)elem;
+                Function* func = func_elem->function();
+                func->set_analyzer_decl(this);
+                functions_->push_back(func);
+            } break;
+            case AnalyzerElement::FLOW: {
+                AnalyzerFlow* flow_elem = (AnalyzerFlow*)elem;
+                ProcessFlowElement(flow_elem);
+            } break;
+            case AnalyzerElement::DATAUNIT: {
+                AnalyzerDataUnit* dataunit_elem = (AnalyzerDataUnit*)elem;
+                ProcessDataUnitElement(dataunit_elem);
+            } break;
+        }
+    }
+}
+
+string AnalyzerDecl::class_name() const { return id_->Name(); }
+
+void AnalyzerDecl::Prepare() {
+    TypeDecl::Prepare();
+
+    ASSERT(statevars_->empty());
+    ASSERT(actions_->empty());
+
+    foreach (i, FunctionList, functions_) {
+        Function* function = *i;
+        function->Prepare(env_);
+    }
+    foreach (i, StateVarList, statevars_) {
+        StateVar* statevar = *i;
+        env_->AddID(statevar->id(), STATE_VAR, statevar->type());
+    }
+    foreach (i, AnalyzerActionList, actions_) {
+        AnalyzerAction* action = *i;
+        action->InstallHook(this);
+    }
+}
+
+void AnalyzerDecl::GenForwardDeclaration(Output* out_h) {
+    out_h->println("class %s;", class_name().c_str());
+    foreach (i, FunctionList, functions_) {
+        Function* function = *i;
+        function->GenForwardDeclaration(out_h);
+    }
+}
+
+void AnalyzerDecl::GenActions(Output* out_h, Output* out_cc) {
+    foreach (i, AnalyzerActionList, actions_) {
+        (*i)->GenCode(out_h, out_cc, this);
+    }
+}
+
+void AnalyzerDecl::GenHelpers(Output* out_h, Output* out_cc) {
+    foreach (i, AnalyzerHelperList, helpers_) {
+        (*i)->GenCode(out_h, out_cc, this);
+    }
+}
+
+void AnalyzerDecl::GenPubDecls(Output* out_h, Output* out_cc) {
+    TypeDecl::GenPubDecls(out_h, out_cc);
+
+    GenProcessFunc(out_h, out_cc);
+    GenGapFunc(out_h, out_cc);
+    GenEOFFunc(out_h, out_cc);
+    out_h->println("");
+
+    if ( ! functions_->empty() ) {
+        out_h->println("// Functions");
+        GenFunctions(out_h, out_cc);
+        out_h->println("");
+    }
+
+    // TODO: export public state variables
+}
+
+void AnalyzerDecl::GenPrivDecls(Output* out_h, Output* out_cc) {
+    TypeDecl::GenPrivDecls(out_h, out_cc);
+
+    if ( ! helpers_->empty() ) {
+        out_h->println("");
+        out_h->println("// Additional members");
+        GenHelpers(out_h, out_cc);
+    }
+
+    // TODO: declare state variables
+}
+
+void AnalyzerDecl::GenInitCode(Output* out_cc) {
+    TypeDecl::GenInitCode(out_cc);
+    foreach (i, AnalyzerHelperList, constructor_helpers_) {
+        (*i)->GenCode(nullptr, out_cc, this);
+    }
+}
+
+void AnalyzerDecl::GenCleanUpCode(Output* out_cc) {
+    TypeDecl::GenCleanUpCode(out_cc);
+    foreach (i, AnalyzerHelperList, destructor_helpers_) {
+        (*i)->GenCode(nullptr, out_cc, this);
+    }
+}
+
+void AnalyzerDecl::GenStateVarDecls(Output* out_h) {
+    foreach (i, StateVarList, statevars_) {
+        StateVar* var = *i;
+        var->GenDecl(out_h, env_);
+    }
+}
+
+void AnalyzerDecl::GenStateVarSetFunctions(Output* out_h) {
+    foreach (i, StateVarList, statevars_) {
+        StateVar* var = *i;
+        var->GenSetFunction(out_h, env_);
+    }
+}
+
+void AnalyzerDecl::GenStateVarInitCode(Output* out_cc) {
+    foreach (i, StateVarList, statevars_) {
+        StateVar* var = *i;
+        var->GenInitCode(out_cc, env_);
+    }
+}
+
+void AnalyzerDecl::GenStateVarCleanUpCode(Output* out_cc) {
+    foreach (i, StateVarList, statevars_) {
+        StateVar* var = *i;
+        var->GenCleanUpCode(out_cc, env_);
+    }
+}
+
+void AnalyzerDecl::GenFunctions(Output* out_h, Output* out_cc) {
+    foreach (i, FunctionList, functions_) {
+        Function* function = *i;
+        function->GenCode(out_h, out_cc);
+    }
+}
+
+AnalyzerState::~AnalyzerState() {
+    // Note: do not delete elements of statevars_, because they
+    // are referenced by the AnalyzerDecl.
+    delete statevars_;
+}
+
+AnalyzerHelper::~AnalyzerHelper() { delete code_; }
+
+void AnalyzerHelper::GenCode(Output* out_h, Output* out_cc, AnalyzerDecl* decl) {
+    Output* out = nullptr;
+    switch ( helper_type_ ) {
+        case MEMBER_DECLS: out = out_h; break;
+        case INIT_CODE:
+        case CLEANUP_CODE:
+        case EOF_CODE: out = out_cc; break;
+    }
+    ASSERT(out);
+    code()->GenCode(out, decl->env());
+}
+
+FlowField::FlowField(ID* flow_id, ParameterizedType* flow_type)
+    : Field(FLOW_FIELD, TYPE_NOT_TO_BE_PARSED | CLASS_MEMBER | PUBLIC_READABLE, flow_id, flow_type) {}
+
+void FlowField::GenInitCode(Output* out_cc, Env* env) { type_->GenPreParsing(out_cc, env); }
+
+AnalyzerFlow::AnalyzerFlow(Direction dir, ID* type_id, ExprList* params)
+    : AnalyzerElement(FLOW), dir_(dir), type_id_(type_id) {
+    if ( ! params )
+        params = new ExprList();
+
+    // Add "this" to the list of params
+    params->insert(params->begin(), new Expr(this_id->clone()));
+
+    ID* flow_id = ((dir == UP) ? upflow_id : downflow_id)->clone();
+
+    ParameterizedType* flow_type = new ParameterizedType(type_id_, params);
+
+    flow_field_ = new FlowField(flow_id, flow_type);
+
+    flow_decl_ = nullptr;
+}
+
+AnalyzerFlow::~AnalyzerFlow() { delete flow_field_; }
+
+FlowDecl* AnalyzerFlow::flow_decl() {
+    DEBUG_MSG("Getting flow_decl for %s\n", type_id_->Name());
+    if ( ! flow_decl_ ) {
+        Decl* decl = Decl::LookUpDecl(type_id_);
+        if ( decl && decl->decl_type() == Decl::FLOW )
+            flow_decl_ = static_cast<FlowDecl*>(decl);
+        if ( ! flow_decl_ ) {
+            throw Exception(this, "cannot find the flow declaration");
+        }
+    }
+    return flow_decl_;
+}

tools/binpac/src/pac_analyzer.h

@@ -0,0 +1,157 @@
+#ifndef pac_analyzer_h
+#define pac_analyzer_h
+
+#include "pac_common.h"
+#include "pac_field.h"
+#include "pac_typedecl.h"
+
+class AnalyzerElement;
+class AnalyzerState;
+class AnalyzerAction; // defined in pac_action.h
+class AnalyzerHelper;
+class AnalyzerFlow;
+class AnalyzerDataUnit;
+class AnalyzerFunction;
+class ConnDecl;
+class FlowDecl;
+typedef vector<AnalyzerHelper*> AnalyzerHelperList;
+typedef vector<Function*> FunctionList;
+
+class AnalyzerDecl : public TypeDecl {
+public:
+    AnalyzerDecl(ID* id, DeclType decl_type, ParamList* params);
+    ~AnalyzerDecl() override;
+
+    void AddElements(AnalyzerElementList* elemlist);
+
+    void Prepare() override;
+    void GenForwardDeclaration(Output* out_h) override;
+    // void GenCode(Output *out_h, Output *out_cc);
+
+    void GenInitCode(Output* out_cc) override;
+    void GenCleanUpCode(Output* out_cc) override;
+
+    string class_name() const;
+    // string cookie_name() const;
+
+protected:
+    virtual void ProcessFlowElement(AnalyzerFlow* flow_elem) = 0;
+    virtual void ProcessDataUnitElement(AnalyzerDataUnit* dataunit_elem) = 0;
+
+    // Generate public/private declarations for member functions and
+    // variables
+    void GenPubDecls(Output* out_h, Output* out_cc) override;
+    void GenPrivDecls(Output* out_h, Output* out_cc) override;
+
+    // Generate the NewData() function
+    virtual void GenProcessFunc(Output* out_h, Output* out_cc) = 0;
+
+    // Generate the NewGap() function
+    virtual void GenGapFunc(Output* out_h, Output* out_cc) = 0;
+
+    // Generate the FlowEOF() function
+    virtual void GenEOFFunc(Output* out_h, Output* out_cc) = 0;
+
+    // Generate the functions
+    void GenFunctions(Output* out_h, Output* out_cc);
+
+    // Generate the action functions
+    void GenActions(Output* out_h, Output* out_cc);
+
+    // Generate the helper code segments
+    void GenHelpers(Output* out_h, Output* out_cc);
+
+    // Generate declarations for state variables and their set functions
+    void GenStateVarDecls(Output* out_h);
+    void GenStateVarSetFunctions(Output* out_h);
+
+    // Generate code for initializing and cleaning up (including
+    // memory de-allocating) state variables
+    void GenStateVarInitCode(Output* out_cc);
+    void GenStateVarCleanUpCode(Output* out_cc);
+
+    StateVarList* statevars_;
+    AnalyzerActionList* actions_;
+    AnalyzerHelperList* helpers_;
+    FunctionList* functions_;
+
+    AnalyzerHelperList* constructor_helpers_;
+    AnalyzerHelperList* destructor_helpers_;
+    AnalyzerHelperList* eof_helpers_;
+};
+
+class AnalyzerElement : public Object {
+public:
+    enum ElementType { STATE, ACTION, FUNCTION, HELPER, FLOW, DATAUNIT };
+    AnalyzerElement(ElementType type) : type_(type) {}
+    virtual ~AnalyzerElement() {}
+
+    ElementType type() const { return type_; }
+
+private:
+    ElementType type_;
+};
+
+// A collection of variables representing analyzer states.
+class AnalyzerState : public AnalyzerElement {
+public:
+    AnalyzerState(StateVarList* statevars) : AnalyzerElement(STATE), statevars_(statevars) {}
+    ~AnalyzerState() override;
+
+    StateVarList* statevars() const { return statevars_; }
+
+private:
+    StateVarList* statevars_;
+};
+
+// A collection of embedded C++ code
+class AnalyzerHelper : public AnalyzerElement {
+public:
+    enum Type {
+        MEMBER_DECLS,
+        INIT_CODE,
+        CLEANUP_CODE,
+        EOF_CODE,
+    };
+    AnalyzerHelper(Type helper_type, EmbeddedCode* code)
+        : AnalyzerElement(HELPER), helper_type_(helper_type), code_(code) {}
+    ~AnalyzerHelper() override;
+
+    Type helper_type() const { return helper_type_; }
+
+    void GenCode(Output* out_h, Output* out_cc, AnalyzerDecl* decl);
+
+    EmbeddedCode* code() const { return code_; }
+
+private:
+    Type helper_type_;
+    EmbeddedCode* code_;
+};
+
+// The type and parameters of (uni-directional) flows of a connection.
+
+class FlowField : public Field {
+public:
+    FlowField(ID* flow_id, ParameterizedType* flow_type);
+    void GenInitCode(Output* out, Env* env) override;
+};
+
+class AnalyzerFlow : public AnalyzerElement {
+public:
+    enum Direction { UP, DOWN };
+    AnalyzerFlow(Direction dir, ID* type_id, ExprList* params);
+    ~AnalyzerFlow() override;
+
+    Direction dir() const { return dir_; }
+    FlowField* flow_field() const { return flow_field_; }
+
+    FlowDecl* flow_decl();
+
+private:
+    Direction dir_;
+    ID* type_id_;
+    FlowField* flow_field_;
+    FlowDecl* flow_decl_;
+};
+
+#endif // pac_analyzer_h

tools/binpac/src/pac_array.cc

@@ -0,0 +1,593 @@
+#include "pac_array.h"
+
+#include "pac_attr.h"
+#include "pac_dataptr.h"
+#include "pac_exception.h"
+#include "pac_expr.h"
+#include "pac_exttype.h"
+#include "pac_id.h"
+#include "pac_number.h"
+#include "pac_output.h"
+#include "pac_utils.h"
+#include "pac_varfield.h"
+
+ArrayType::ArrayType(Type* elemtype, Expr* length) : Type(ARRAY), elemtype_(elemtype), length_(length) {
+    init();
+
+    switch ( elemtype_->tot() ) {
+        case BUILTIN:
+        case PARAMETERIZED:
+        case STRING:
+        case EXTERN: break;
+
+        case ARRAY:
+        case CASE:
+        case DUMMY:
+        case EMPTY:
+        case RECORD:
+        case UNDEF: ASSERT(0); break;
+    }
+}
+
+void ArrayType::init() {
+    arraylength_var_field_ = nullptr;
+    elem_it_var_field_ = nullptr;
+    elem_var_field_ = nullptr;
+    elem_dataptr_var_field_ = nullptr;
+    elem_input_var_field_ = nullptr;
+
+    elem_dataptr_until_expr_ = nullptr;
+
+    end_of_array_loop_label_ = "@@@";
+
+    vector_str_ = strfmt("vector<%s>", elemtype_->DataTypeStr().c_str());
+
+    datatype_str_ = strfmt("%s*", vector_str_.c_str());
+
+    attr_generic_until_expr_ = nullptr;
+    attr_until_element_expr_ = nullptr;
+    attr_until_input_expr_ = nullptr;
+}
+
+ArrayType::~ArrayType() {
+    delete arraylength_var_field_;
+    delete elem_it_var_field_;
+    delete elem_var_field_;
+    delete elem_dataptr_var_field_;
+    delete elem_input_var_field_;
+
+    delete elem_dataptr_until_expr_;
+}
+
+Type* ArrayType::DoClone() const {
+    Type* elemtype = elemtype_->Clone();
+    if ( ! elemtype )
+        return nullptr;
+    return new ArrayType(elemtype, length_);
+}
+
+bool ArrayType::DefineValueVar() const { return true; }
+
+string ArrayType::DataTypeStr() const { return datatype_str_; }
+
+Type* ArrayType::ElementDataType() const { return elemtype_; }
+
+string ArrayType::EvalElement(const string& array, const string& index) const {
+    if ( attr_transient_ )
+        throw Exception(this, "cannot access element in &transient array");
+
+    return strfmt("(*(%s))[%s]", array.c_str(), index.c_str());
+}
+
+const ID* ArrayType::arraylength_var() const { return arraylength_var_field_ ? arraylength_var_field_->id() : nullptr; }
+
+const ID* ArrayType::elem_it_var() const { return elem_it_var_field_ ? elem_it_var_field_->id() : nullptr; }
+
+const ID* ArrayType::elem_var() const { return elem_var_field_ ? elem_var_field_->id() : nullptr; }
+
+const ID* ArrayType::elem_dataptr_var() const {
+    return elem_dataptr_var_field_ ? elem_dataptr_var_field_->id() : nullptr;
+}
+
+const ID* ArrayType::elem_input_var() const { return elem_input_var_field_ ? elem_input_var_field_->id() : nullptr; }
+
+void ArrayType::ProcessAttr(Attr* a) {
+    Type::ProcessAttr(a);
+
+    switch ( a->type() ) {
+        case ATTR_RESTOFDATA: {
+            if ( elemtype_->StaticSize(env()) != 1 ) {
+                throw Exception(elemtype_,
+                                "&restofdata can be applied"
+                                " to only byte arrays");
+            }
+            if ( length_ ) {
+                throw Exception(length_,
+                                "&restofdata cannot be applied"
+                                " to arrays with specified length");
+            }
+            attr_restofdata_ = true;
+            // As the array automatically extends to the end of
+            // data, we do not have to check boundary.
+            SetBoundaryChecked();
+        } break;
+
+        case ATTR_RESTOFFLOW:
+            attr_restofflow_ = true;
+            // TODO: handle &restofflow
+            break;
+
+        case ATTR_UNTIL: {
+            bool ref_element = a->expr()->HasReference(element_macro_id);
+            bool ref_input = a->expr()->HasReference(input_macro_id);
+            if ( ref_element && ref_input ) {
+                throw Exception(a->expr(),
+                                "cannot reference both $element and $input "
+                                "in the same &until---please separate them.");
+            }
+
+            if ( ref_element ) {
+                if ( attr_until_element_expr_ ) {
+                    throw Exception(a->expr(), "multiple &until on $element");
+                }
+                attr_until_element_expr_ = a->expr();
+            }
+            else if ( ref_input ) {
+                if ( attr_until_input_expr_ ) {
+                    throw Exception(a->expr(), "multiple &until on $input");
+                }
+                attr_until_input_expr_ = a->expr();
+            }
+            else {
+                if ( attr_generic_until_expr_ ) {
+                    throw Exception(a->expr(), "multiple &until condition");
+                }
+                attr_generic_until_expr_ = a->expr();
+            }
+        } break;
+
+        default: break;
+    }
+}
+
+void ArrayType::Prepare(Env* env, int flags) {
+    if ( flags & TO_BE_PARSED ) {
+        ID* arraylength_var = new ID(strfmt("%s__arraylength", value_var()->Name()));
+        ID* elem_var = new ID(strfmt("%s__elem", value_var()->Name()));
+        ID* elem_it_var = new ID(strfmt("%s__it", elem_var->Name()));
+
+        elem_var_field_ = new ParseVarField(Field::CLASS_MEMBER, elem_var, elemtype_);
+        AddField(elem_var_field_);
+
+        if ( incremental_parsing() ) {
+            arraylength_var_field_ = new PrivVarField(arraylength_var, extern_type_int->Clone());
+            elem_it_var_field_ = new PrivVarField(elem_it_var, extern_type_int->Clone());
+
+            AddField(arraylength_var_field_);
+            AddField(elem_it_var_field_);
+        }
+        else {
+            arraylength_var_field_ = new TempVarField(arraylength_var, extern_type_int->Clone());
+            elem_it_var_field_ = new TempVarField(elem_it_var, extern_type_int->Clone());
+
+            arraylength_var_field_->Prepare(env);
+            elem_it_var_field_->Prepare(env);
+
+            // Add elem_dataptr_var only when not parsing incrementally
+            ID* elem_dataptr_var = new ID(strfmt("%s__dataptr", elem_var->Name()));
+            elem_dataptr_var_field_ = new TempVarField(elem_dataptr_var, extern_type_const_byteptr->Clone());
+            elem_dataptr_var_field_->Prepare(env);
+
+            // until(dataptr >= end_of_data)
+            elem_dataptr_until_expr_ =
+                new Expr(Expr::EXPR_GE, new Expr(elem_dataptr_var->clone()), new Expr(end_of_data->clone()));
+        }
+
+        if ( attr_until_input_expr_ ) {
+            elemtype_->SetUntilCheck(this);
+        }
+
+        end_of_array_loop_label_ = strfmt("end_of_%s", value_var()->Name());
+    }
+
+    Type::Prepare(env, flags);
+}
+
+void ArrayType::GenArrayLength(Output* out_cc, Env* env, const DataPtr& data) {
+    if ( env->Evaluated(arraylength_var()) )
+        return;
+
+    if ( ! incremental_parsing() ) {
+        arraylength_var_field_->GenTempDecls(out_cc, env);
+        // This is about to get initialized below, don't initialize it twice.
+        if ( ! length_ && ! attr_restofdata_ )
+            arraylength_var_field_->GenInitCode(out_cc, env);
+    }
+
+    if ( length_ ) {
+        out_cc->println("%s = %s;", env->LValue(arraylength_var()), length_->EvalExpr(out_cc, env));
+
+        env->SetEvaluated(arraylength_var());
+
+        // Check negative array length
+        out_cc->println("if ( %s < 0 ) {", env->LValue(arraylength_var()));
+        out_cc->inc_indent();
+        out_cc->println("throw binpac::ExceptionOutOfBound(\"%s\",", data_id_str_.c_str());
+        out_cc->println("  %s, (%s) - (%s));", env->LValue(arraylength_var()), env->RValue(end_of_data),
+                        env->RValue(begin_of_data));
+        out_cc->dec_indent();
+        out_cc->println("}");
+
+        int element_size;
+
+        if ( elemtype_->StaticSize(env) == -1 ) {
+            // Check for overlong array quantity.  We cap it at the maximum
+            // array size (assume 1-byte elements * array length) as we can't
+            // possibly store more elements.  e.g. this helps prevent
+            // user-controlled length fields from causing an excessive
+            // iteration and/or memory-allocation (for the array we'll be
+            // parsing into) unless they actually sent enough data to go along
+            // with it.  Note that this check is *not* looking for whether the
+            // contents of the array will extend past the end of the data
+            // buffer.
+            out_cc->println("// Check array element quantity: %s", data_id_str_.c_str());
+            element_size = 1;
+        }
+        else {
+            // Boundary check the entire array if elements have static size.
+            out_cc->println("// Check bounds for static-size array: %s", data_id_str_.c_str());
+            elemtype_->SetBoundaryChecked();
+            element_size = elemtype_->StaticSize(env);
+
+            if ( element_size == 0 ) {
+                // If we know we have an array of empty elements, probably
+                // better to structure the parser as just a single empty
+                // field to avoid DoS vulnerability of allocating
+                // arbitrary number of empty records (i.e. cheap for them,
+                // but costly for us unless we have special optimization
+                // for this scenario to forgo the usual allocation).
+                throw Exception(this, "using an array of known-to-be-empty elements is possibly a bad idea");
+            }
+        }
+
+        const char* array_ptr_expr = data.ptr_expr();
+        string max_elements_available =
+            strfmt("((%s - %s) / %d)", env->RValue(end_of_data), array_ptr_expr, element_size);
+
+        out_cc->println("if ( %s > %s )", env->RValue(arraylength_var()), max_elements_available.c_str());
+        out_cc->inc_indent();
+        out_cc->println("throw binpac::ExceptionOutOfBound(\"%s\",", data_id_str_.c_str());
+        out_cc->println("  %s, (%s) - (%s));", env->RValue(arraylength_var()), env->RValue(end_of_data),
+                        array_ptr_expr);
+        out_cc->dec_indent();
+    }
+    else if ( attr_restofdata_ ) {
+        ASSERT(elemtype_->StaticSize(env) == 1);
+        out_cc->println("%s = (%s) - (%s);", env->LValue(arraylength_var()), env->RValue(end_of_data), data.ptr_expr());
+        env->SetEvaluated(arraylength_var());
+    }
+}
+
+void ArrayType::GenPubDecls(Output* out_h, Env* env) {
+    Type::GenPubDecls(out_h, env);
+
+    if ( declared_as_type() ) {
+        if ( attr_transient_ )
+            throw Exception(this, "cannot access element in &transient array");
+
+        out_h->println("int size() const	{ return %s ? %s->size() : 0; }", env->RValue(value_var()),
+                       env->RValue(value_var()));
+        out_h->println("%s operator[](int index) const { BINPAC_ASSERT(%s); return (*%s)[index]; }",
+                       elemtype_->DataTypeConstRefStr().c_str(), env->RValue(value_var()), env->RValue(value_var()));
+    }
+}
+
+void ArrayType::GenPrivDecls(Output* out_h, Env* env) {
+    ASSERT(elem_var_field_->type() == elemtype_);
+    ASSERT(elemtype_->value_var());
+    Type::GenPrivDecls(out_h, env);
+}
+
+void ArrayType::GenInitCode(Output* out_cc, Env* env) {
+    // Do not initiate the array here
+    // out_cc->println("%s = new %s;", lvalue(), vector_str_.c_str());
+    out_cc->println("%s = nullptr;", lvalue());
+
+    Type::GenInitCode(out_cc, env);
+    if ( incremental_parsing() ) {
+        out_cc->println("%s = -1;", env->LValue(elem_it_var()));
+    }
+}
+
+void ArrayType::GenCleanUpCode(Output* out_cc, Env* env) {
+    Type::GenCleanUpCode(out_cc, env);
+    if ( elemtype_->NeedsCleanUp() ) {
+        if ( ! elem_var_field_ ) {
+            ID* elem_var = new ID(strfmt("%s__elem", value_var()->Name()));
+            elem_var_field_ = new ParseVarField(Field::NOT_CLASS_MEMBER, elem_var, elemtype_);
+            elem_var_field_->Prepare(env);
+        }
+
+        out_cc->println("if ( %s ) {", env->RValue(value_var()));
+        out_cc->inc_indent();
+
+        out_cc->println("for ( auto* %s : *%s ) {", env->LValue(elem_var()), env->RValue(value_var()));
+        out_cc->inc_indent();
+        elemtype_->GenCleanUpCode(out_cc, env);
+        out_cc->dec_indent();
+        out_cc->println("}");
+
+        out_cc->dec_indent();
+        out_cc->println("}");
+    }
+    out_cc->println("delete %s;", lvalue());
+}
+
+string ArrayType::GenArrayInit(Output* out_cc, Env* env, bool known_array_length) {
+    string array_str;
+
+    array_str = lvalue();
+    if ( incremental_parsing() ) {
+        out_cc->println("if ( %s < 0 ) {", env->LValue(elem_it_var()));
+        out_cc->inc_indent();
+        out_cc->println("// Initialize only once");
+        out_cc->println("%s = 0;", env->LValue(elem_it_var()));
+    }
+
+    out_cc->println("%s = new %s;", lvalue(), vector_str_.c_str());
+
+    if ( known_array_length ) {
+        out_cc->println("%s->reserve(%s);", lvalue(), env->RValue(arraylength_var()));
+    }
+
+    if ( incremental_parsing() ) {
+        out_cc->dec_indent();
+        out_cc->println("}");
+    }
+
+    return array_str;
+}
+
+void ArrayType::GenElementAssignment(Output* out_cc, Env* env, string const& array_str, bool use_vector) {
+    if ( attr_transient_ ) {
+        // Just discard.
+        out_cc->println("delete %s;", env->LValue(elem_var()));
+        return;
+    }
+
+    // Assign the element
+    if ( ! use_vector ) {
+        out_cc->println("%s[%s] = %s;", array_str.c_str(), env->LValue(elem_it_var()), env->LValue(elem_var()));
+    }
+    else {
+        out_cc->println("%s->push_back(%s);", array_str.c_str(), env->LValue(elem_var()));
+    }
+}
+
+void ArrayType::DoGenParseCode(Output* out_cc, Env* env, const DataPtr& data, int flags) {
+    GenArrayLength(out_cc, env, data);
+
+    // Otherwise these variables are declared as member variables
+    if ( ! incremental_parsing() ) {
+        // Declare and initialize temporary variables
+        elem_var_field_->GenInitCode(out_cc, env);
+        elem_it_var_field_->GenTempDecls(out_cc, env);
+        out_cc->println("%s = 0;", env->LValue(elem_it_var()));
+        env->SetEvaluated(elem_it_var());
+    }
+
+    /*
+    If the input length can be determined without parsing
+    individual elements, generate the boundary checking before
+    parsing (unless in the case of incremental parsing).
+
+    There are two cases when the input length can be determined:
+    1. The array has a static size;
+    2. The array length can be computed before parsing and
+    each element is of constant size.
+    */
+
+    bool compute_size_var = false;
+
+    if ( incremental_input() ) {
+        // Do not compute size_var on incremental input
+        compute_size_var = false;
+
+        if ( ! incremental_parsing() &&
+             (StaticSize(env) >= 0 || (env->Evaluated(arraylength_var()) && elemtype_->StaticSize(env) >= 0)) ) {
+            GenBoundaryCheck(out_cc, env, data);
+        }
+    }
+    else {
+        compute_size_var = AddSizeVar(out_cc, env);
+    }
+
+    bool known_array_length = env->Evaluated(arraylength_var());
+    string array_str = GenArrayInit(out_cc, env, known_array_length);
+
+    bool use_vector = true;
+
+    ASSERT(elem_it_var());
+
+    DataPtr elem_data(env, nullptr, 0);
+
+    if ( elem_dataptr_var() ) {
+        out_cc->println("const_byteptr %s = %s;", env->LValue(elem_dataptr_var()), data.ptr_expr());
+        env->SetEvaluated(elem_dataptr_var());
+
+        elem_data = DataPtr(env, elem_dataptr_var(), 0);
+    }
+
+    string for_condition = known_array_length ?
+                               strfmt("%s < %s", env->LValue(elem_it_var()), env->RValue(arraylength_var())) :
+                               "/* forever */";
+
+    out_cc->println("for (; %s; ++%s) {", for_condition.c_str(), env->LValue(elem_it_var()));
+    out_cc->inc_indent();
+
+    if ( attr_generic_until_expr_ )
+        GenUntilCheck(out_cc, env, attr_generic_until_expr_, true);
+
+    if ( elem_dataptr_var() ) {
+        if ( length_ ) {
+            // Array has a known-length expression like uint16[4] vs. uint16[].
+            // Here, arriving at the end of the data buffer should not be a
+            // valid loop-termination condition (which is what the
+            // GenUntilCheck() call produces).  Instead, rely on the loop
+            // counter to terminate iteration or else the parsing code
+            // generated for each element should throw an OOB exception if
+            // there's insufficient data in the buffer.
+        }
+        else {
+            GenUntilCheck(out_cc, env, elem_dataptr_until_expr_, false);
+        }
+    }
+
+    elemtype_->GenPreParsing(out_cc, env);
+    elemtype_->GenParseCode(out_cc, env, elem_data, flags);
+
+    if ( incremental_parsing() ) {
+        out_cc->println("if ( ! %s )", elemtype_->parsing_complete(env).c_str());
+        out_cc->inc_indent();
+        out_cc->println("goto %s;", kNeedMoreData);
+        out_cc->dec_indent();
+    }
+
+    GenElementAssignment(out_cc, env, array_str, use_vector);
+
+    if ( elem_dataptr_var() ) {
+        out_cc->println("%s += %s;", env->LValue(elem_dataptr_var()),
+                        elemtype_->DataSize(nullptr, env, elem_data).c_str());
+        out_cc->println("BINPAC_ASSERT(%s <= %s);", env->RValue(elem_dataptr_var()), env->RValue(end_of_data));
+    }
+
+    if ( attr_until_element_expr_ )
+        GenUntilCheck(out_cc, env, attr_until_element_expr_, false);
+
+    if ( elemtype_->IsPointerType() )
+        out_cc->println("%s = nullptr;", env->LValue(elem_var()));
+
+    out_cc->dec_indent();
+    out_cc->println("}");
+
+    out_cc->dec_indent();
+    out_cc->println("%s: ;", end_of_array_loop_label_.c_str());
+    out_cc->inc_indent();
+
+    if ( compute_size_var && elem_dataptr_var() && ! env->Evaluated(size_var()) ) {
+        // Compute the data size
+        out_cc->println("%s = %s - (%s);", env->LValue(size_var()), env->RValue(elem_dataptr_var()), data.ptr_expr());
+        env->SetEvaluated(size_var());
+    }
+}
+
+void ArrayType::GenUntilInputCheck(Output* out_cc, Env* env) {
+    ID* elem_input_var_id = new ID(strfmt("%s__elem_input", value_var()->Name()));
+    elem_input_var_field_ = new TempVarField(elem_input_var_id, extern_type_const_bytestring->Clone());
+    elem_input_var_field_->Prepare(env);
+
+    out_cc->println("%s %s(%s, %s);", extern_type_const_bytestring->DataTypeStr().c_str(),
+                    env->LValue(elem_input_var()), env->RValue(begin_of_data), env->RValue(end_of_data));
+    env->SetEvaluated(elem_input_var());
+
+    GenUntilCheck(out_cc, env, attr_until_input_expr_, true);
+}
+
+void ArrayType::GenUntilCheck(Output* out_cc, Env* env, Expr* until_expr, bool delete_elem) {
+    ASSERT(until_expr);
+
+    Env check_env(env, this);
+    check_env.AddMacro(element_macro_id, new Expr(elem_var()->clone()));
+    if ( elem_input_var() ) {
+        check_env.AddMacro(input_macro_id, new Expr(elem_input_var()->clone()));
+    }
+
+    out_cc->println("// Check &until(%s)", until_expr->orig());
+    out_cc->println("if ( %s ) {", until_expr->EvalExpr(out_cc, &check_env));
+    out_cc->inc_indent();
+    if ( parsing_complete_var() ) {
+        out_cc->println("%s = true;", env->LValue(parsing_complete_var()));
+    }
+
+    if ( elemtype_->IsPointerType() ) {
+        if ( delete_elem )
+            elemtype_->GenCleanUpCode(out_cc, env);
+        else
+            out_cc->println("%s = nullptr;", env->LValue(elem_var()));
+    }
+
+    out_cc->println("goto %s;", end_of_array_loop_label_.c_str());
+    out_cc->dec_indent();
+    out_cc->println("}");
+}
+
+void ArrayType::GenDynamicSize(Output* out_cc, Env* env, const DataPtr& data) {
+    ASSERT(! incremental_input());
+    DEBUG_MSG("Generating dynamic size for array `%s'\n", value_var()->Name());
+
+    int elem_w = elemtype_->StaticSize(env);
+    if ( elem_w >= 0 && ! attr_until_element_expr_ && ! attr_until_input_expr_ && (length_ || attr_restofdata_) ) {
+        // If the elements have a fixed size,
+        // we only need to compute the number of elements
+        bool compute_size_var = AddSizeVar(out_cc, env);
+        ASSERT(compute_size_var);
+        GenArrayLength(out_cc, env, data);
+        ASSERT(env->Evaluated(arraylength_var()));
+        out_cc->println("%s = %d * %s;", env->LValue(size_var()), elem_w, env->RValue(arraylength_var()));
+        env->SetEvaluated(size_var());
+    }
+    else {
+        // Otherwise we need parse the array dynamically
+        GenParseCode(out_cc, env, data, 0);
+    }
+}
+
+int ArrayType::StaticSize(Env* env) const {
+    int num = 0;
+
+    if ( ! length_ || ! length_->ConstFold(env, &num) )
+        return -1;
+
+    int elem_w = elemtype_->StaticSize(env);
+    if ( elem_w < 0 )
+        return -1;
+
+    DEBUG_MSG("static size of %s:%s = %d * %d\n", decl_id()->Name(), lvalue(), elem_w, num);
+
+    return num * elem_w;
+}
+
+void ArrayType::SetBoundaryChecked() {
+    Type::SetBoundaryChecked();
+
+    if ( attr_length_expr_ ) {
+        // When using &length on an array, only treat its elements as
+        // already-bounds-checked if they are a single byte in length.
+        if ( elemtype_->StaticSize(env()) == 1 )
+            elemtype_->SetBoundaryChecked();
+
+        return;
+    }
+
+    elemtype_->SetBoundaryChecked();
+}
+
+void ArrayType::DoMarkIncrementalInput() { elemtype_->MarkIncrementalInput(); }
+
+bool ArrayType::RequiresAnalyzerContext() {
+    return Type::RequiresAnalyzerContext() || (length_ && length_->RequiresAnalyzerContext()) ||
+           elemtype_->RequiresAnalyzerContext();
+}
+
+bool ArrayType::DoTraverse(DataDepVisitor* visitor) {
+    if ( ! Type::DoTraverse(visitor) )
+        return false;
+
+    if ( length_ && ! length_->Traverse(visitor) )
+        return false;
+
+    if ( ! elemtype_->Traverse(visitor) )
+        return false;
+
+    return true;
+}

tools/binpac/src/pac_array.h

@@ -0,0 +1,86 @@
+#ifndef pac_array_h
+#define pac_array_h
+
+#include "pac_common.h"
+#include "pac_type.h"
+
+// Fixed-length array and variable length sequence with an ending pattern
+
+class ArrayType : public Type {
+public:
+    ArrayType(Type* arg_elemtype, Expr* arg_length = nullptr);
+    ~ArrayType() override;
+
+    bool DefineValueVar() const override;
+    string DataTypeStr() const override;
+    string DefaultValue() const override { return "0"; }
+    Type* ElementDataType() const override;
+
+    string EvalElement(const string& array, const string& index) const override;
+
+    void ProcessAttr(Attr* a) override;
+
+    void Prepare(Env* env, int flags) override;
+
+    void GenPubDecls(Output* out, Env* env) override;
+    void GenPrivDecls(Output* out, Env* env) override;
+
+    void GenInitCode(Output* out, Env* env) override;
+    void GenCleanUpCode(Output* out, Env* env) override;
+
+    int StaticSize(Env* env) const override;
+
+    void SetBoundaryChecked() override;
+    void GenUntilInputCheck(Output* out_cc, Env* env);
+
+    bool IsPointerType() const override { return true; }
+
+protected:
+    void init();
+
+    void DoGenParseCode(Output* out, Env* env, const DataPtr& data, int flags) override;
+    void GenDynamicSize(Output* out, Env* env, const DataPtr& data) override;
+    void GenArrayLength(Output* out_cc, Env* env, const DataPtr& data);
+    string GenArrayInit(Output* out_cc, Env* env, bool known_array_length);
+    void GenElementAssignment(Output* out_cc, Env* env, string const& array_str, bool use_vector);
+    void GenUntilCheck(Output* out_cc, Env* env, Expr* until_condition, bool delete_elem);
+
+    bool ByteOrderSensitive() const override { return elemtype_->RequiresByteOrder(); }
+    bool RequiresAnalyzerContext() override;
+
+    Type* DoClone() const override;
+
+    void DoMarkIncrementalInput() override;
+
+    const ID* arraylength_var() const;
+    const ID* elem_it_var() const;
+    const ID* elem_var() const;
+    const ID* elem_dataptr_var() const;
+    const ID* elem_input_var() const;
+
+protected:
+    bool DoTraverse(DataDepVisitor* visitor) override;
+
+private:
+    Type* elemtype_;
+    Expr* length_;
+
+    string vector_str_;
+    string datatype_str_;
+    string end_of_array_loop_label_;
+
+    Field* arraylength_var_field_;
+    Field* elem_it_var_field_;
+    Field* elem_var_field_;
+    Field* elem_dataptr_var_field_;
+    Field* elem_input_var_field_;
+
+    // This does not come from &until, but is internally generated
+    Expr* elem_dataptr_until_expr_;
+
+    Expr* attr_generic_until_expr_;
+    Expr* attr_until_element_expr_;
+    Expr* attr_until_input_expr_;
+};
+
+#endif // pac_array_h

tools/binpac/src/pac_attr.cc

@@ -0,0 +1,48 @@
+#include "pac_attr.h"
+
+#include "pac_expr.h"
+
+bool Attr::DoTraverse(DataDepVisitor* visitor) {
+    if ( expr_ && ! expr_->Traverse(visitor) )
+        return false;
+    return true;
+}
+
+bool Attr::RequiresAnalyzerContext() const { return (expr_ && expr_->RequiresAnalyzerContext()); }
+
+void Attr::init() {
+    expr_ = nullptr;
+    seqend_ = nullptr;
+    delete_expr_ = false;
+}
+
+Attr::Attr(AttrType type) : DataDepElement(DataDepElement::ATTR) {
+    type_ = type;
+    init();
+}
+
+Attr::Attr(AttrType type, Expr* expr) : DataDepElement(DataDepElement::ATTR) {
+    type_ = type;
+    init();
+    expr_ = expr;
+}
+
+Attr::Attr(AttrType type, ExprList* exprlist) : DataDepElement(DataDepElement::ATTR) {
+    type_ = type;
+    init();
+    expr_ = new Expr(exprlist);
+    delete_expr_ = true;
+}
+
+Attr::Attr(AttrType type, SeqEnd* seqend) : DataDepElement(DataDepElement::ATTR) {
+    type_ = type;
+    init();
+    seqend_ = seqend;
+}
+
+Attr::~Attr() {
+    if ( delete_expr_ )
+        delete expr_;
+}
+
+LetAttr::LetAttr(FieldList* letfields) : Attr(ATTR_LET) { letfields_ = letfields; }

tools/binpac/src/pac_attr.h

@@ -0,0 +1,63 @@
+#ifndef pac_attr_h
+#define pac_attr_h
+
+#include "pac_common.h"
+#include "pac_datadep.h"
+
+enum AttrType {
+    ATTR_BYTEORDER,
+    ATTR_CHECK,
+    ATTR_CHUNKED,
+    ATTR_ENFORCE,
+    ATTR_EXPORTSOURCEDATA,
+    ATTR_IF,
+    ATTR_LENGTH,
+    ATTR_LET,
+    ATTR_LINEBREAKER,
+    ATTR_MULTILINE,
+    ATTR_ONELINE,
+    ATTR_REFCOUNT,
+    ATTR_REQUIRES,
+    ATTR_RESTOFDATA,
+    ATTR_RESTOFFLOW,
+    ATTR_TRANSIENT,
+    ATTR_UNTIL,
+};
+
+class Attr : public Object, public DataDepElement {
+public:
+    Attr(AttrType type);
+    Attr(AttrType type, Expr* expr);
+    Attr(AttrType type, ExprList* exprlist);
+    Attr(AttrType type, SeqEnd* seqend);
+
+    ~Attr() override;
+
+    AttrType type() const { return type_; }
+    Expr* expr() const { return expr_; }
+    SeqEnd* seqend() const { return seqend_; }
+
+    bool RequiresAnalyzerContext() const;
+
+protected:
+    bool DoTraverse(DataDepVisitor* visitor) override;
+
+protected:
+    void init();
+
+    AttrType type_;
+    Expr* expr_;
+    SeqEnd* seqend_;
+    bool delete_expr_;
+};
+
+class LetAttr : public Attr {
+public:
+    LetAttr(FieldList* letfields);
+    FieldList* letfields() const { return letfields_; }
+
+private:
+    FieldList* letfields_;
+};
+
+#endif // pac_attr_h

tools/binpac/src/pac_btype.cc

@@ -0,0 +1,117 @@
+#include "pac_btype.h"
+
+#include "pac_dataptr.h"
+#include "pac_id.h"
+#include "pac_output.h"
+
+Type* BuiltInType::DoClone() const { return new BuiltInType(bit_type()); }
+
+bool BuiltInType::IsNumericType() const {
+    BITType t = bit_type();
+    return (t == INT8 || t == INT16 || t == INT32 || t == INT64 || t == UINT8 || t == UINT16 || t == UINT32 ||
+            t == UINT64);
+}
+
+bool BuiltInType::CompatibleBuiltInTypes(BuiltInType* type1, BuiltInType* type2) {
+    return type1->IsNumericType() && type2->IsNumericType();
+}
+
+static const char* basic_pactype_name[] = {
+#define TYPE_DEF(name, pactype, ctype, size) pactype,
+#include "pac_type.def"
+#undef TYPE_DEF
+    nullptr,
+};
+
+void BuiltInType::static_init() {
+    for ( int bit_type = 0; basic_pactype_name[bit_type]; ++bit_type ) {
+        Type::AddPredefinedType(basic_pactype_name[bit_type], new BuiltInType((BITType)bit_type));
+    }
+}
+
+int BuiltInType::LookUpByName(const char* name) {
+    ASSERT(0);
+    for ( int i = 0; basic_pactype_name[i]; ++i )
+        if ( strcmp(basic_pactype_name[i], name) == 0 )
+            return i;
+    return -1;
+}
+
+static const char* basic_ctype_name[] = {
+#define TYPE_DEF(name, pactype, ctype, size) ctype,
+#include "pac_type.def"
+#undef TYPE_DEF
+    nullptr,
+};
+
+bool BuiltInType::DefineValueVar() const { return bit_type_ != EMPTY; }
+
+string BuiltInType::DataTypeStr() const { return basic_ctype_name[bit_type_]; }
+
+int BuiltInType::StaticSize(Env* /* env */) const {
+    static const size_t basic_type_size[] = {
+#define TYPE_DEF(name, pactype, ctype, size) size,
+#include "pac_type.def"
+#undef TYPE_DEF
+    };
+
+    return basic_type_size[bit_type_];
+}
+
+void BuiltInType::DoMarkIncrementalInput() {
+    if ( bit_type_ == EMPTY )
+        return;
+    Type::DoMarkIncrementalInput();
+}
+
+void BuiltInType::GenInitCode(Output* out_cc, Env* env) {
+    if ( bit_type_ != EMPTY )
+        out_cc->println("%s = 0;", env->LValue(value_var()));
+    Type::GenInitCode(out_cc, env);
+}
+
+void BuiltInType::GenDynamicSize(Output* out_cc, Env* env, const DataPtr& data) {
+    /* should never be called */
+    ASSERT(0);
+}
+
+void BuiltInType::DoGenParseCode(Output* out_cc, Env* env, const DataPtr& data, int flags) {
+    if ( bit_type_ == EMPTY )
+        return;
+
+    // There is no need to generate the size variable
+    // out_cc->println("%s = sizeof(%s);", size_var(), DataTypeStr().c_str());
+
+    GenBoundaryCheck(out_cc, env, data);
+
+    if ( anonymous_value_var() )
+        return;
+
+    switch ( bit_type_ ) {
+        case EMPTY:
+            // do nothing
+            break;
+
+        case INT8:
+        case UINT8:
+            out_cc->println("%s = *((%s const*)(%s));", lvalue(), DataTypeStr().c_str(), data.ptr_expr());
+            break;
+        case INT16:
+        case UINT16:
+        case INT32:
+        case UINT32:
+        case INT64:
+        case UINT64:
+#if 0
+			out_cc->println("%s = UnMarshall<%s>(%s, %s);",
+				lvalue(),
+				DataTypeStr().c_str(),
+				data.ptr_expr(),
+				EvalByteOrder(out_cc, env).c_str());
+#else
+            out_cc->println("%s = FixByteOrder(%s, *((%s const*)(%s)));", lvalue(), EvalByteOrder(out_cc, env).c_str(),
+                            DataTypeStr().c_str(), data.ptr_expr());
+#endif
+            break;
+    }
+}

tools/binpac/src/pac_btype.h

@@ -0,0 +1,48 @@
+#ifndef pac_btype_h
+#define pac_btype_h
+
+#include "pac_type.h"
+
+class BuiltInType : public Type {
+public:
+    enum BITType {
+#define TYPE_DEF(name, pactype, ctype, size) name,
+#include "pac_type.def"
+#undef TYPE_DEF
+    };
+
+    static int LookUpByName(const char* name);
+
+    BuiltInType(BITType bit_type) : Type(bit_type == BuiltInType::EMPTY ? Type::EMPTY : BUILTIN), bit_type_(bit_type) {}
+
+    BITType bit_type() const { return bit_type_; }
+
+    bool IsNumericType() const override;
+
+    bool DefineValueVar() const override;
+    string DataTypeStr() const override;
+    string DefaultValue() const override { return "0"; }
+
+    int StaticSize(Env* env) const override;
+
+    bool IsPointerType() const override { return false; }
+
+    bool ByteOrderSensitive() const override { return StaticSize(0) >= 2; }
+
+    void GenInitCode(Output* out_cc, Env* env) override;
+
+    void DoMarkIncrementalInput() override;
+
+protected:
+    void DoGenParseCode(Output* out, Env* env, const DataPtr& data, int flags) override;
+    void GenDynamicSize(Output* out, Env* env, const DataPtr& data) override;
+    Type* DoClone() const override;
+
+    BITType bit_type_;
+
+public:
+    static void static_init();
+    static bool CompatibleBuiltInTypes(BuiltInType* type1, BuiltInType* type2);
+};
+
+#endif // pac_btype_h

tools/binpac/src/pac_case.cc

@@ -0,0 +1,404 @@
+#include "pac_case.h"
+
+#include <stdint.h>
+#include <limits>
+
+#include "pac_btype.h"
+#include "pac_exception.h"
+#include "pac_expr.h"
+#include "pac_exttype.h"
+#include "pac_id.h"
+#include "pac_output.h"
+#include "pac_typedecl.h"
+#include "pac_utils.h"
+
+CaseType::CaseType(Expr* index_expr, CaseFieldList* cases) : Type(CASE), index_expr_(index_expr), cases_(cases) {
+    index_var_ = nullptr;
+    foreach (i, CaseFieldList, cases_)
+        AddField(*i);
+}
+
+CaseType::~CaseType() {
+    delete index_var_;
+    delete index_expr_;
+    delete cases_;
+}
+
+void CaseType::AddCaseField(CaseField* f) {
+    // All fields must be added before Prepare()
+    ASSERT(! env());
+
+    AddField(f);
+    cases_->push_back(f);
+}
+
+bool CaseType::DefineValueVar() const { return false; }
+
+string CaseType::DataTypeStr() const {
+    ASSERT(type_decl());
+    return strfmt("%s*", type_decl()->class_name().c_str());
+}
+
+Type* CaseType::ValueType() const {
+    foreach (i, CaseFieldList, cases_) {
+        CaseField* c = *i;
+        return c->type();
+    }
+    ASSERT(0);
+    return nullptr;
+}
+
+string CaseType::DefaultValue() const { return ValueType()->DefaultValue(); }
+
+void CaseType::Prepare(Env* env, int flags) {
+    ASSERT(flags & TO_BE_PARSED);
+
+    index_var_ = new ID(strfmt("%s_case_index", value_var()->Name()));
+    // Unable to get the type for index_var_ at this moment, but we'll
+    // generate the right type based on index_expr_ later.
+    env->AddID(index_var_, MEMBER_VAR, nullptr);
+
+    // Sort the cases_ to put the default case at the end of the list
+    CaseFieldList::iterator default_case_it = cases_->end(); // to avoid warning
+    CaseField* default_case = nullptr;
+
+    foreach (i, CaseFieldList, cases_) {
+        CaseField* c = *i;
+        if ( ! c->index() ) {
+            if ( default_case )
+                throw Exception(c, "duplicate default case");
+            default_case_it = i;
+            default_case = c;
+        }
+    }
+    if ( default_case ) {
+        cases_->erase(default_case_it);
+        cases_->push_back(default_case);
+    }
+
+    foreach (i, CaseFieldList, cases_) {
+        CaseField* c = *i;
+        c->set_index_var(index_var_);
+        c->set_case_type(this);
+    }
+
+    Type::Prepare(env, flags);
+}
+
+void CaseType::GenPrivDecls(Output* out_h, Env* env) {
+    Type* t = index_expr_->DataType(env);
+
+    if ( t->tot() != Type::BUILTIN )
+        // It's a Type::EXTERN with a C++ type of "int", "bool", or "enum",
+        // any of which will convert consistently using an int as storage type.
+        t = extern_type_int;
+
+    out_h->println("%s %s;", t->DataTypeStr().c_str(), env->LValue(index_var_));
+    Type::GenPrivDecls(out_h, env);
+}
+
+void CaseType::GenPubDecls(Output* out_h, Env* env) {
+    Type* t = index_expr_->DataType(env);
+
+    if ( t->tot() != Type::BUILTIN )
+        t = extern_type_int;
+
+    out_h->println("%s %s const	{ return %s; }", t->DataTypeStr().c_str(), env->RValue(index_var_),
+                   env->LValue(index_var_));
+    Type::GenPubDecls(out_h, env);
+}
+
+void CaseType::GenInitCode(Output* out_cc, Env* env) {
+    out_cc->println("%s = -1;", env->LValue(index_var_));
+    Type::GenInitCode(out_cc, env);
+}
+
+void CaseType::GenCleanUpCode(Output* out_cc, Env* env) {
+    Type::GenCleanUpCode(out_cc, env);
+
+    env->set_in_branch(true);
+    out_cc->println("// NOLINTBEGIN(bugprone-branch-clone)");
+    out_cc->println("switch ( %s ) {", env->RValue(index_var_));
+    out_cc->inc_indent();
+    foreach (i, CaseFieldList, cases_) {
+        CaseField* c = *i;
+        c->GenCleanUpCode(out_cc, env);
+    }
+    out_cc->dec_indent();
+    out_cc->println("}");
+    out_cc->println("// NOLINTEND(bugprone-branch-clone)");
+    env->set_in_branch(false);
+}
+
+void CaseType::DoGenParseCode(Output* out_cc, Env* env, const DataPtr& data, int flags) {
+    if ( StaticSize(env) >= 0 )
+        GenBoundaryCheck(out_cc, env, data);
+
+    bool compute_size_var = false;
+
+    if ( ! incremental_input() )
+        compute_size_var = AddSizeVar(out_cc, env);
+
+    out_cc->println("%s = %s;", env->LValue(index_var_), index_expr_->EvalExpr(out_cc, env));
+    env->SetEvaluated(index_var_);
+
+    env->set_in_branch(true);
+    out_cc->println("// NOLINTBEGIN(bugprone-branch-clone)");
+    out_cc->println("switch ( %s ) {", env->RValue(index_var_));
+    out_cc->inc_indent();
+    bool has_default_case = false;
+    foreach (i, CaseFieldList, cases_) {
+        CaseField* c = *i;
+        c->GenParseCode(out_cc, env, data, compute_size_var ? size_var() : nullptr);
+        if ( c->IsDefaultCase() )
+            has_default_case = true;
+    }
+
+    if ( ! has_default_case ) {
+        out_cc->println("default:");
+        out_cc->inc_indent();
+        out_cc->println("throw binpac::ExceptionInvalidCaseIndex(\"%s\", (int64)%s);", decl_id()->Name(),
+                        env->RValue(index_var_));
+        out_cc->println("break;");
+        out_cc->dec_indent();
+    }
+    out_cc->dec_indent();
+    out_cc->println("}");
+    out_cc->println("// NOLINTEND(bugprone-branch-clone)");
+    env->set_in_branch(false);
+
+    if ( compute_size_var )
+        env->SetEvaluated(size_var());
+}
+
+void CaseType::GenDynamicSize(Output* out_cc, Env* env, const DataPtr& data) { GenParseCode(out_cc, env, data, 0); }
+
+int CaseType::StaticSize(Env* env) const {
+    int static_w = -1;
+    foreach (i, CaseFieldList, cases_) {
+        CaseField* c = *i;
+        int w = c->StaticSize(env);
+        if ( w < 0 || (static_w >= 0 && w != static_w) )
+            return -1;
+        static_w = w;
+    }
+    return static_w;
+}
+
+void CaseType::SetBoundaryChecked() {
+    Type::SetBoundaryChecked();
+    foreach (i, CaseFieldList, cases_) {
+        CaseField* c = *i;
+        c->SetBoundaryChecked();
+    }
+}
+
+void CaseType::DoMarkIncrementalInput() {
+    foreach (i, CaseFieldList, cases_) {
+        CaseField* c = *i;
+        c->type()->MarkIncrementalInput();
+    }
+}
+
+bool CaseType::ByteOrderSensitive() const {
+    foreach (i, CaseFieldList, cases_) {
+        CaseField* c = *i;
+        if ( c->RequiresByteOrder() )
+            return true;
+    }
+    return false;
+}
+
+CaseField::CaseField(ExprList* index, ID* id, Type* type)
+    : Field(CASE_FIELD, TYPE_TO_BE_PARSED | CLASS_MEMBER | PUBLIC_READABLE, id, type), index_(index) {
+    ASSERT(type_);
+    type_->set_value_var(id, MEMBER_VAR);
+    case_type_ = nullptr;
+    index_var_ = nullptr;
+}
+
+CaseField::~CaseField() { delete_list(ExprList, index_); }
+
+void GenCaseStr(ExprList* index_list, Output* out_cc, Env* env, Type* switch_type) {
+    if ( index_list ) {
+        foreach (i, ExprList, index_list) {
+            Expr* index_expr = *i;
+
+            Type* case_type = index_expr->DataType(env);
+
+            if ( case_type->tot() == Type::BUILTIN && case_type->StaticSize(env) > 4 )
+                throw ExceptionInvalidCaseSizeExpr(index_expr);
+
+            int index_const;
+
+            if ( ! index_expr->ConstFold(env, &index_const) )
+                throw ExceptionNonConstExpr(index_expr);
+
+            // External C++ types like "int", "bool", "enum"
+            // all use "int" type internally by default.
+            int case_type_width = 4;
+            int switch_type_width = 4;
+
+            if ( switch_type->tot() == Type::BUILTIN )
+                switch_type_width = switch_type->StaticSize(env);
+
+            if ( case_type->tot() == Type::BUILTIN )
+                case_type_width = case_type->StaticSize(env);
+
+            if ( case_type_width > switch_type_width ) {
+                BuiltInType* st = (BuiltInType*)switch_type;
+
+                if ( switch_type_width == 1 ) {
+                    if ( st->bit_type() == BuiltInType::INT8 ) {
+                        if ( index_const < std::numeric_limits<int8_t>::min() )
+                            throw ExceptionInvalidCaseLimitExpr(index_expr);
+                        if ( index_const > std::numeric_limits<int8_t>::max() )
+                            throw ExceptionInvalidCaseLimitExpr(index_expr);
+                    }
+                    else {
+                        if ( index_const < std::numeric_limits<uint8_t>::min() )
+                            throw ExceptionInvalidCaseLimitExpr(index_expr);
+                        if ( index_const > std::numeric_limits<uint8_t>::max() )
+                            throw ExceptionInvalidCaseLimitExpr(index_expr);
+                    }
+                }
+                else if ( switch_type_width == 2 ) {
+                    if ( st->bit_type() == BuiltInType::INT16 ) {
+                        if ( index_const < std::numeric_limits<int16_t>::min() )
+                            throw ExceptionInvalidCaseLimitExpr(index_expr);
+                        if ( index_const > std::numeric_limits<int16_t>::max() )
+                            throw ExceptionInvalidCaseLimitExpr(index_expr);
+                    }
+                    else {
+                        if ( index_const < std::numeric_limits<uint16_t>::min() )
+                            throw ExceptionInvalidCaseLimitExpr(index_expr);
+                        if ( index_const > std::numeric_limits<uint16_t>::max() )
+                            throw ExceptionInvalidCaseLimitExpr(index_expr);
+                    }
+                }
+                else {
+                    assert(0);
+                }
+            }
+
+            // We're always using "int" for storage, so ok to just
+            // cast into the type used by the switch statement since
+            // some unsafe stuff is already checked above.
+            if ( ! switch_type->IsBooleanType() )
+                out_cc->println("case ((%s)%d):", switch_type->DataTypeStr().c_str(), index_const);
+            else
+                out_cc->println("case %s:", index_const == 0 ? "false" : "true");
+        }
+    }
+    else {
+        out_cc->println("default:");
+    }
+}
+
+void CaseField::Prepare(Env* env) {
+    ASSERT(index_var_);
+    Field::Prepare(env);
+}
+
+void CaseField::GenPubDecls(Output* out_h, Env* env) {
+    if ( ! ((flags_ & PUBLIC_READABLE) && (flags_ & CLASS_MEMBER)) )
+        return;
+
+    // Skip type "empty"
+    if ( type_->DataTypeStr().empty() )
+        return;
+
+    out_h->println("%s %s const {", type_->DataTypeConstRefStr().c_str(), env->RValue(id_));
+    out_h->inc_indent();
+
+    if ( ! index_ )
+        out_h->println("return %s;", lvalue());
+    else {
+        out_h->println("// NOLINTBEGIN(bugprone-branch-clone)");
+        out_h->println("switch ( %s ) {", env->RValue(index_var_));
+        out_h->inc_indent();
+        GenCaseStr(index_, out_h, env, case_type()->IndexExpr()->DataType(env));
+        out_h->inc_indent();
+        out_h->println("break;  // OK");
+        out_h->dec_indent();
+
+        out_h->println("default:");
+        out_h->inc_indent();
+        out_h->println("throw binpac::ExceptionInvalidCase(\"%s\", (int64)%s, \"%s\");", id_->LocName(),
+                       env->RValue(index_var_), OrigExprList(index_).c_str());
+        out_h->println("break;");
+        out_h->dec_indent();
+
+        out_h->dec_indent();
+        out_h->println("}");
+        out_h->println("// NOLINTEND(bugprone-branch-clone)");
+
+        out_h->println("return %s;", lvalue());
+    }
+
+    out_h->dec_indent();
+    out_h->println("}");
+}
+
+void CaseField::GenInitCode(Output* out_cc, Env* env) {
+    // GenCaseStr(index_, out_cc, env);
+    // out_cc->inc_indent();
+    // out_cc->println("{");
+    // out_cc->println("// Initialize \"%s\"", id_->Name());
+    type_->GenInitCode(out_cc, env);
+    // out_cc->println("}");
+    // out_cc->println("break;");
+    // out_cc->dec_indent();
+}
+
+void CaseField::GenCleanUpCode(Output* out_cc, Env* env) {
+    GenCaseStr(index_, out_cc, env, case_type()->IndexExpr()->DataType(env));
+    out_cc->inc_indent();
+    out_cc->println("// Clean up \"%s\"", id_->Name());
+    if ( ! anonymous_field() ) {
+        out_cc->println("{");
+        out_cc->inc_indent();
+        type_->GenCleanUpCode(out_cc, env);
+        out_cc->dec_indent();
+        out_cc->println("}");
+    }
+    else
+        out_cc->println("{}");
+
+    out_cc->println("break;");
+    out_cc->dec_indent();
+}
+
+void CaseField::GenParseCode(Output* out_cc, Env* env, const DataPtr& data, const ID* size_var) {
+    GenCaseStr(index_, out_cc, env, case_type()->IndexExpr()->DataType(env));
+    out_cc->inc_indent();
+    out_cc->println("// Parse \"%s\"", id_->Name());
+    out_cc->println("{");
+    out_cc->inc_indent();
+
+    {
+        Env case_env(env, this);
+
+        type_->GenPreParsing(out_cc, &case_env);
+        type_->GenParseCode(out_cc, &case_env, data, 0);
+        if ( size_var ) {
+            out_cc->println("%s = %s;", case_env.LValue(size_var), type_->DataSize(out_cc, &case_env, data).c_str());
+        }
+        if ( type_->incremental_input() ) {
+            ASSERT(case_type()->parsing_complete_var());
+            out_cc->println("%s = %s;", case_env.LValue(case_type()->parsing_complete_var()),
+                            case_env.RValue(type_->parsing_complete_var()));
+        }
+    }
+
+    out_cc->dec_indent();
+    out_cc->println("}");
+    out_cc->println("break;");
+    out_cc->dec_indent();
+}
+
+bool CaseField::DoTraverse(DataDepVisitor* visitor) { return Field::DoTraverse(visitor) && type()->Traverse(visitor); }
+
+bool CaseField::RequiresAnalyzerContext() const {
+    return Field::RequiresAnalyzerContext() || type()->RequiresAnalyzerContext();
+}

tools/binpac/src/pac_case.h

@@ -0,0 +1,98 @@
+#ifndef pac_case_h
+#define pac_case_h
+
+#include "pac_common.h"
+#include "pac_field.h"
+#include "pac_id.h"
+#include "pac_type.h"
+
+class CaseType : public Type {
+public:
+    CaseType(Expr* index, CaseFieldList* cases);
+    ~CaseType() override;
+
+    void AddCaseField(CaseField* f);
+
+    bool DefineValueVar() const override;
+    string DataTypeStr() const override;
+    string DefaultValue() const override;
+
+    void Prepare(Env* env, int flags) override;
+
+    void GenPubDecls(Output* out, Env* env) override;
+    void GenPrivDecls(Output* out, Env* env) override;
+
+    void GenInitCode(Output* out, Env* env) override;
+    void GenCleanUpCode(Output* out, Env* env) override;
+
+    int StaticSize(Env* env) const override;
+
+    void SetBoundaryChecked() override;
+
+    Type* ValueType() const;
+
+    Expr* IndexExpr() const { return index_expr_; }
+
+    bool IsPointerType() const override { return ValueType()->IsPointerType(); }
+
+protected:
+    void DoGenParseCode(Output* out, Env* env, const DataPtr& data, int flags) override;
+    void GenDynamicSize(Output* out, Env* env, const DataPtr& data) override;
+    Type* DoClone() const override { return nullptr; }
+    void DoMarkIncrementalInput() override;
+
+    bool ByteOrderSensitive() const override;
+
+    Expr* index_expr_;
+    ID* index_var_;
+    CaseFieldList* cases_;
+
+    typedef map<const ID*, CaseField*, ID_ptr_cmp> member_map_t;
+    member_map_t member_map_;
+};
+
+class CaseField : public Field {
+public:
+    CaseField(ExprList* index, ID* id, Type* type);
+    ~CaseField() override;
+
+    CaseType* case_type() const { return case_type_; }
+    void set_case_type(CaseType* t) { case_type_ = t; }
+
+    ExprList* index() const { return index_; }
+
+    const char* lvalue() const { return type_->lvalue(); }
+
+    const char* CaseStr(Env* env);
+    void set_index_var(const ID* var) { index_var_ = var; }
+
+    void Prepare(Env* env) override;
+
+    void GenPubDecls(Output* out, Env* env) override;
+
+    void GenInitCode(Output* out, Env* env) override;
+    void GenCleanUpCode(Output* out, Env* env) override;
+    void GenParseCode(Output* out, Env* env, const DataPtr& data, const ID* size_var);
+
+    int StaticSize(Env* env) const { return type_->StaticSize(env); }
+
+    bool IsDefaultCase() const { return ! index_; }
+    void SetBoundaryChecked() { type_->SetBoundaryChecked(); }
+
+    bool RequiresByteOrder() const { return type_->RequiresByteOrder(); }
+    bool RequiresAnalyzerContext() const override;
+
+protected:
+    bool DoTraverse(DataDepVisitor* visitor) override;
+
+protected:
+    CaseType* case_type_;
+    ExprList* index_;
+    const ID* index_var_;
+};
+
+// Generate a list of "case X:" lines from index_list. Each index
+// expression must be constant foldable.
+void GenCaseStr(ExprList* index_list, Output* out_cc, Env* env, Type* switch_type);
+
+#endif // pac_case_h

tools/binpac/src/pac_cclass.h

@@ -0,0 +1,77 @@
+#ifndef pac_cclass_h
+#define pac_cclass_h
+
+class CClass;
+class CClassMember;
+class CClassMethod;
+class CType;
+class CVariable;
+
+typedef vector<CClassMember*> CClassMemberList;
+typedef vector<CClassMethod*> CClassMethodList;
+typedef vector<CVariable*> CVariableList;
+
+#include "pac_common.h"
+
+// Represents a C++ class.
+//
+// For now we adopt a simple model:
+//
+// 1. All members have a protected member variable "name_" and a
+// public constant access method "name()".
+//
+// 2. All methods are public.
+//
+// 3. We do not check repeated names.
+
+class CClass {
+public:
+    CClass(const string& class_name);
+
+    void AddMember(CClassMember* member);
+    void AddMethod(CClassMember* method);
+
+    void GenForwardDeclaration(Output* out_h);
+    void GenCode(Output* out_h, Output* out_cc);
+
+protected:
+    string class_name_;
+    CClassMemberList* members_;
+    CClassMethodList* methods_;
+};
+
+class CVariable {
+public:
+    CClassMember(const string& name, CType* type);
+
+    string name() const { return name_; }
+    CType* type() const { return type_; }
+
+protected:
+    string name_;
+    CType* type_;
+};
+
+class CClassMember {
+public:
+    CClassMember(CVariable* var);
+    void GenCode(Output* out_h, Output* out_cc);
+
+    string decl() const;
+
+protected:
+    CVariable* var_;
+};
+
+class CClassMethod {
+public:
+    CClassMethod(CVariable* var, CVariableList* params);
+
+    string decl() const;
+
+protected:
+    CVariable* var_;
+    CVariableList* params_;
+};
+
+#endif // pac_cclass_h

tools/binpac/src/pac_common.h

@@ -0,0 +1,131 @@
+#ifndef pac_common_h
+#define pac_common_h
+
+#include <ctype.h>
+#include <stdlib.h>
+#include <string.h>
+#include <vector>
+
+#include "pac_utils.h"
+
+using namespace std;
+
+extern bool FLAGS_pac_debug;
+extern bool FLAGS_quiet;
+extern vector<string> FLAGS_include_directories;
+extern string input_filename;
+extern int line_number;
+
+// Definition of class Object, which is the base class for all objects
+// representing language elements -- identifiers, types, expressions,
+// etc.
+
+class Object {
+public:
+    Object() {
+        filename = input_filename;
+        line_num = line_number;
+        location = strfmt("%s:%d", filename.c_str(), line_number);
+    }
+
+    ~Object() {}
+
+    const char* Location() const { return location.c_str(); }
+
+protected:
+    string filename;
+    int line_num;
+    string location;
+};
+
+class ActionParam;
+class ActionParamType;
+class AnalyzerAction;
+class AnalyzerContextDecl;
+class AnalyzerDecl;
+class AnalyzerElement;
+class ArrayType;
+class Attr;
+class CClass;
+class CType;
+class ConstString;
+class CaseExpr;
+class CaseField;
+class ContextField;
+class DataPtr;
+class Decl;
+class EmbeddedCode;
+class Enum;
+class Env;
+class ExternType;
+class Expr;
+class Field;
+class Function;
+class InputBuffer;
+class LetDef;
+class LetField;
+class ID;
+class Nullptr;
+class Number;
+class Output;
+class PacPrimitive;
+class Param;
+class ParameterizedType;
+class RecordType;
+class RecordField;
+class RecordDataField;
+class RecordPaddingField;
+class RegEx;
+class SeqEnd;
+class StateVar;
+class Type;
+class TypeDecl;
+class WithInputField;
+
+// The ID of the current declaration.
+extern const ID* current_decl_id;
+
+typedef vector<ActionParam*> ActionParamList;
+typedef vector<AnalyzerAction*> AnalyzerActionList;
+typedef vector<AnalyzerElement*> AnalyzerElementList;
+typedef vector<Attr*> AttrList;
+typedef vector<CaseExpr*> CaseExprList;
+typedef vector<CaseField*> CaseFieldList;
+typedef vector<ContextField*> ContextFieldList;
+typedef vector<Decl*> DeclList;
+typedef vector<Enum*> EnumList;
+typedef vector<Expr*> ExprList;
+typedef vector<Field*> FieldList;
+typedef vector<LetField*> LetFieldList;
+typedef vector<Number*> NumList;
+typedef vector<Param*> ParamList;
+typedef vector<RecordField*> RecordFieldList;
+typedef vector<StateVar*> StateVarList;
+
+#define foreach(i, ct, pc)                                                                                             \
+    if ( pc )                                                                                                          \
+        for ( ct::iterator i = (pc)->begin(); i != (pc)->end(); ++i )
+
+#define delete_list(ct, pc)                                                                                            \
+    {                                                                                                                  \
+        foreach (delete_list_i, ct, pc)                                                                                \
+            delete *delete_list_i;                                                                                     \
+        delete pc;                                                                                                     \
+        pc = 0;                                                                                                        \
+    }
+
+// Constants
+const char* const kComputeFrameLength = "compute_frame_length";
+const char* const kFlowBufferClass = "FlowBuffer";
+const char* const kFlowBufferVar = "flow_buffer";
+const char* const kFlowEOF = "FlowEOF";
+const char* const kFlowGap = "NewGap";
+const char* const kInitialBufferLengthFunc = "initial_buffer_length";
+const char* const kNeedMoreData = "need_more_data";
+const char* const kNewData = "NewData";
+const char* const kParseFuncWithBuffer = "ParseBuffer";
+const char* const kParseFuncWithoutBuffer = "Parse";
+const char* const kRefCountClass = "binpac::RefCount";
+const char* const kTypeWithLengthClass = "binpac::TypeWithLength";
+
+#endif // pac_common_h

tools/binpac/src/pac_conn.cc

@@ -0,0 +1,130 @@
+#include "pac_conn.h"
+
+#include "pac_analyzer.h"
+#include "pac_dataunit.h"
+#include "pac_embedded.h"
+#include "pac_exception.h"
+#include "pac_expr.h"
+#include "pac_flow.h"
+#include "pac_output.h"
+#include "pac_paramtype.h"
+#include "pac_type.h"
+
+ConnDecl::ConnDecl(ID* conn_id, ParamList* params, AnalyzerElementList* elemlist)
+    : AnalyzerDecl(conn_id, CONN, params) {
+    flows_[0] = flows_[1] = nullptr;
+    AddElements(elemlist);
+    data_type_ = new ParameterizedType(conn_id->clone(), nullptr);
+}
+
+ConnDecl::~ConnDecl() {
+    delete flows_[0];
+    delete flows_[1];
+    delete data_type_;
+}
+
+void ConnDecl::AddBaseClass(vector<string>* base_classes) const {
+    base_classes->push_back("binpac::ConnectionAnalyzer");
+}
+
+void ConnDecl::ProcessFlowElement(AnalyzerFlow* flow_elem) {
+    int flow_index;
+
+    if ( flow_elem->dir() == AnalyzerFlow::UP )
+        flow_index = 0;
+    else
+        flow_index = 1;
+
+    if ( flows_[flow_index] ) {
+        throw Exception(flow_elem, strfmt("%sflow already defined", flow_index == 0 ? "up" : "down"));
+    }
+
+    flows_[flow_index] = flow_elem;
+    type_->AddField(flow_elem->flow_field());
+}
+
+void ConnDecl::ProcessDataUnitElement(AnalyzerDataUnit* dataunit_elem) {
+    throw Exception(dataunit_elem, "dataunit should be defined in only a flow declaration");
+}
+
+void ConnDecl::Prepare() {
+    AnalyzerDecl::Prepare();
+
+    flows_[0]->flow_decl()->set_conn_decl(this);
+    flows_[1]->flow_decl()->set_conn_decl(this);
+}
+
+void ConnDecl::GenPubDecls(Output* out_h, Output* out_cc) { AnalyzerDecl::GenPubDecls(out_h, out_cc); }
+
+void ConnDecl::GenPrivDecls(Output* out_h, Output* out_cc) { AnalyzerDecl::GenPrivDecls(out_h, out_cc); }
+
+void ConnDecl::GenEOFFunc(Output* out_h, Output* out_cc) {
+    string proto = strfmt("%s(bool is_orig)", kFlowEOF);
+
+    out_h->println("void %s;", proto.c_str());
+
+    out_cc->println("void %s::%s {", class_name().c_str(), proto.c_str());
+    out_cc->inc_indent();
+
+    out_cc->println("if ( is_orig )");
+    out_cc->inc_indent();
+    out_cc->println("%s->%s();", env_->LValue(upflow_id), kFlowEOF);
+    out_cc->dec_indent();
+    out_cc->println("else");
+    out_cc->inc_indent();
+    out_cc->println("%s->%s();", env_->LValue(downflow_id), kFlowEOF);
+
+    foreach (i, AnalyzerHelperList, eof_helpers_) {
+        (*i)->GenCode(nullptr, out_cc, this);
+    }
+
+    out_cc->dec_indent();
+
+    out_cc->dec_indent();
+    out_cc->println("}");
+    out_cc->println("");
+}
+
+void ConnDecl::GenGapFunc(Output* out_h, Output* out_cc) {
+    string proto = strfmt("%s(bool is_orig, int gap_length)", kFlowGap);
+
+    out_h->println("void %s;", proto.c_str());
+
+    out_cc->println("void %s::%s {", class_name().c_str(), proto.c_str());
+    out_cc->inc_indent();
+
+    out_cc->println("if ( is_orig )");
+    out_cc->inc_indent();
+    out_cc->println("%s->%s(gap_length);", env_->LValue(upflow_id), kFlowGap);
+    out_cc->dec_indent();
+    out_cc->println("else");
+    out_cc->inc_indent();
+    out_cc->println("%s->%s(gap_length);", env_->LValue(downflow_id), kFlowGap);
+    out_cc->dec_indent();
+
+    out_cc->dec_indent();
+    out_cc->println("}");
+    out_cc->println("");
+}
+
+void ConnDecl::GenProcessFunc(Output* out_h, Output* out_cc) {
+    string proto = strfmt("%s(bool is_orig, const_byteptr begin, const_byteptr end)", kNewData);
+
+    out_h->println("void %s override;", proto.c_str());
+
+    out_cc->println("void %s::%s {", class_name().c_str(), proto.c_str());
+    out_cc->inc_indent();
+
+    out_cc->println("if ( is_orig )");
+    out_cc->inc_indent();
+    out_cc->println("%s->%s(begin, end);", env_->LValue(upflow_id), kNewData);
+    out_cc->dec_indent();
+    out_cc->println("else");
+    out_cc->inc_indent();
+    out_cc->println("%s->%s(begin, end);", env_->LValue(downflow_id), kNewData);
+    out_cc->dec_indent();
+
+    out_cc->dec_indent();
+    out_cc->println("}");
+    out_cc->println("");
+}

tools/binpac/src/pac_conn.h

@@ -0,0 +1,33 @@
+#ifndef pac_conn_h
+#define pac_conn_h
+
+#include "pac_analyzer.h"
+#include "pac_decl.h"
+
+class ConnDecl : public AnalyzerDecl {
+public:
+    ConnDecl(ID* conn_id, ParamList* params, AnalyzerElementList* elemlist);
+    ~ConnDecl() override;
+
+    void Prepare() override;
+
+    Type* DataType() const { return data_type_; }
+
+protected:
+    void AddBaseClass(vector<string>* base_classes) const override;
+
+    void GenProcessFunc(Output* out_h, Output* out_cc) override;
+    void GenGapFunc(Output* out_h, Output* out_cc) override;
+    void GenEOFFunc(Output* out_h, Output* out_cc) override;
+
+    void GenPubDecls(Output* out_h, Output* out_cc) override;
+    void GenPrivDecls(Output* out_h, Output* out_cc) override;
+
+    void ProcessFlowElement(AnalyzerFlow* flow_elem) override;
+    void ProcessDataUnitElement(AnalyzerDataUnit* dataunit_elem) override;
+
+    AnalyzerFlow* flows_[2];
+    Type* data_type_;
+};
+
+#endif // pac_conn_h

tools/binpac/src/pac_context.cc

@@ -0,0 +1,94 @@
+#include "pac_context.h"
+
+#include "pac_analyzer.h"
+#include "pac_exception.h"
+#include "pac_exttype.h"
+#include "pac_flow.h"
+#include "pac_id.h"
+#include "pac_output.h"
+#include "pac_param.h"
+#include "pac_paramtype.h"
+#include "pac_type.h"
+#include "pac_utils.h"
+
+ContextField::ContextField(ID* id, Type* type)
+    : Field(CONTEXT_FIELD, TYPE_NOT_TO_BE_PARSED | CLASS_MEMBER | PUBLIC_READABLE, id, type) {}
+
+AnalyzerContextDecl* AnalyzerContextDecl::current_analyzer_context_ = nullptr;
+
+namespace {
+ParamList* ContextFieldsToParams(ContextFieldList* context_fields) {
+    // Convert context fields to parameters
+    ParamList* params = new ParamList();
+    foreach (i, ContextFieldList, context_fields) {
+        ContextField* f = *i;
+        params->push_back(new Param(f->id()->clone(), f->type()));
+    }
+    return params;
+}
+} // namespace
+
+AnalyzerContextDecl::AnalyzerContextDecl(ID* id, ContextFieldList* context_fields)
+    : TypeDecl(new ID(strfmt("Context%s", id->Name())), ContextFieldsToParams(context_fields), new DummyType()) {
+    context_name_id_ = id;
+    if ( current_analyzer_context_ != nullptr ) {
+        throw Exception(this, strfmt("multiple declaration of analyzer context; "
+                                     "the previous one is `%s'",
+                                     current_analyzer_context_->id()->Name()));
+    }
+    else
+        current_analyzer_context_ = this;
+
+    context_fields_ = context_fields;
+
+    param_type_ = new ParameterizedType(id_->clone(), nullptr);
+
+    flow_buffer_added_ = false;
+
+    DEBUG_MSG("Context type: %s\n", param_type()->class_name().c_str());
+}
+
+AnalyzerContextDecl::~AnalyzerContextDecl() {
+    delete context_name_id_;
+    delete param_type_;
+    delete_list(ContextFieldList, context_fields_);
+}
+
+void AnalyzerContextDecl::GenForwardDeclaration(Output* out_h) {
+    GenNamespaceBegin(out_h);
+    TypeDecl::GenForwardDeclaration(out_h);
+}
+
+void AnalyzerContextDecl::GenCode(Output* out_h, Output* out_cc) {
+    GenNamespaceBegin(out_h);
+    GenNamespaceBegin(out_cc);
+    TypeDecl::GenCode(out_h, out_cc);
+}
+
+void AnalyzerContextDecl::GenNamespaceBegin(Output* out) const {
+    out->println("namespace %s {", context_name_id()->Name());
+}
+
+void AnalyzerContextDecl::GenNamespaceEnd(Output* out) const {
+    out->println("} // namespace %s", context_name_id()->Name());
+}
+
+void AnalyzerContextDecl::AddFlowBuffer() {
+    if ( flow_buffer_added_ )
+        return;
+
+    AddParam(new Param(new ID(kFlowBufferVar), FlowDecl::flow_buffer_type()->Clone()));
+
+    flow_buffer_added_ = true;
+}
+
+string AnalyzerContextDecl::mb_buffer(Env* env) {
+    // A hack. The orthodox way would be to build an Expr of
+    // context.flow_buffer_var, and then EvalExpr.
+    return strfmt("%s->%s()", env->RValue(analyzer_context_id), kFlowBufferVar);
+}
+
+Type* DummyType::DoClone() const {
+    // Fields will be copied in Type::Clone().
+    return new DummyType();
+}

tools/binpac/src/pac_context.h

@@ -0,0 +1,97 @@
+#ifndef pac_context_h
+#define pac_context_h
+
+#include "pac_common.h"
+#include "pac_field.h"
+#include "pac_type.h"
+#include "pac_typedecl.h"
+
+// AnalyzerContext represents a cookie that an analyzer gives to
+// parse functions of various message types. The cookie is parsed
+// to every parse function (if necessary) as parameter 'binpac_context'.
+//
+// The members of the cookie is declared through 'analyzer' declarations,
+// such as in:
+//
+// analyzer SunRPC withcontext {
+//        connection:     RPC_Conn;
+//        flow:           RPC_Flow;
+// };
+//
+// The cookie usually contains the connection and flow in which
+// the message appears, and the context information can be
+// accessed as members of the cookie, such as
+// ``binpac_context.connection''.
+
+class ContextField : public Field {
+public:
+    ContextField(ID* id, Type* type);
+};
+
+class AnalyzerContextDecl : public TypeDecl {
+public:
+    AnalyzerContextDecl(ID* id, ContextFieldList* context_fields);
+    ~AnalyzerContextDecl() override;
+
+    void AddFlowBuffer();
+
+    const ID* context_name_id() const { return context_name_id_; }
+
+    // The type of analyzer context as a parameter
+    ParameterizedType* param_type() const { return param_type_; }
+
+    void GenForwardDeclaration(Output* out_h) override;
+    void GenCode(Output* out_h, Output* out_cc) override;
+
+    void GenNamespaceBegin(Output* out) const;
+    void GenNamespaceEnd(Output* out) const;
+
+private:
+    ID* context_name_id_;
+    ContextFieldList* context_fields_;
+    ParameterizedType* param_type_;
+    bool flow_buffer_added_;
+
+    // static members
+public:
+    static AnalyzerContextDecl* current_analyzer_context() { return current_analyzer_context_; }
+
+    static string mb_buffer(Env* env);
+
+private:
+    static AnalyzerContextDecl* current_analyzer_context_;
+};
+
+class DummyType : public Type {
+public:
+    DummyType() : Type(DUMMY) {}
+
+    bool DefineValueVar() const override { return false; }
+    string DataTypeStr() const override {
+        ASSERT(0);
+        return "";
+    }
+
+    int StaticSize(Env* env) const override {
+        ASSERT(0);
+        return -1;
+    }
+
+    bool ByteOrderSensitive() const override { return false; }
+
+    bool IsPointerType() const override {
+        ASSERT(0);
+        return false;
+    }
+
+    void DoGenParseCode(Output* out, Env* env, const DataPtr& data, int flags) override { ASSERT(0); }
+
+    // Generate code for computing the dynamic size of the type
+    void GenDynamicSize(Output* out, Env* env, const DataPtr& data) override { ASSERT(0); }
+
+protected:
+    Type* DoClone() const override;
+    void DoMarkIncrementalInput() override { ASSERT(0); }
+};
+
+#endif // pac_context_h

tools/binpac/src/pac_cstr.cc

@@ -0,0 +1,110 @@
+#include "pac_cstr.h"
+
+#include "pac_dbg.h"
+#include "pac_exception.h"
+
+namespace {
+
+class EscapeException {
+public:
+    explicit EscapeException(const string& s) { msg_ = s; }
+
+    const string& msg() const { return msg_; }
+
+private:
+    string msg_;
+};
+
+// Copied from util.cc of Zeek
+int expand_escape(const char*& s) {
+    switch ( *(s++) ) {
+        case 'b': return '\b';
+        case 'f': return '\f';
+        case 'n': return '\n';
+        case 'r': return '\r';
+        case 't': return '\t';
+        case 'a': return '\a';
+        case 'v': return '\v';
+
+        case '0':
+        case '1':
+        case '2':
+        case '3':
+        case '4':
+        case '5':
+        case '6':
+        case '7': { // \<octal>{1,3}
+            --s;    // put back the first octal digit
+            const char* start = s;
+
+            // Don't increment inside loop control
+            // because if isdigit() is a macro it might
+            // expand into multiple increments ...
+
+            // Here we define a maximum length for escape sequence
+            // to allow easy handling of string like: "^H0" as
+            // "\0100".
+
+            for ( int len = 0; len < 3 && isascii(*s) && isdigit(*s); ++s, ++len )
+                ;
+
+            int result;
+            if ( sscanf(start, "%3o", &result) != 1 )
+                throw EscapeException(strfmt("bad octal escape: \"%s", start));
+
+            return result;
+        }
+
+        case 'x': { /* \x<hex> */
+            const char* start = s;
+
+            // Look at most 2 characters, so that "\x0ddir" -> "^Mdir".
+            for ( int len = 0; len < 2 && isascii(*s) && isxdigit(*s); ++s, ++len )
+                ;
+
+            int result;
+            if ( sscanf(start, "%2x", &result) != 1 )
+                throw EscapeException(strfmt("bad hexadecimal escape: \"%s", start));
+
+            return result;
+        }
+
+        default: return s[-1];
+    }
+}
+
+} // namespace
+
+ConstString::ConstString(const string& s) : str_(s) {
+    // Copied from scan.l of Zeek
+    try {
+        const char* text = str_.c_str();
+        int len = strlen(text) + 1;
+        int i = 0;
+
+        char* new_s = new char[len];
+
+        // Skip leading quote.
+        for ( ++text; *text; ++text ) {
+            if ( *text == '\\' ) {
+                ++text; // skip '\'
+                new_s[i++] = expand_escape(text);
+                --text; // point to end of sequence
+            }
+            else {
+                new_s[i++] = *text;
+            }
+        }
+        ASSERT(i < len);
+
+        // Get rid of trailing quote.
+        ASSERT(new_s[i - 1] == '"');
+        new_s[i - 1] = '\0';
+
+        unescaped_ = new_s;
+        delete[] new_s;
+    } catch ( EscapeException const& e ) {
+        // Throw again with the object
+        throw Exception(this, e.msg().c_str());
+    }
+}

tools/binpac/src/pac_cstr.h

@@ -0,0 +1,22 @@
+#ifndef pac_cstr_h
+#define pac_cstr_h
+
+#include "pac_common.h"
+
+class ConstString : public Object {
+public:
+    ConstString(const string& s);
+
+    // The string in its escaped form, with surrounding '"'s
+    const string& str() const { return str_; }
+    const char* c_str() const { return str_.c_str(); }
+
+    // The unescaped string, without surrounding '"'s
+    const string& unescaped() const { return unescaped_; }
+
+private:
+    string str_;
+    string unescaped_;
+};
+
+#endif // pac_cstr_h

tools/binpac/src/pac_ctype.cc

@@ -0,0 +1,13 @@
+#include "pac_ctype.h"
+
+string CType::DeclareInstance(const string& var) const { return strfmt("%s %s", name().c_str(), var.c_str()); }
+
+string CType::DeclareConstReference(const string& var) const {
+    return strfmt("%s const& %s", name().c_str(), var.c_str());
+}
+
+string CType::DeclareConstPointer(const string& var) const {
+    return strfmt("%s const* %s", name().c_str(), var.c_str());
+}
+
+string CType::DeclarePointer(const string& var) const { return strfmt("%s* %s", name().c_str(), var.c_str()); }

tools/binpac/src/pac_ctype.h

@@ -0,0 +1,22 @@
+#ifndef pac_ctype_h
+#define pac_ctype_h
+
+#include "pac_common.h"
+
+// Represents a C++ type
+class CType {
+public:
+    CType(const string& name);
+
+    string name() const { return name_; }
+
+    string DeclareInstance(const string& var) const;
+    string DeclareConstReference(const string& var) const;
+    string DeclareConstPointer(const string& var) const;
+    string DeclarePointer(const string& var) const;
+
+protected:
+    string name_;
+};
+
+#endif // pac_ctype_h

tools/binpac/src/pac_datadep.cc

@@ -0,0 +1,56 @@
+#include "pac_datadep.h"
+
+#include "pac_expr.h"
+#include "pac_id.h"
+#include "pac_type.h"
+
+DataDepElement::DataDepElement(DDE_Type type) : dde_type_(type), in_traversal(false) {}
+
+bool DataDepElement::Traverse(DataDepVisitor* visitor) {
+    // Avoid infinite loop
+    if ( in_traversal )
+        return true;
+    if ( ! visitor->PreProcess(this) )
+        return false;
+
+    in_traversal = true;
+    bool cont = DoTraverse(visitor);
+    in_traversal = false;
+
+    if ( ! cont )
+        return false;
+    if ( ! visitor->PostProcess(this) )
+        return false;
+    return true;
+}
+
+Expr* DataDepElement::expr() { return static_cast<Expr*>(this); }
+
+Type* DataDepElement::type() { return static_cast<Type*>(this); }
+
+bool RequiresAnalyzerContext::PreProcess(DataDepElement* element) {
+    switch ( element->dde_type() ) {
+        case DataDepElement::EXPR: ProcessExpr(element->expr()); break;
+        default: break;
+    }
+
+    // Continue traversal until we know the answer is 'yes'
+    return ! requires_analyzer_context_;
+}
+
+bool RequiresAnalyzerContext::PostProcess(DataDepElement* element) { return ! requires_analyzer_context_; }
+
+void RequiresAnalyzerContext::ProcessExpr(Expr* expr) {
+    if ( expr->expr_type() == Expr::EXPR_ID ) {
+        requires_analyzer_context_ =
+            (requires_analyzer_context_ || *expr->id() == *analyzer_context_id || *expr->id() == *context_macro_id);
+    }
+}
+
+bool RequiresAnalyzerContext::compute(DataDepElement* element) {
+    RequiresAnalyzerContext visitor;
+    // This result is intentionally ignored. We want to traverse, but always return
+    // the same result.
+    std::ignore = element->Traverse(&visitor);
+    return visitor.requires_analyzer_context_;
+}

tools/binpac/src/pac_datadep.h

@@ -0,0 +1,68 @@
+#ifndef pac_datadep_h
+#define pac_datadep_h
+
+// To provide a way to traverse through the data dependency graph.
+// That is, to evaluate X, what must be evaluated.
+
+#include "pac_common.h"
+#include "pac_dbg.h"
+
+class DataDepVisitor;
+
+class DataDepElement {
+public:
+    enum DDE_Type {
+        ATTR,
+        CASEEXPR,
+        EXPR,
+        FIELD,
+        INPUT_BUFFER,
+        PARAM,
+        TYPE,
+    };
+
+    DataDepElement(DDE_Type type);
+    virtual ~DataDepElement() {}
+
+    // Returns whether to continue traversal
+    bool Traverse(DataDepVisitor* visitor);
+
+    // Returns whether to continue traversal
+    virtual bool DoTraverse(DataDepVisitor* visitor) = 0;
+
+    DDE_Type dde_type() const { return dde_type_; }
+    Expr* expr();
+    Type* type();
+
+protected:
+    DDE_Type dde_type_;
+    bool in_traversal;
+};
+
+class DataDepVisitor {
+public:
+    virtual ~DataDepVisitor() {}
+    // Returns whether to continue traversal
+    virtual bool PreProcess(DataDepElement* element) = 0;
+    virtual bool PostProcess(DataDepElement* element) = 0;
+};
+
+class RequiresAnalyzerContext : public DataDepVisitor {
+public:
+    RequiresAnalyzerContext() : requires_analyzer_context_(false) {}
+
+    // Returns whether to continue traversal
+    bool PreProcess(DataDepElement* element) override;
+    bool PostProcess(DataDepElement* element) override;
+
+    bool requires_analyzer_context() const { return requires_analyzer_context_; }
+
+    static bool compute(DataDepElement* element);
+
+protected:
+    void ProcessExpr(Expr* expr);
+
+    bool requires_analyzer_context_;
+};
+
+#endif // pac_datadep_h

tools/binpac/src/pac_dataptr.cc

@@ -0,0 +1,51 @@
+#include "pac_dataptr.h"
+
+#include "pac_exception.h"
+#include "pac_id.h"
+#include "pac_output.h"
+#include "pac_utils.h"
+
+DataPtr::DataPtr(Env* env, const ID* id, const int offset) : id_(id), offset_(offset) {
+    if ( id_ ) {
+        if ( ! env->Evaluated(id_) )
+            throw ExceptionIDNotEvaluated(id_);
+
+        if ( offset_ == 0 )
+            ptr_expr_ = strfmt("%s", env->RValue(id_));
+        else
+            ptr_expr_ = strfmt("(%s + %d)", env->RValue(id_), offset_);
+    }
+    else
+        ptr_expr_ = "(null id)";
+}
+
+int DataPtr::AbsOffset(const ID* base_ptr) const { return (id() == base_ptr) ? offset() : -1; }
+
+char* DataPtr::AbsOffsetExpr(Env* env, const ID* base_ptr) const {
+    if ( AbsOffset(base_ptr) >= 0 )
+        return nfmt("%d", offset());
+    else
+        return nfmt("(%s - %s)", ptr_expr(), env->RValue(base_ptr));
+}
+
+void DataPtr::GenBoundaryCheck(Output* out_cc, Env* env, const char* data_size, const char* data_name) const {
+    ASSERT(id_);
+
+    out_cc->println("// Checking out-of-bound for \"%s\"", data_name);
+    out_cc->println("if ( %s + (%s) > %s || %s + (%s) < %s ) {", ptr_expr(), data_size, env->RValue(end_of_data),
+                    ptr_expr(), data_size, ptr_expr());
+
+    out_cc->inc_indent();
+
+    char* data_offset = AbsOffsetExpr(env, begin_of_data);
+
+    out_cc->println("// Handle out-of-bound condition");
+    out_cc->println("throw binpac::ExceptionOutOfBound(\"%s\",", data_name);
+    out_cc->println("	(%s) + (%s), ", data_offset, data_size);
+    out_cc->println("	(%s) - (%s));", env->RValue(end_of_data), env->RValue(begin_of_data));
+
+    delete[] data_offset;
+
+    out_cc->dec_indent();
+    out_cc->println("}");
+}

tools/binpac/src/pac_dataptr.h

@@ -0,0 +1,44 @@
+#ifndef pac_dataptr_h
+#define pac_dataptr_h
+
+#include <string>
+
+#include "pac_common.h"
+#include "pac_dbg.h"
+
+// A data pointer is represented by an data pointer variable
+// plus a constant offset.
+
+class DataPtr {
+public:
+    DataPtr(Env* env, const ID* arg_id, const int arg_off);
+    DataPtr(DataPtr const& x) { *this = x; }
+
+    DataPtr const& operator=(DataPtr const& x) {
+        id_ = x.id();
+        offset_ = x.offset();
+        ptr_expr_ = x.ptr_expr();
+
+        return *this;
+    }
+
+    const ID* id() const { return id_; }
+    int offset() const { return offset_; }
+
+    const char* ptr_expr() const {
+        ASSERT(id_);
+        return ptr_expr_.c_str();
+    }
+
+    int AbsOffset(const ID* base_ptr) const;
+    char* AbsOffsetExpr(Env* env, const ID* base_ptr) const;
+
+    void GenBoundaryCheck(Output* out, Env* env, const char* data_size, const char* data_name) const;
+
+protected:
+    const ID* id_;
+    int offset_;
+    string ptr_expr_;
+};
+
+#endif // pac_dataptr_h

tools/binpac/src/pac_dataunit.cc

@@ -0,0 +1,37 @@
+#include "pac_dataunit.h"
+
+#include "pac_context.h"
+#include "pac_output.h"
+#include "pac_paramtype.h"
+#include "pac_varfield.h"
+
+AnalyzerDataUnit::AnalyzerDataUnit(DataUnitType type, ID* id, ExprList* type_params, ExprList* context_params)
+    : AnalyzerElement(DATAUNIT), type_(type), id_(id), type_params_(type_params), context_params_(context_params) {
+    data_type_ = new ParameterizedType(id_, type_params_);
+    context_type_ =
+        new ParameterizedType(AnalyzerContextDecl::current_analyzer_context()->id()->clone(), context_params_);
+
+    dataunit_var_field_ = new ParseVarField(Field::CLASS_MEMBER, dataunit_id->clone(), data_type());
+    context_var_field_ = new PrivVarField(analyzer_context_id->clone(), context_type());
+}
+
+AnalyzerDataUnit::~AnalyzerDataUnit() {
+    delete dataunit_var_field_;
+    delete context_var_field_;
+}
+
+void AnalyzerDataUnit::Prepare(Env* env) {
+    dataunit_var_field_->Prepare(env);
+    context_var_field_->Prepare(env);
+}
+
+void AnalyzerDataUnit::GenNewDataUnit(Output* out_cc, Env* env) {
+    out_cc->println("%s = new %s(%s);", env->LValue(dataunit_id), data_type()->class_name().c_str(),
+                    data_type()->EvalParameters(out_cc, env).c_str());
+}
+
+void AnalyzerDataUnit::GenNewContext(Output* out_cc, Env* env) {
+    out_cc->println("%s = new %s(%s);", env->LValue(analyzer_context_id), context_type()->class_name().c_str(),
+                    context_type()->EvalParameters(out_cc, env).c_str());
+    env->SetEvaluated(analyzer_context_id);
+}

tools/binpac/src/pac_dataunit.h

@@ -0,0 +1,44 @@
+#ifndef pac_dataunit_h
+#define pac_dataunit_h
+
+#include "pac_analyzer.h"
+
+// The type and parameters of input data unit of a flow. For instance, the
+// data unit of a DCE/RPC flow is DCE_RPC_PDU.
+
+class AnalyzerDataUnit : public AnalyzerElement {
+public:
+    enum DataUnitType { DATAGRAM, FLOWUNIT };
+    AnalyzerDataUnit(DataUnitType type, ID* id, ExprList* type_params, ExprList* context_params);
+    ~AnalyzerDataUnit() override;
+
+    void Prepare(Env* env);
+
+    // Initializes dataunit_id
+    void GenNewDataUnit(Output* out_cc, Env* env);
+    // Initializes analyzer_context_id
+    void GenNewContext(Output* out_cc, Env* env);
+
+    DataUnitType type() const { return type_; }
+    const ID* id() const { return id_; }
+    ExprList* type_params() const { return type_params_; }
+    ExprList* context_params() const { return context_params_; }
+
+    ParameterizedType* data_type() const { return data_type_; }
+    ParameterizedType* context_type() const { return context_type_; }
+
+    Field* dataunit_var_field() const { return dataunit_var_field_; }
+    Field* context_var_field() const { return context_var_field_; }
+
+private:
+    DataUnitType type_;
+    ID* id_;
+    ExprList* type_params_;
+    ExprList* context_params_;
+    ParameterizedType* data_type_;
+    ParameterizedType* context_type_;
+    Field* dataunit_var_field_;
+    Field* context_var_field_;
+};
+
+#endif // pac_dataunit_h

tools/binpac/src/pac_dbg.h

@@ -0,0 +1,14 @@
+#ifndef pac_dbg_h
+#define pac_dbg_h
+
+#include <assert.h>
+#include <stdio.h>
+
+extern bool FLAGS_pac_debug;
+
+#define ASSERT(x) assert(x)
+#define DEBUG_MSG(...)                                                                                                 \
+    if ( FLAGS_pac_debug )                                                                                             \
+    fprintf(stderr, __VA_ARGS__)
+
+#endif /* pac_dbg_h */

tools/binpac/src/pac_decl-inl.h

@@ -0,0 +1,6 @@
+#ifndef pac_decl_inl_h
+#define pac_decl_inl_h
+
+#include "pac_id.h"
+
+#endif // pac_decl_inl_h

tools/binpac/src/pac_decl.cc

@@ -0,0 +1,163 @@
+#include "pac_decl.h"
+
+#include "pac_attr.h"
+#include "pac_context.h"
+#include "pac_dataptr.h"
+#include "pac_embedded.h"
+#include "pac_exception.h"
+#include "pac_expr.h"
+#include "pac_exttype.h"
+#include "pac_id.h"
+#include "pac_output.h"
+#include "pac_param.h"
+#include "pac_record.h"
+#include "pac_type.h"
+#include "pac_utils.h"
+
+DeclList* Decl::decl_list_ = nullptr;
+Decl::DeclMap Decl::decl_map_;
+
+Decl::Decl(ID* id, DeclType decl_type) : id_(id), decl_type_(decl_type), attrlist_(nullptr) {
+    decl_map_[id_] = this;
+    if ( ! decl_list_ )
+        decl_list_ = new DeclList();
+    decl_list_->push_back(this);
+
+    DEBUG_MSG("Finished Decl %s\n", id_->Name());
+
+    analyzer_context_ = nullptr;
+}
+
+Decl::~Decl() {
+    delete id_;
+    delete_list(AttrList, attrlist_);
+}
+
+void Decl::AddAttrs(AttrList* attrs) {
+    if ( ! attrs )
+        return;
+    if ( ! attrlist_ )
+        attrlist_ = new AttrList();
+    foreach (i, AttrList, attrs) {
+        attrlist_->push_back(*i);
+        ProcessAttr(*i);
+    }
+}
+
+void Decl::ProcessAttr(Attr* attr) { throw Exception(attr, "unhandled attribute"); }
+
+void Decl::SetAnalyzerContext() {
+    analyzer_context_ = AnalyzerContextDecl::current_analyzer_context();
+    if ( ! analyzer_context_ ) {
+        throw Exception(this, "analyzer context not defined");
+    }
+}
+
+void Decl::ProcessDecls(Output* out_h, Output* out_cc) {
+    if ( ! decl_list_ )
+        return;
+
+    foreach (i, DeclList, decl_list_) {
+        Decl* decl = *i;
+        current_decl_id = decl->id();
+        decl->Prepare();
+    }
+
+    foreach (i, DeclList, decl_list_) {
+        Decl* decl = *i;
+        current_decl_id = decl->id();
+        decl->GenExternDeclaration(out_h);
+    }
+
+    out_h->println("namespace binpac {\n");
+    out_cc->println("namespace binpac {\n");
+
+    AnalyzerContextDecl* analyzer_context = AnalyzerContextDecl::current_analyzer_context();
+
+    foreach (i, DeclList, decl_list_) {
+        Decl* decl = *i;
+        current_decl_id = decl->id();
+        decl->GenForwardDeclaration(out_h);
+    }
+
+    if ( analyzer_context )
+        analyzer_context->GenNamespaceEnd(out_h);
+
+    out_h->println("");
+
+    foreach (i, DeclList, decl_list_) {
+        Decl* decl = *i;
+        current_decl_id = decl->id();
+        decl->GenCode(out_h, out_cc);
+    }
+
+    if ( analyzer_context ) {
+        analyzer_context->GenNamespaceEnd(out_h);
+        analyzer_context->GenNamespaceEnd(out_cc);
+    }
+
+    out_h->println("}  // namespace binpac");
+    out_cc->println("}  // namespace binpac");
+}
+
+Decl* Decl::LookUpDecl(const ID* id) {
+    DeclMap::iterator it = decl_map_.find(id);
+    if ( it == decl_map_.end() )
+        return nullptr;
+    return it->second;
+}
+
+int HelperDecl::helper_id_seq = 0;
+
+HelperDecl::HelperDecl(HelperType helper_type, ID* context_id, EmbeddedCode* code)
+    : Decl(new ID(strfmt("helper_%d", ++helper_id_seq)), HELPER),
+      helper_type_(helper_type),
+      context_id_(context_id),
+      code_(code) {}
+
+HelperDecl::~HelperDecl() {
+    delete context_id_;
+    delete code_;
+}
+
+void HelperDecl::Prepare() {
+    // Do nothing
+}
+
+void HelperDecl::GenExternDeclaration(Output* out_h) {
+    if ( helper_type_ == EXTERN )
+        code_->GenCode(out_h, global_env());
+}
+
+void HelperDecl::GenCode(Output* out_h, Output* out_cc) {
+    Env* env = global_env();
+
+#if 0
+	if ( context_id_ )
+		{
+		Decl *decl = Decl::LookUpDecl(context_id_);
+		if ( ! decl )
+			{
+			throw Exception(context_id_,
+			                fmt("cannot find declaration for %s",
+			                    context_id_->Name()));
+			}
+		env = decl->env();
+		if ( ! env )
+			{
+			throw Exception(context_id_,
+			                fmt("not a type or analyzer: %s",
+			                    context_id_->Name()));
+			}
+		}
+#endif
+
+    if ( helper_type_ == HEADER )
+        code_->GenCode(out_h, env);
+    else if ( helper_type_ == CODE )
+        code_->GenCode(out_cc, env);
+    else if ( helper_type_ == EXTERN )
+        ; // do nothing
+    else
+        ASSERT(0);
+}

tools/binpac/src/pac_decl.h

@@ -0,0 +1,78 @@
+#ifndef pac_decl_h
+#define pac_decl_h
+
+#include "pac_common.h"
+#include "pac_id.h"
+
+class Decl : public Object {
+public:
+    // Note: ANALYZER is not for AnalyzerDecl (which is an
+    // abstract class) , but for AnalyzerContextDecl.
+    enum DeclType { ENUM, LET, TYPE, FUNC, CONN, FLOW, ANALYZER, HELPER, REGEX };
+
+    Decl(ID* id, DeclType decl_type);
+    virtual ~Decl();
+
+    const ID* id() const { return id_; }
+    DeclType decl_type() const { return decl_type_; }
+    AnalyzerContextDecl* analyzer_context() const { return analyzer_context_; }
+
+    // NULL except for TypeDecl or AnalyzerDecl
+    virtual Env* env() const { return nullptr; }
+
+    virtual void Prepare() = 0;
+
+    // Generate declarations out of the "binpac" namespace
+    virtual void GenExternDeclaration(Output* out_h) { /* do nothing */ }
+
+    // Generate declarations before definition of classes
+    virtual void GenForwardDeclaration(Output* out_h) = 0;
+
+    virtual void GenCode(Output* out_h, Output* out_cc) = 0;
+
+    void TakeExprList();
+    void AddAttrs(AttrList* attrlist);
+    void SetAnalyzerContext();
+
+protected:
+    virtual void ProcessAttr(Attr* a);
+
+    ID* id_;
+    DeclType decl_type_;
+    AttrList* attrlist_;
+    AnalyzerContextDecl* analyzer_context_;
+
+public:
+    static void ProcessDecls(Output* out_h, Output* out_cc);
+    static Decl* LookUpDecl(const ID* id);
+
+private:
+    static DeclList* decl_list_;
+    typedef map<const ID*, Decl*, ID_ptr_cmp> DeclMap;
+    static DeclMap decl_map_;
+};
+
+class HelperDecl : public Decl {
+public:
+    enum HelperType {
+        HEADER,
+        CODE,
+        EXTERN,
+    };
+    HelperDecl(HelperType type, ID* context_id, EmbeddedCode* code);
+    ~HelperDecl() override;
+
+    void Prepare() override;
+    void GenExternDeclaration(Output* out_h) override;
+    void GenForwardDeclaration(Output* out_h) override { /* do nothing */ }
+    void GenCode(Output* out_h, Output* out_cc) override;
+
+private:
+    HelperType helper_type_;
+    ID* context_id_;
+    EmbeddedCode* code_;
+
+    static int helper_id_seq;
+};
+
+#endif // pac_decl_h

tools/binpac/src/pac_embedded.cc

@@ -0,0 +1,55 @@
+#include "pac_embedded.h"
+
+#include "pac_id.h"
+#include "pac_output.h"
+#include "pac_primitive.h"
+
+EmbeddedCodeSegment::EmbeddedCodeSegment(const string& s) : s_(s), primitive_(nullptr) {}
+
+EmbeddedCodeSegment::EmbeddedCodeSegment(PacPrimitive* primitive) : s_(""), primitive_(primitive) {}
+
+EmbeddedCodeSegment::~EmbeddedCodeSegment() { delete primitive_; }
+
+string EmbeddedCodeSegment::ToCode(Env* env) {
+    if ( primitive_ && s_.empty() )
+        s_ = primitive_->ToCode(env);
+    return s_;
+}
+
+EmbeddedCode::EmbeddedCode() { segments_ = new EmbeddedCodeSegmentList(); }
+
+EmbeddedCode::~EmbeddedCode() { delete_list(EmbeddedCodeSegmentList, segments_); }
+
+void EmbeddedCode::Append(int atom) { current_segment_ += static_cast<char>(atom); }
+
+void EmbeddedCode::Append(const char* str) { current_segment_ += str; }
+
+void EmbeddedCode::Append(PacPrimitive* primitive) {
+    if ( ! current_segment_.empty() ) {
+        segments_->push_back(new EmbeddedCodeSegment(current_segment_));
+        current_segment_ = "";
+    }
+    segments_->push_back(new EmbeddedCodeSegment(primitive));
+}
+
+void EmbeddedCode::GenCode(Output* out, Env* env) {
+    if ( ! current_segment_.empty() ) {
+        segments_->push_back(new EmbeddedCodeSegment(current_segment_));
+        current_segment_ = "";
+    }
+
+    // TODO: return to the generated file after embedded code
+    // out->print("#line %d \"%s\"\n", line_num, filename.c_str());
+
+    // Allow use of RValue for undefined ID, in which case the
+    // ID's name is used as its RValue
+    env->set_allow_undefined_id(true);
+
+    foreach (i, EmbeddedCodeSegmentList, segments_) {
+        EmbeddedCodeSegment* segment = *i;
+        out->print("%s", segment->ToCode(env).c_str());
+    }
+
+    env->set_allow_undefined_id(false);
+    out->print("\n");
+}

tools/binpac/src/pac_embedded.h

@@ -0,0 +1,40 @@
+#ifndef pac_embedded_h
+#define pac_embedded_h
+
+#include "pac_common.h"
+
+class EmbeddedCodeSegment {
+public:
+    explicit EmbeddedCodeSegment(const string& s);
+    explicit EmbeddedCodeSegment(PacPrimitive* primitive);
+    ~EmbeddedCodeSegment();
+
+    string ToCode(Env* env);
+
+private:
+    string s_;
+    PacPrimitive* primitive_;
+};
+
+typedef vector<EmbeddedCodeSegment*> EmbeddedCodeSegmentList;
+
+class EmbeddedCode : public Object {
+public:
+    EmbeddedCode();
+    ~EmbeddedCode();
+
+    // Append a character
+    void Append(int atom);
+    void Append(const char* str);
+
+    // Append a PAC primitive
+    void Append(PacPrimitive* primitive);
+
+    void GenCode(Output* out, Env* env);
+
+private:
+    string current_segment_;
+    EmbeddedCodeSegmentList* segments_;
+};
+
+#endif // pac_embedded_h

tools/binpac/src/pac_enum.cc

@@ -0,0 +1,58 @@
+#include "pac_enum.h"
+
+#include "pac_exception.h"
+#include "pac_expr.h"
+#include "pac_exttype.h"
+#include "pac_output.h"
+#include "pac_typedecl.h"
+
+Enum::Enum(ID* id, Expr* expr) : id_(id), expr_(expr) {}
+
+Enum::~Enum() {
+    delete id_;
+    delete expr_;
+}
+
+void Enum::GenHeader(Output* out_h, int* pval) {
+    ASSERT(pval);
+    if ( expr_ ) {
+        if ( ! expr_->ConstFold(global_env(), pval) )
+            throw ExceptionNonConstExpr(expr_);
+        out_h->println("%s = %d,", id_->Name(), *pval);
+    }
+    else
+        out_h->println("%s,", id_->Name());
+    global_env()->AddConstID(id_, *pval);
+}
+
+EnumDecl::EnumDecl(ID* id, EnumList* enumlist) : Decl(id, ENUM), enumlist_(enumlist) {
+    ID* type_id = id->clone();
+    datatype_ = new ExternType(type_id, ExternType::NUMBER);
+    extern_typedecl_ = new TypeDecl(type_id, nullptr, datatype_);
+}
+
+EnumDecl::~EnumDecl() {
+    delete_list(EnumList, enumlist_);
+    delete extern_typedecl_;
+}
+
+void EnumDecl::Prepare() {
+    // Do nothing
+}
+
+void EnumDecl::GenForwardDeclaration(Output* out_h) {
+    out_h->println("// NOLINTNEXTLINE(performance-enum-size)");
+    out_h->println("enum %s {", id_->Name());
+    out_h->inc_indent();
+    int c = 0;
+    foreach (i, EnumList, enumlist_) {
+        (*i)->GenHeader(out_h, &c);
+        ++c;
+    }
+    out_h->dec_indent();
+    out_h->println("};");
+}
+
+void EnumDecl::GenCode(Output* out_h, Output* /* out_cc */) {
+    // Do nothing
+}

tools/binpac/src/pac_enum.h

@@ -0,0 +1,35 @@
+#ifndef pac_enum_h
+#define pac_enum_h
+
+#include "pac_decl.h"
+
+class Enum {
+public:
+    Enum(ID* id, Expr* expr = 0);
+    ~Enum();
+
+    void GenHeader(Output* out_h, int* pval);
+
+private:
+    ID* id_;
+    Expr* expr_;
+};
+
+class EnumDecl : public Decl {
+public:
+    EnumDecl(ID* id, EnumList* enumlist);
+    ~EnumDecl() override;
+
+    Type* DataType() const { return datatype_; }
+
+    void Prepare() override;
+    void GenForwardDeclaration(Output* out_h) override;
+    void GenCode(Output* out_h, Output* out_cc) override;
+
+private:
+    EnumList* enumlist_;
+    Type* datatype_;
+    TypeDecl* extern_typedecl_;
+};
+
+#endif // pac_enum_h

tools/binpac/src/pac_exception.cc

@@ -0,0 +1,61 @@
+#include "pac_exception.h"
+
+#include "pac_expr.h"
+#include "pac_id.h"
+#include "pac_utils.h"
+
+Exception::Exception(const Object* o, string msg) {
+    if ( o ) {
+        msg_ = o->Location();
+        msg_ += ": error : ";
+    }
+
+    msg_ += msg;
+
+    if ( FLAGS_pac_debug ) {
+        DEBUG_MSG("Exception: %s\n", msg_.c_str());
+        abort();
+    }
+}
+
+ExceptionIDNotFound::ExceptionIDNotFound(const ID* id) : Exception(id), id_(id) {
+    append(strfmt("`%s' undeclared", id_->Name()));
+}
+
+ExceptionIDRedefinition::ExceptionIDRedefinition(const ID* id) : Exception(id), id_(id) {
+    append(strfmt("`%s' redefined", id_->Name()));
+}
+
+ExceptionIDNotEvaluated::ExceptionIDNotEvaluated(const ID* id) : Exception(id), id_(id) {
+    append(strfmt("ID `%s' not evaluated before used", id->Name()));
+}
+
+ExceptionIDNotField::ExceptionIDNotField(const ID* id) : Exception(id), id_(id) {
+    append(strfmt("ID `%s' is not a field", id_->Name()));
+}
+
+ExceptionMemberNotFound::ExceptionMemberNotFound(const ID* type_id, const ID* member_id)
+    : Exception(member_id), type_id_(type_id), member_id_(member_id) {
+    append(strfmt("type %s does not have member `%s'", type_id_->Name(), member_id_->Name()));
+}
+
+ExceptionCyclicDependence::ExceptionCyclicDependence(const ID* id) : Exception(id), id_(id) {
+    append(strfmt("cyclic dependence through `%s'", id_->Name()));
+}
+
+ExceptionPaddingError::ExceptionPaddingError(const Object* o, string msg) : Exception(o) { append(msg.c_str()); }
+
+ExceptionNonConstExpr::ExceptionNonConstExpr(const Expr* expr) : Exception(expr), expr(expr) {
+    append(strfmt("Expression `%s' is not constant", expr->orig()));
+}
+
+ExceptionInvalidCaseSizeExpr::ExceptionInvalidCaseSizeExpr(const Expr* expr) : Exception(expr), expr(expr) {
+    append(strfmt("Expression `%s' is greater than the 32-bit limit for use as a case index", expr->orig()));
+}
+
+ExceptionInvalidCaseLimitExpr::ExceptionInvalidCaseLimitExpr(const Expr* expr) : Exception(expr), expr(expr) {
+    append(
+        strfmt("Expression `%s' as a case index is outside the numeric limit of the type used "
+               "for the switch expression",
+               expr->orig()));
+}

tools/binpac/src/pac_exception.h

@@ -0,0 +1,102 @@
+#ifndef pac_exception_h
+#define pac_exception_h
+
+#include <string>
+using namespace std;
+
+#include "pac_common.h"
+
+class Exception {
+public:
+    Exception(const Object* o, string msg = "");
+
+    const char* msg() const { return msg_.c_str(); }
+    void append(string s) { msg_ += s; }
+
+private:
+    string msg_;
+};
+
+class ExceptionIDNotFound : public Exception {
+public:
+    ExceptionIDNotFound(const ID* id);
+    const ID* id() const { return id_; }
+
+private:
+    const ID* id_;
+};
+
+class ExceptionIDRedefinition : public Exception {
+public:
+    ExceptionIDRedefinition(const ID* id);
+    const ID* id() const { return id_; }
+
+private:
+    const ID* id_;
+};
+
+class ExceptionIDNotEvaluated : public Exception {
+public:
+    ExceptionIDNotEvaluated(const ID* id);
+    const ID* id() const { return id_; }
+
+private:
+    const ID* id_;
+};
+
+class ExceptionCyclicDependence : public Exception {
+public:
+    ExceptionCyclicDependence(const ID* id);
+    const ID* id() const { return id_; }
+
+private:
+    const ID* id_;
+};
+
+class ExceptionPaddingError : public Exception {
+public:
+    ExceptionPaddingError(const Object* o, string msg);
+};
+
+class ExceptionIDNotField : public Exception {
+public:
+    ExceptionIDNotField(const ID* id);
+    const ID* id() const { return id_; }
+
+private:
+    const ID* id_;
+};
+
+class ExceptionMemberNotFound : public Exception {
+public:
+    ExceptionMemberNotFound(const ID* type_id, const ID* member_id);
+
+private:
+    const ID *type_id_, *member_id_;
+};
+
+class ExceptionNonConstExpr : public Exception {
+public:
+    ExceptionNonConstExpr(const Expr* expr);
+
+private:
+    const Expr* expr;
+};
+
+class ExceptionInvalidCaseSizeExpr : public Exception {
+public:
+    ExceptionInvalidCaseSizeExpr(const Expr* expr);
+
+private:
+    const Expr* expr;
+};
+
+class ExceptionInvalidCaseLimitExpr : public Exception {
+public:
+    ExceptionInvalidCaseLimitExpr(const Expr* expr);
+
+private:
+    const Expr* expr;
+};
+
+#endif /* pac_exception_h */

tools/binpac/src/pac_expr.cc

@@ -0,0 +1,858 @@
+#include "pac_expr.h"
+
+#include "pac_case.h"
+#include "pac_cstr.h"
+#include "pac_exception.h"
+#include "pac_exttype.h"
+#include "pac_id.h"
+#include "pac_nullptr.h"
+#include "pac_number.h"
+#include "pac_output.h"
+#include "pac_record.h"
+#include "pac_regex.h"
+#include "pac_strtype.h"
+#include "pac_typedecl.h"
+#include "pac_utils.h"
+
+string OrigExprList(ExprList* list) {
+    bool first = true;
+    string str;
+    foreach (i, ExprList, list) {
+        Expr* expr = *i;
+        if ( first )
+            first = false;
+        else
+            str += ", ";
+        str += expr->orig();
+    }
+    return str;
+}
+
+string EvalExprList(ExprList* exprlist, Output* out, Env* env) {
+    string val_list("");
+    bool first = true;
+
+    foreach (i, ExprList, exprlist) {
+        if ( ! first )
+            val_list += ", ";
+        val_list += (*i)->EvalExpr(out, env);
+        first = false;
+    }
+
+    return val_list;
+}
+
+static const char* expr_fmt[] = {
+#define EXPR_DEF(type, num_op, fmt) fmt,
+#include "pac_expr.def"
+#undef EXPR_DEF
+};
+
+void Expr::init() {
+    id_ = nullptr;
+    num_ = nullptr;
+    cstr_ = nullptr;
+    regex_ = nullptr;
+    num_operands_ = 0;
+    operand_[0] = nullptr;
+    operand_[1] = nullptr;
+    operand_[2] = nullptr;
+    args_ = nullptr;
+    cases_ = nullptr;
+}
+
+Expr::Expr(ID* arg_id) : DataDepElement(EXPR) {
+    init();
+    expr_type_ = EXPR_ID;
+    id_ = arg_id;
+    orig_ = strfmt("%s", id_->Name());
+}
+
+Expr::Expr(Number* arg_num) : DataDepElement(EXPR) {
+    init();
+    expr_type_ = EXPR_NUM;
+    num_ = arg_num;
+    orig_ = strfmt("((int) %s)", num_->Str());
+}
+
+Expr::Expr(Nullptr* arg_nullp) : DataDepElement(EXPR) {
+    init();
+    expr_type_ = EXPR_NULLPTR;
+    nullp_ = arg_nullp;
+    orig_ = strfmt("%s", nullp_->Str());
+}
+
+Expr::Expr(ConstString* cstr) : DataDepElement(EXPR) {
+    init();
+    expr_type_ = EXPR_CSTR;
+    cstr_ = cstr;
+    orig_ = cstr_->str();
+}
+
+Expr::Expr(RegEx* regex) : DataDepElement(EXPR) {
+    init();
+    expr_type_ = EXPR_REGEX;
+    regex_ = regex;
+    orig_ = strfmt("/%s/", regex_->str().c_str());
+}
+
+Expr::Expr(ExprType arg_type, Expr* op1) : DataDepElement(EXPR) {
+    init();
+    expr_type_ = arg_type;
+    num_operands_ = 1;
+    operand_[0] = op1;
+    orig_ = strfmt(expr_fmt[expr_type_], op1->orig());
+}
+
+Expr::Expr(ExprType arg_type, Expr* op1, Expr* op2) : DataDepElement(EXPR) {
+    init();
+    expr_type_ = arg_type;
+    num_operands_ = 2;
+    operand_[0] = op1;
+    operand_[1] = op2;
+    operand_[2] = nullptr;
+    orig_ = strfmt(expr_fmt[expr_type_], op1->orig(), op2->orig());
+}
+
+Expr::Expr(ExprType arg_type, Expr* op1, Expr* op2, Expr* op3) : DataDepElement(EXPR) {
+    init();
+    expr_type_ = arg_type;
+    num_operands_ = 3;
+    operand_[0] = op1;
+    operand_[1] = op2;
+    operand_[2] = op3;
+    orig_ = strfmt(expr_fmt[expr_type_], op1->orig(), op2->orig(), op3->orig());
+}
+
+Expr::Expr(ExprList* args) : DataDepElement(EXPR) {
+    init();
+    expr_type_ = EXPR_CALLARGS;
+    num_operands_ = -1;
+    args_ = args;
+
+    orig_ = OrigExprList(args_);
+}
+
+Expr::Expr(Expr* index, CaseExprList* cases) : DataDepElement(EXPR) {
+    init();
+    expr_type_ = EXPR_CASE;
+    num_operands_ = -1;
+    operand_[0] = index;
+    cases_ = cases;
+
+    orig_ = strfmt("case %s of { ", index->orig());
+    foreach (i, CaseExprList, cases_) {
+        CaseExpr* c = *i;
+        orig_ += strfmt("%s => %s; ", OrigExprList(c->index()).c_str(), c->value()->orig());
+    }
+    orig_ += "}";
+}
+
+Expr::~Expr() {
+    delete id_;
+    delete operand_[0];
+    delete operand_[1];
+    delete operand_[2];
+    delete_list(ExprList, args_);
+    delete_list(CaseExprList, cases_);
+}
+
+void Expr::AddCaseExpr(CaseExpr* case_expr) {
+    ASSERT(str_.empty());
+    ASSERT(expr_type_ == EXPR_CASE);
+    ASSERT(cases_);
+    cases_->push_back(case_expr);
+}
+
+void Expr::GenStrFromFormat(Env* env) {
+    // The format != "@custom@"
+    ASSERT(*expr_fmt[expr_type_] != '@');
+
+    switch ( num_operands_ ) {
+        case 1: str_ = strfmt(expr_fmt[expr_type_], operand_[0]->str()); break;
+        case 2: str_ = strfmt(expr_fmt[expr_type_], operand_[0]->str(), operand_[1]->str()); break;
+        case 3: str_ = strfmt(expr_fmt[expr_type_], operand_[0]->str(), operand_[1]->str(), operand_[2]->str()); break;
+        default:
+            DEBUG_MSG("num_operands_ = %d, orig = %s\n", num_operands_, orig());
+            ASSERT(0);
+            break;
+    }
+}
+
+namespace {
+
+RecordField* GetRecordField(const ID* id, Env* env) {
+    Field* field = env->GetField(id);
+    ASSERT(field);
+    if ( field->tof() != RECORD_FIELD && field->tof() != PADDING_FIELD )
+        throw Exception(id, "not a record field");
+    RecordField* r = static_cast<RecordField*>(field);
+    ASSERT(r);
+    return r;
+}
+
+} // namespace
+
+void Expr::GenCaseEval(Output* out_cc, Env* env) {
+    ASSERT(expr_type_ == EXPR_CASE);
+    ASSERT(operand_[0]);
+    ASSERT(cases_);
+
+    Type* val_type = DataType(env);
+    ID* val_var = env->AddTempID(val_type);
+
+    // DataType(env) can return a null pointer if an enum value is not
+    // defined.
+    if ( ! val_type )
+        throw Exception(this, "undefined case value");
+
+    out_cc->println("%s %s;", val_type->DataTypeStr().c_str(), env->LValue(val_var));
+
+    // force evaluation of IDs appearing in case stmt
+    operand_[0]->ForceIDEval(out_cc, env);
+    foreach (i, CaseExprList, cases_)
+        (*i)->value()->ForceIDEval(out_cc, env);
+
+    out_cc->println("// NOLINTBEGIN(bugprone-branch-clone)");
+    out_cc->println("switch ( %s ) {", operand_[0]->EvalExpr(out_cc, env));
+    Type* switch_type = operand_[0]->DataType(env);
+
+    out_cc->inc_indent();
+
+    CaseExpr* default_case = nullptr;
+    foreach (i, CaseExprList, cases_) {
+        CaseExpr* c = *i;
+        ExprList* index = c->index();
+        if ( ! index ) {
+            if ( default_case )
+                throw Exception(c, "duplicate default cases");
+            default_case = c;
+        }
+        else {
+            GenCaseStr(index, out_cc, env, switch_type);
+            out_cc->inc_indent();
+            out_cc->println("%s = %s;", env->LValue(val_var), c->value()->EvalExpr(out_cc, env));
+            out_cc->println("break;");
+            out_cc->dec_indent();
+        }
+    }
+
+    // Generate the default case after all other cases
+    GenCaseStr(nullptr, out_cc, env, switch_type);
+    out_cc->inc_indent();
+    if ( default_case ) {
+        out_cc->println("%s = %s;", env->LValue(val_var), default_case->value()->EvalExpr(out_cc, env));
+    }
+    else {
+        out_cc->println("throw binpac::ExceptionInvalidCaseIndex(\"%s\", (int64)%s);", Location(),
+                        operand_[0]->EvalExpr(out_cc, env));
+    }
+    out_cc->println("break;");
+    out_cc->dec_indent();
+
+    out_cc->dec_indent();
+    out_cc->println("}");
+    out_cc->println("// NOLINTEND(bugprone-branch-clone)");
+
+    env->SetEvaluated(val_var);
+    str_ = env->RValue(val_var);
+}
+
+void Expr::GenEval(Output* out_cc, Env* env) {
+    switch ( expr_type_ ) {
+        case EXPR_NUM: str_ = num_->Str(); break;
+        case EXPR_NULLPTR: str_ = nullp_->Str(); break;
+
+        case EXPR_ID:
+            if ( ! env->Evaluated(id_) )
+                env->Evaluate(out_cc, id_);
+            str_ = env->RValue(id_);
+            break;
+
+        case EXPR_MEMBER: {
+            /*
+            For member expressions such X.Y, evaluating
+            X only is sufficient. (Actually trying to
+            evaluate Y will lead to error because Y is
+            not defined in the current environment.)
+            */
+            operand_[0]->GenEval(out_cc, env);
+
+            Type* ty0 = operand_[0]->DataType(env);
+
+            if ( ty0 ) {
+                str_ = strfmt("%s%s", operand_[0]->EvalExpr(out_cc, env), ty0->EvalMember(operand_[1]->id()).c_str());
+            }
+            else {
+                string tmp = strfmt("->%s()", operand_[1]->id()->Name());
+                str_ = strfmt("%s%s", operand_[0]->EvalExpr(out_cc, env), tmp.c_str());
+            }
+        } break;
+
+        case EXPR_SUBSCRIPT: {
+            operand_[0]->GenEval(out_cc, env);
+            operand_[1]->GenEval(out_cc, env);
+
+            string v0 = operand_[0]->EvalExpr(out_cc, env);
+            string v1 = operand_[1]->EvalExpr(out_cc, env);
+
+            Type* ty0 = operand_[0]->DataType(env);
+            if ( ty0 )
+                str_ = ty0->EvalElement(v0, v1);
+            else
+                str_ = strfmt("%s[%s]", v0.c_str(), v1.c_str());
+        } break;
+
+        case EXPR_SIZEOF: {
+            const ID* id = operand_[0]->id();
+            RecordField* rf;
+            Type* ty;
+
+            try {
+                if ( (rf = GetRecordField(id, env)) != nullptr ) {
+                    str_ = strfmt("%s", rf->FieldSize(out_cc, env));
+                }
+            } catch ( ExceptionIDNotFound& e ) {
+                if ( (ty = TypeDecl::LookUpType(id)) != nullptr ) {
+                    int ty_size = ty->StaticSize(global_env());
+                    if ( ty_size >= 0 )
+                        str_ = strfmt("%d", ty_size);
+                    else
+                        throw Exception(id, "unknown size");
+                }
+                else
+                    throw Exception(id, "not a record field or type");
+            }
+        } break;
+
+        case EXPR_OFFSETOF: {
+            const ID* id = operand_[0]->id();
+            RecordField* rf = GetRecordField(id, env);
+            str_ = strfmt("%s", rf->FieldOffset(out_cc, env));
+        } break;
+
+        case EXPR_CALLARGS: str_ = EvalExprList(args_, out_cc, env); break;
+
+        case EXPR_CASE: GenCaseEval(out_cc, env); break;
+
+        default:
+            // Evaluate every operand by default
+            for ( int i = 0; i < 3; ++i )
+                if ( operand_[i] )
+                    operand_[i]->GenEval(out_cc, env);
+            GenStrFromFormat(env);
+            break;
+    }
+}
+
+void Expr::ForceIDEval(Output* out_cc, Env* env) {
+    switch ( expr_type_ ) {
+        case EXPR_NUM:
+        case EXPR_SIZEOF:
+        case EXPR_OFFSETOF: break;
+
+        case EXPR_ID:
+            if ( ! env->Evaluated(id_) )
+                env->Evaluate(out_cc, id_);
+            break;
+
+        case EXPR_MEMBER: operand_[0]->ForceIDEval(out_cc, env); break;
+
+        case EXPR_CALLARGS: {
+            foreach (i, ExprList, args_)
+                (*i)->ForceIDEval(out_cc, env);
+        } break;
+
+        case EXPR_CASE: {
+            operand_[0]->ForceIDEval(out_cc, env);
+            foreach (i, CaseExprList, cases_)
+                (*i)->value()->ForceIDEval(out_cc, env);
+        } break;
+
+        default:
+            // Evaluate every operand by default
+            for ( int i = 0; i < 3; ++i )
+                if ( operand_[i] )
+                    operand_[i]->ForceIDEval(out_cc, env);
+            break;
+    }
+}
+
+const char* Expr::EvalExpr(Output* out_cc, Env* env) {
+    GenEval(out_cc, env);
+    return str();
+}
+
+Type* Expr::DataType(Env* env) const {
+    Type* data_type;
+
+    switch ( expr_type_ ) {
+        case EXPR_ID: data_type = env->GetDataType(id_); break;
+
+        case EXPR_MEMBER: {
+            // Get type of the parent
+            Type* parent_type = operand_[0]->DataType(env);
+            if ( ! parent_type )
+                return nullptr;
+            data_type = parent_type->MemberDataType(operand_[1]->id());
+        } break;
+
+        case EXPR_SUBSCRIPT: {
+            // Get type of the parent
+            Type* parent_type = operand_[0]->DataType(env);
+            data_type = parent_type->ElementDataType();
+        } break;
+
+        case EXPR_PAREN: data_type = operand_[0]->DataType(env); break;
+
+        case EXPR_COND: {
+            Type* type1 = operand_[1]->DataType(env);
+            Type* type2 = operand_[2]->DataType(env);
+            if ( ! Type::CompatibleTypes(type1, type2) ) {
+                throw Exception(this, strfmt("type mismatch: %s vs %s", type1->DataTypeStr().c_str(),
+                                             type2->DataTypeStr().c_str()));
+            }
+            data_type = type1;
+        } break;
+
+        case EXPR_CALL: data_type = operand_[0]->DataType(env); break;
+
+        case EXPR_CASE: {
+            if ( cases_ && ! cases_->empty() ) {
+                Type* type1 = cases_->front()->value()->DataType(env);
+                Type* numeric_with_largest_width = nullptr;
+
+                foreach (i, CaseExprList, cases_) {
+                    Type* type2 = (*i)->value()->DataType(env);
+                    if ( ! Type::CompatibleTypes(type1, type2) ) {
+                        throw Exception(this, strfmt("type mismatch: %s vs %s", type1->DataTypeStr().c_str(),
+                                                     type2->DataTypeStr().c_str()));
+                    }
+                    if ( type1 == extern_type_nullptr )
+                        type1 = type2;
+
+                    if ( type2 && type2->IsNumericType() ) {
+                        if ( numeric_with_largest_width ) {
+                            int largest;
+                            int contender;
+
+                            // External C++ types like "int", "bool", "enum" use "int"
+                            // storage internally.
+                            if ( numeric_with_largest_width->tot() == Type::EXTERN )
+                                largest = sizeof(int);
+                            else
+                                largest = numeric_with_largest_width->StaticSize(env);
+
+                            if ( type2->tot() == Type::EXTERN )
+                                contender = sizeof(int);
+                            else
+                                contender = type2->StaticSize(env);
+
+                            if ( contender > largest )
+                                numeric_with_largest_width = type2;
+                        }
+                        else
+                            numeric_with_largest_width = type2;
+                    }
+                }
+                data_type = numeric_with_largest_width ? numeric_with_largest_width : type1;
+            }
+            else
+                data_type = nullptr;
+        } break;
+
+        case EXPR_NUM:
+        case EXPR_SIZEOF:
+        case EXPR_OFFSETOF:
+        case EXPR_NEG:
+        case EXPR_PLUS:
+        case EXPR_MINUS:
+        case EXPR_TIMES:
+        case EXPR_DIV:
+        case EXPR_MOD:
+        case EXPR_BITNOT:
+        case EXPR_BITAND:
+        case EXPR_BITOR:
+        case EXPR_BITXOR:
+        case EXPR_LSHIFT:
+        case EXPR_RSHIFT:
+        case EXPR_EQUAL:
+        case EXPR_GE:
+        case EXPR_LE:
+        case EXPR_GT:
+        case EXPR_LT:
+        case EXPR_NOT:
+        case EXPR_AND:
+        case EXPR_OR: data_type = extern_type_int; break;
+
+        default: data_type = nullptr; break;
+    }
+
+    return data_type;
+}
+
+string Expr::DataTypeStr(Env* env) const {
+    Type* type = DataType(env);
+
+    if ( ! type ) {
+        throw Exception(this, strfmt("cannot find data type for expression `%s'", orig()));
+    }
+
+    return type->DataTypeStr();
+}
+
+string Expr::SetFunc(Output* out, Env* env) {
+    switch ( expr_type_ ) {
+        case EXPR_ID: return set_function(id_);
+        case EXPR_MEMBER: {
+            // Evaluate the parent
+            string parent_val(operand_[0]->EvalExpr(out, env));
+            return parent_val + "->" + set_function(operand_[1]->id());
+        } break;
+        default:
+            throw Exception(this, strfmt("cannot generate set function "
+                                         "for expression `%s'",
+                                         orig()));
+            break;
+    }
+}
+
+bool Expr::ConstFold(Env* env, int* pn) const {
+    switch ( expr_type_ ) {
+        case EXPR_NUM: *pn = num_->Num(); return true;
+        case EXPR_ID: return env->GetConstant(id_, pn);
+        default:
+            // ### FIXME: folding consts
+            return false;
+    }
+}
+
+// TODO: build a generic data dependency extraction process
+namespace {
+
+// Maximum of two minimal header sizes
+int mhs_max(int h1, int h2) {
+    if ( h1 < 0 || h2 < 0 )
+        return -1;
+    else {
+        // return max(h1, h2);
+        return h1 > h2 ? h1 : h2;
+    }
+}
+
+// MHS required to evaluate the field
+int mhs_letfield(Env* env, LetField* field) { return field->expr()->MinimalHeaderSize(env); }
+
+int mhs_recordfield(Env* env, RecordField* field) {
+    int offset = field->static_offset();
+    if ( offset < 0 ) // offset cannot be statically determined
+        return -1;
+    int size = field->StaticSize(env, offset);
+    if ( size < 0 ) // size cannot be statically determined
+        return -1;
+    return offset + size;
+}
+
+int mhs_casefield(Env* env, CaseField* field) {
+    // TODO: deal with the index
+    int size = field->StaticSize(env);
+    if ( size < 0 ) // size cannot be statically determined
+        return -1;
+    return size;
+}
+
+int mhs_field(Env* env, Field* field) {
+    int mhs = -1;
+    switch ( field->tof() ) {
+        case LET_FIELD: {
+            LetField* f = static_cast<LetField*>(field);
+            ASSERT(f);
+            mhs = mhs_letfield(env, f);
+        } break;
+
+        case CONTEXT_FIELD:
+        case FLOW_FIELD: ASSERT(0); break;
+
+        case PARAM_FIELD: mhs = 0; break;
+
+        case RECORD_FIELD:
+        case PADDING_FIELD: {
+            RecordField* f = static_cast<RecordField*>(field);
+            ASSERT(f);
+            mhs = mhs_recordfield(env, f);
+        } break;
+
+        case CASE_FIELD: {
+            CaseField* f = static_cast<CaseField*>(field);
+            ASSERT(f);
+            mhs = mhs_casefield(env, f);
+        } break;
+
+        case PARSE_VAR_FIELD:
+        case PRIV_VAR_FIELD:
+        case PUB_VAR_FIELD:
+        case TEMP_VAR_FIELD: mhs = 0; break;
+
+        case WITHINPUT_FIELD: {
+            // ### TODO: fix this
+            mhs = -1;
+        } break;
+    }
+    return mhs;
+}
+
+int mhs_id(Env* env, const ID* id) {
+    int mhs = -1;
+    switch ( env->GetIDType(id) ) {
+        case CONST:
+        case GLOBAL_VAR:
+        case TEMP_VAR:
+        case STATE_VAR:
+        case FUNC_ID:
+        case FUNC_PARAM: mhs = 0; break;
+        case MEMBER_VAR:
+        case PRIV_MEMBER_VAR: {
+            Field* field = env->GetField(id);
+            if ( ! field )
+                throw ExceptionIDNotField(id);
+            mhs = mhs_field(env, field);
+        } break;
+        case UNION_VAR:
+            // TODO: deal with UNION_VAR
+            mhs = -1;
+            break;
+        case MACRO: {
+            Expr* e = env->GetMacro(id);
+            mhs = e->MinimalHeaderSize(env);
+        } break;
+    }
+    return mhs;
+}
+} // namespace
+
+int Expr::MinimalHeaderSize(Env* env) {
+    int mhs;
+
+    switch ( expr_type_ ) {
+        case EXPR_NUM:
+            // Zero byte is required
+            mhs = 0;
+            break;
+
+        case EXPR_ID: mhs = mhs_id(env, id_); break;
+
+        case EXPR_MEMBER:
+            // TODO: this is not a tight bound because
+            // one actually does not have to parse the
+            // whole record to compute one particular
+            // field.
+            mhs = operand_[0]->MinimalHeaderSize(env);
+            break;
+
+        case EXPR_SUBSCRIPT: {
+            int index;
+            Type* array_type = operand_[0]->DataType(env);
+            Type* elem_type = array_type->ElementDataType();
+            int elem_size = elem_type->StaticSize(env);
+            if ( elem_size >= 0 && operand_[1]->ConstFold(env, &index) ) {
+                mhs = elem_size * index;
+            }
+            else {
+                mhs = -1;
+            }
+        } break;
+
+        case EXPR_SIZEOF: {
+            const ID* id = operand_[0]->id();
+            ASSERT(id);
+            RecordField* rf;
+            Type* ty;
+
+            if ( (rf = GetRecordField(id, env)) != nullptr ) {
+                if ( rf->StaticSize(env, -1) >= 0 )
+                    mhs = 0;
+                else
+                    mhs = mhs_recordfield(env, rf);
+            }
+
+            else if ( (ty = TypeDecl::LookUpType(id)) != nullptr ) {
+                mhs = 0;
+            }
+
+            else
+                throw Exception(id, "not a record field or type");
+        } break;
+
+        case EXPR_OFFSETOF: {
+            const ID* id = operand_[0]->id();
+            ASSERT(id);
+            RecordField* field = GetRecordField(id, env);
+
+            mhs = field->static_offset();
+            if ( mhs < 0 ) {
+                mhs = 0;
+                // Take the MHS of the preceding (non-let) field
+                RecordField* prev_field = field->prev();
+                ASSERT(prev_field);
+                mhs = mhs_recordfield(env, prev_field);
+            }
+        } break;
+
+        case EXPR_CALLARGS: {
+            mhs = 0;
+            if ( args_ )
+                for ( unsigned int i = 0; i < args_->size(); ++i )
+                    mhs = mhs_max(mhs, (*args_)[i]->MinimalHeaderSize(env));
+        } break;
+        case EXPR_CASE: {
+            mhs = operand_[0]->MinimalHeaderSize(env);
+            for ( unsigned int i = 0; i < cases_->size(); ++i ) {
+                CaseExpr* ce = (*cases_)[i];
+                if ( ce->index() )
+                    for ( unsigned int j = 0; j < ce->index()->size(); ++j )
+                        mhs = mhs_max(mhs, (*ce->index())[j]->MinimalHeaderSize(env));
+                mhs = mhs_max(mhs, ce->value()->MinimalHeaderSize(env));
+            }
+        } break;
+        default:
+            // Evaluate every operand by default
+            mhs = 0;
+            for ( int i = 0; i < 3; ++i )
+                if ( operand_[i] )
+                    mhs = mhs_max(mhs, operand_[i]->MinimalHeaderSize(env));
+            break;
+    }
+
+    return mhs;
+}
+
+bool Expr::HasReference(const ID* id) const {
+    switch ( expr_type_ ) {
+        case EXPR_ID: return *id == *id_;
+
+        case EXPR_MEMBER: return operand_[0]->HasReference(id);
+
+        case EXPR_CALLARGS: {
+            foreach (i, ExprList, args_)
+                if ( (*i)->HasReference(id) )
+                    return true;
+        }
+            return false;
+
+        case EXPR_CASE: {
+            foreach (i, CaseExprList, cases_)
+                if ( (*i)->HasReference(id) )
+                    return true;
+        }
+            return false;
+
+        default:
+            // Evaluate every operand by default
+            for ( int i = 0; i < 3; ++i ) {
+                if ( operand_[i] && operand_[i]->HasReference(id) ) {
+                    return true;
+                }
+            }
+            return false;
+    }
+}
+
+bool Expr::DoTraverse(DataDepVisitor* visitor) {
+    switch ( expr_type_ ) {
+        case EXPR_ID: break;
+
+        case EXPR_MEMBER:
+            /*
+            For member expressions such X.Y, evaluating
+            X only is sufficient. (Actually trying to
+            evaluate Y will lead to error because Y is
+            not defined in the current environment.)
+            */
+            if ( ! operand_[0]->Traverse(visitor) )
+                return false;
+            break;
+
+        case EXPR_CALLARGS: {
+            foreach (i, ExprList, args_)
+                if ( ! (*i)->Traverse(visitor) )
+                    return false;
+        } break;
+
+        case EXPR_CASE: {
+            foreach (i, CaseExprList, cases_)
+                if ( ! (*i)->Traverse(visitor) )
+                    return false;
+        } break;
+
+        default:
+            // Evaluate every operand by default
+            for ( int i = 0; i < 3; ++i ) {
+                if ( operand_[i] && ! operand_[i]->Traverse(visitor) ) {
+                    return false;
+                }
+            }
+            break;
+    }
+
+    return true;
+}
+
+bool Expr::RequiresAnalyzerContext() const {
+    switch ( expr_type_ ) {
+        case EXPR_ID: return *id_ == *analyzer_context_id;
+
+        case EXPR_MEMBER:
+            /*
+            For member expressions such X.Y, evaluating
+            X only is sufficient. (Actually trying to
+            evaluate Y will lead to error because Y is
+            not defined in the current environment.)
+            */
+            return operand_[0]->RequiresAnalyzerContext();
+
+        case EXPR_CALLARGS: {
+            foreach (i, ExprList, args_)
+                if ( (*i)->RequiresAnalyzerContext() )
+                    return true;
+        }
+            return false;
+
+        case EXPR_CASE: {
+            foreach (i, CaseExprList, cases_)
+                if ( (*i)->RequiresAnalyzerContext() )
+                    return true;
+        }
+            return false;
+
+        default:
+            // Evaluate every operand by default
+            for ( int i = 0; i < 3; ++i )
+                if ( operand_[i] && operand_[i]->RequiresAnalyzerContext() ) {
+                    DEBUG_MSG("'%s' requires analyzer context\n", operand_[i]->orig());
+                    return true;
+                }
+            return false;
+    }
+}
+
+CaseExpr::CaseExpr(ExprList* index, Expr* value)
+    : DataDepElement(DataDepElement::CASEEXPR), index_(index), value_(value) {}
+
+CaseExpr::~CaseExpr() {
+    delete_list(ExprList, index_);
+    delete value_;
+}
+
+bool CaseExpr::DoTraverse(DataDepVisitor* visitor) {
+    foreach (i, ExprList, index_)
+        if ( ! (*i)->Traverse(visitor) )
+            return false;
+    return value_->Traverse(visitor);
+}
+
+bool CaseExpr::HasReference(const ID* id) const { return value_->HasReference(id); }
+
+bool CaseExpr::RequiresAnalyzerContext() const {
+    // index_ should evaluate to constants
+    return value_->RequiresAnalyzerContext();
+}

tools/binpac/src/pac_expr.def

@@ -0,0 +1,35 @@
+EXPR_DEF(EXPR_ID, 		0, "%s")
+EXPR_DEF(EXPR_NUM,		0, "%s")
+EXPR_DEF(EXPR_NULLPTR,		0, "%s")
+EXPR_DEF(EXPR_CSTR,		0, "%s")
+EXPR_DEF(EXPR_REGEX,		0, "REGEX(%s)")
+EXPR_DEF(EXPR_SUBSCRIPT,	2, "@element@(%s[%s])") 
+EXPR_DEF(EXPR_MEMBER,		2, "@%s->%s@") 
+EXPR_DEF(EXPR_PAREN,		1, " ( %s ) ") 
+EXPR_DEF(EXPR_CALL,		1, "%s(%s)")
+EXPR_DEF(EXPR_CALLARGS,		-1, "@custom@")
+EXPR_DEF(EXPR_SIZEOF,		1, "@sizeof(%s)@")
+EXPR_DEF(EXPR_OFFSETOF,		1, "@offsetof(%s)@")
+EXPR_DEF(EXPR_NEG,		1, "-%s") 
+EXPR_DEF(EXPR_PLUS,		2, "%s + %s")
+EXPR_DEF(EXPR_MINUS,		2, "%s - %s")
+EXPR_DEF(EXPR_TIMES,		2, "%s * %s")
+EXPR_DEF(EXPR_DIV,		2, "%s / %s")
+EXPR_DEF(EXPR_MOD,		2, "%s %% %s")
+EXPR_DEF(EXPR_BITNOT,		1, "~%s")
+EXPR_DEF(EXPR_BITAND,		2, "%s & %s")
+EXPR_DEF(EXPR_BITOR,		2, "%s | %s")
+EXPR_DEF(EXPR_BITXOR,		2, "%s ^ %s")
+EXPR_DEF(EXPR_LSHIFT,		2, "%s << %s")
+EXPR_DEF(EXPR_RSHIFT,		2, "%s >> %s")
+EXPR_DEF(EXPR_EQUAL,		2, "%s == %s")
+EXPR_DEF(EXPR_NEQ,		2, "%s != %s")
+EXPR_DEF(EXPR_GE,		2, "%s >= %s")
+EXPR_DEF(EXPR_LE,		2, "%s <= %s")
+EXPR_DEF(EXPR_GT,		2, "%s > %s")
+EXPR_DEF(EXPR_LT,		2, "%s < %s")
+EXPR_DEF(EXPR_NOT,		1, "! %s")
+EXPR_DEF(EXPR_AND,		2, "%s && %s")
+EXPR_DEF(EXPR_OR,		2, "%s || %s")
+EXPR_DEF(EXPR_COND,		3, "%s ? %s : %s")
+EXPR_DEF(EXPR_CASE,		-1, "@custom@")

tools/binpac/src/pac_expr.h

@@ -0,0 +1,141 @@
+#ifndef pac_expr_h
+#define pac_expr_h
+
+#include <cstdint>
+
+#include "pac_common.h"
+#include "pac_datadep.h"
+
+class CaseExpr;
+
+class Expr : public Object, public DataDepElement {
+public:
+    enum ExprType : uint8_t {
+#define EXPR_DEF(type, x, y) type,
+#include "pac_expr.def"
+#undef EXPR_DEF
+    };
+
+    void init();
+
+    Expr(ID* id);
+    Expr(Number* num);
+    Expr(Nullptr* nullp);
+    Expr(ConstString* s);
+    Expr(RegEx* regex);
+    Expr(ExprList* args); // for EXPR_CALLARGS
+    Expr(Expr* index, CaseExprList* cases);
+
+    Expr(ExprType type, Expr* op1);
+    Expr(ExprType type, Expr* op1, Expr* op2);
+    Expr(ExprType type, Expr* op1, Expr* op2, Expr* op3);
+
+    ~Expr() override;
+
+    const char* orig() const { return orig_.c_str(); }
+    const ID* id() const { return id_; }
+    const char* str() const { return str_.c_str(); }
+    ExprType expr_type() const { return expr_type_; }
+
+    void AddCaseExpr(CaseExpr* case_expr);
+
+    // Returns the data "type" of the expression. Here we only
+    // do a serious job for the EXPR_MEMBER and EXPR_SUBSCRIPT
+    // operators. For arithmetic operations, we fall back
+    // to "int".
+    Type* DataType(Env* env) const;
+    string DataTypeStr(Env* env) const;
+
+    // Note: EvalExpr() may generate C++ statements in order to evaluate
+    // variables in the expression, so the following is wrong:
+    //
+    // out->print("int x = ");
+    // out->println("%s", expr->EvalExpr(out, env));
+    //
+    // While putting them together is right:
+    //
+    // out->println("int x = %s", expr->EvalExpr(out, env));
+    //
+    const char* EvalExpr(Output* out, Env* env);
+
+    // force evaluation of IDs contained in this expression;
+    // necessary with case expr and conditional let fields (&if)
+    // for correct parsing of fields
+    void ForceIDEval(Output* out_cc, Env* env);
+
+    // Returns the set_* function of the expression.
+    // The expression must be of form ID or x.ID.
+    string SetFunc(Output* out, Env* env);
+
+    // Returns true if the expression folds to an integer
+    // constant with env, and puts the constant in *pn.
+    //
+    bool ConstFold(Env* env, int* pn) const;
+
+    // Whether id is referenced in the expression
+    bool HasReference(const ID* id) const;
+
+    // Suppose the data for type might be incomplete, what is
+    // the minimal number of bytes from data head required to
+    // compute the expression? For example, how many bytes of frame
+    // header do we need to determine the length of the frame?
+    //
+    // The parameter <env> points to the Env of a type.
+    //
+    // Returns -1 if the number is not a constant.
+    //
+    int MinimalHeaderSize(Env* env);
+
+    // Whether evaluation of the expression requires the analyzer context
+    bool RequiresAnalyzerContext() const;
+
+protected:
+    bool DoTraverse(DataDepVisitor* visitor) override;
+
+private:
+    ExprType expr_type_;
+
+    int num_operands_ = 0;
+    Expr* operand_[3] = {nullptr};
+
+    ID* id_ = nullptr;              // EXPR_ID
+    Number* num_ = nullptr;         // EXPR_NUM
+    ConstString* cstr_ = nullptr;   // EXPR_CSTR
+    RegEx* regex_ = nullptr;        // EXPR_REGEX
+    ExprList* args_ = nullptr;      // EXPR_CALLARGS
+    CaseExprList* cases_ = nullptr; // EXPR_CASE
+    Nullptr* nullp_ = nullptr;      // EXPR_NULLPTR
+
+    string str_;  // value string
+    string orig_; // original string for debugging info
+
+    void GenStrFromFormat(Env* env);
+    void GenEval(Output* out, Env* env);
+    void GenCaseEval(Output* out_cc, Env* env);
+};
+
+string OrigExprList(ExprList* exprlist);
+string EvalExprList(ExprList* exprlist, Output* out, Env* env);
+
+// An entry of the case expression, consisting of one or more constant
+// expressions for the case index and a value expression.
+class CaseExpr : public Object, public DataDepElement {
+public:
+    CaseExpr(ExprList* index, Expr* value);
+    ~CaseExpr() override;
+
+    ExprList* index() const { return index_; }
+    Expr* value() const { return value_; }
+
+    bool HasReference(const ID* id) const;
+    bool RequiresAnalyzerContext() const;
+
+protected:
+    bool DoTraverse(DataDepVisitor* visitor) override;
+
+private:
+    ExprList* index_;
+    Expr* value_;
+};
+
+#endif // pac_expr_h

tools/binpac/src/pac_externtype.def

@@ -0,0 +1,15 @@
+EXTERNTYPE(bool, bool, BOOLEAN)
+EXTERNTYPE(int, int, NUMBER)
+EXTERNTYPE(double, double, NUMBER)
+EXTERNTYPE(string, string, PLAIN)
+EXTERNTYPE(void, void, PLAIN)
+EXTERNTYPE(voidptr, void, POINTER)
+EXTERNTYPE(nullptr, nullptr, PLAIN)
+EXTERNTYPE(bytearray, bytearray, PLAIN)
+EXTERNTYPE(const_charptr, const_charptr, PLAIN)
+EXTERNTYPE(const_byteptr, const_byteptr, PLAIN)
+// EXTERNTYPE(const_byteseg, const_byteseg, PLAIN)
+EXTERNTYPE(const_bytestring, const_bytestring, PLAIN)
+// EXTERNTYPE(bytestring, bytestring, PLAIN)
+EXTERNTYPE(re_matcher, re_matcher, PLAIN)
+EXTERNTYPE(flowbuffer, FlowBuffer, POINTER)

tools/binpac/src/pac_exttype.cc

@@ -0,0 +1,64 @@
+#include "pac_exttype.h"
+
+#include "pac_decl.h"
+#include "pac_id.h"
+#include "pac_output.h"
+
+bool ExternType::DefineValueVar() const { return true; }
+
+string ExternType::DataTypeStr() const {
+    switch ( ext_type_ ) {
+        case PLAIN:
+        case NUMBER:
+        case BOOLEAN: return id_->Name();
+        case POINTER: return string(id_->Name()) + " *";
+        default: ASSERT(0); return "";
+    }
+}
+
+int ExternType::StaticSize(Env* env) const {
+    ASSERT(0);
+    return -1;
+}
+
+bool ExternType::ByteOrderSensitive() const { return false; }
+
+string ExternType::EvalMember(const ID* member_id) const {
+    return strfmt("%s%s", ext_type_ == POINTER ? "->" : ".", member_id->Name());
+}
+
+void ExternType::GenInitCode(Output* out_cc, Env* env) {
+    if ( IsNumericType() )
+        out_cc->println("%s = 0;", env->LValue(value_var()));
+    else if ( IsPointerType() )
+        out_cc->println("%s = nullptr;", env->LValue(value_var()));
+    else if ( IsBooleanType() )
+        out_cc->println("%s = false;", env->LValue(value_var()));
+
+    Type::GenInitCode(out_cc, env);
+}
+
+void ExternType::DoGenParseCode(Output* out, Env* env, const DataPtr& data, int flags) { ASSERT(0); }
+
+void ExternType::GenDynamicSize(Output* out, Env* env, const DataPtr& data) { ASSERT(0); }
+
+Type* ExternType::DoClone() const { return new ExternType(id_->clone(), ext_type_); }
+
+// Definitions of pre-defined external types
+
+#define EXTERNTYPE(name, ctype, exttype) ExternType* extern_type_##name = 0;
+#include "pac_externtype.def"
+#undef EXTERNTYPE
+
+void ExternType::static_init() {
+    ID* id;
+    // TypeDecl *decl;
+    // decl = new TypeDecl(id, 0, extern_type_##name);
+
+#define EXTERNTYPE(name, ctype, exttype)                                                                               \
+    id = new ID(#ctype);                                                                                               \
+    extern_type_##name = new ExternType(id, ExternType::exttype);                                                      \
+    Type::AddPredefinedType(#name, extern_type_##name);
+#include "pac_externtype.def"
+#undef EXTERNTYPE
+}

tools/binpac/src/pac_exttype.h

@@ -0,0 +1,46 @@
+#ifndef pac_exttype_h
+#define pac_exttype_h
+
+#include "pac_type.h"
+
+// ExternType represent external C++ types that are not defined in
+// PAC specification (therefore they cannot appear in data layout
+// specification, e.g., in a record field). The type name is copied
+// literally to the compiled code.
+
+class ExternType : public Type {
+public:
+    enum EXTType { PLAIN, NUMBER, POINTER, BOOLEAN };
+    ExternType(const ID* id, EXTType ext_type) : Type(EXTERN), id_(id), ext_type_(ext_type) {}
+
+    bool DefineValueVar() const override;
+    string DataTypeStr() const override;
+    int StaticSize(Env* env) const override;
+    bool ByteOrderSensitive() const override;
+
+    string EvalMember(const ID* member_id) const override;
+    bool IsNumericType() const override { return ext_type_ == NUMBER; }
+    bool IsPointerType() const override { return ext_type_ == POINTER; }
+    bool IsBooleanType() const override { return ext_type_ == BOOLEAN; }
+
+    void GenInitCode(Output* out_cc, Env* env) override;
+
+protected:
+    void DoGenParseCode(Output* out, Env* env, const DataPtr& data, int flags) override;
+    void GenDynamicSize(Output* out, Env* env, const DataPtr& data) override;
+
+    Type* DoClone() const override;
+
+private:
+    const ID* id_;
+    EXTType ext_type_;
+
+public:
+    static void static_init();
+};
+
+#define EXTERNTYPE(name, ctype, exttype) extern ExternType* extern_type_##name;
+#include "pac_externtype.def"
+#undef EXTERNTYPE
+
+#endif // pac_exttype_h

tools/binpac/src/pac_field.cc

@@ -0,0 +1,123 @@
+#include "pac_field.h"
+
+#include "pac_attr.h"
+#include "pac_common.h"
+#include "pac_exception.h"
+#include "pac_id.h"
+#include "pac_type.h"
+
+Field::Field(FieldType tof, int flags, ID* id, Type* type)
+    : DataDepElement(DataDepElement::FIELD), tof_(tof), flags_(flags), id_(id), type_(type) {
+    decl_id_ = current_decl_id;
+    field_id_str_ = strfmt("%s:%s", decl_id()->Name(), id_->Name());
+    attrs_ = nullptr;
+}
+
+Field::~Field() {
+    delete id_;
+    delete type_;
+    delete_list(AttrList, attrs_);
+}
+
+void Field::AddAttr(AttrList* attrs) {
+    bool delete_attrs = false;
+
+    if ( ! attrs_ ) {
+        attrs_ = attrs;
+    }
+    else {
+        attrs_->insert(attrs_->end(), attrs->begin(), attrs->end());
+        delete_attrs = true;
+    }
+
+    foreach (i, AttrList, attrs)
+        ProcessAttr(*i);
+
+    if ( delete_attrs )
+        delete attrs;
+}
+
+void Field::ProcessAttr(Attr* a) {
+    switch ( a->type() ) {
+        case ATTR_IF:
+            if ( tof() != LET_FIELD && tof() != WITHINPUT_FIELD ) {
+                throw Exception(a,
+                                "&if can only be applied to a "
+                                "let field");
+            }
+            break;
+        default: break;
+    }
+
+    if ( type_ )
+        type_->ProcessAttr(a);
+}
+
+bool Field::anonymous_field() const { return type_ && type_->anonymous_value_var(); }
+
+int Field::ValueVarType() const {
+    if ( flags_ & CLASS_MEMBER )
+        return (flags_ & PUBLIC_READABLE) ? MEMBER_VAR : PRIV_MEMBER_VAR;
+    else
+        return TEMP_VAR;
+}
+
+void Field::Prepare(Env* env) {
+    if ( type_ ) {
+        if ( anonymous_field() )
+            flags_ &= ~(CLASS_MEMBER | PUBLIC_READABLE);
+        if ( ! type_->persistent() )
+            flags_ &= (~PUBLIC_READABLE);
+
+        type_->set_value_var(id(), ValueVarType());
+        type_->Prepare(env, flags_ & TYPE_TO_BE_PARSED ? Type::TO_BE_PARSED : 0);
+        env->SetField(id(), this);
+    }
+}
+
+void Field::GenPubDecls(Output* out_h, Env* env) {
+    if ( type_ && (flags_ & PUBLIC_READABLE) && (flags_ & CLASS_MEMBER) )
+        type_->GenPubDecls(out_h, env);
+}
+
+void Field::GenPrivDecls(Output* out_h, Env* env) {
+    // Generate private declaration only if it is a class member
+    if ( type_ && (flags_ & CLASS_MEMBER) )
+        type_->GenPrivDecls(out_h, env);
+}
+
+void Field::GenTempDecls(Output* out_h, Env* env) {
+    // Generate temp field
+    if ( type_ && ! (flags_ & CLASS_MEMBER) )
+        type_->GenPrivDecls(out_h, env);
+}
+
+void Field::GenInitCode(Output* out_cc, Env* env) {
+    if ( type_ && ! anonymous_field() )
+        type_->GenInitCode(out_cc, env);
+}
+
+void Field::GenCleanUpCode(Output* out_cc, Env* env) {
+    if ( type_ && ! anonymous_field() )
+        type_->GenCleanUpCode(out_cc, env);
+}
+
+bool Field::DoTraverse(DataDepVisitor* visitor) {
+    // Check parameterized type
+    if ( type_ && ! type_->Traverse(visitor) )
+        return false;
+    foreach (i, AttrList, attrs_)
+        if ( ! (*i)->Traverse(visitor) )
+            return false;
+    return true;
+}
+
+bool Field::RequiresAnalyzerContext() const {
+    // Check parameterized type
+    if ( type_ && type_->RequiresAnalyzerContext() )
+        return true;
+    foreach (i, AttrList, attrs_)
+        if ( (*i)->RequiresAnalyzerContext() )
+            return true;
+    return false;
+}

tools/binpac/src/pac_field.h

@@ -0,0 +1,83 @@
+#ifndef pac_field_h
+#define pac_field_h
+
+#include "pac_common.h"
+#include "pac_datadep.h"
+
+// A "field" is a member of class.
+
+enum FieldType {
+    CASE_FIELD,
+    CONTEXT_FIELD,
+    FLOW_FIELD,
+    LET_FIELD,
+    PADDING_FIELD,
+    PARAM_FIELD,
+    RECORD_FIELD,
+    PARSE_VAR_FIELD,
+    PRIV_VAR_FIELD,
+    PUB_VAR_FIELD,
+    TEMP_VAR_FIELD,
+    WITHINPUT_FIELD,
+};
+
+class Field : public Object, public DataDepElement {
+public:
+    Field(FieldType tof, int flags, ID* id, Type* type);
+    // Field flags
+
+    // Whether the field will be evaluated by calling the Parse()
+    // function of the type
+    static const int TYPE_TO_BE_PARSED = 1;
+    static const int TYPE_NOT_TO_BE_PARSED = 0;
+
+    // Whether the field is a member of the class or a temp
+    // variable
+    static const int CLASS_MEMBER = 2;
+    static const int NOT_CLASS_MEMBER = 0;
+
+    // Whether the field is public readable
+    static const int PUBLIC_READABLE = 4;
+    static const int NOT_PUBLIC_READABLE = 0;
+
+    ~Field() override;
+
+    FieldType tof() const { return tof_; }
+    const ID* id() const { return id_; }
+    Type* type() const { return type_; }
+    const ID* decl_id() const { return decl_id_; }
+
+    bool anonymous_field() const;
+
+    void AddAttr(AttrList* attrs);
+
+    // The field interface
+    virtual void ProcessAttr(Attr* attr);
+    virtual void Prepare(Env* env);
+
+    virtual void GenPubDecls(Output* out, Env* env);
+    virtual void GenPrivDecls(Output* out, Env* env);
+    virtual void GenTempDecls(Output* out, Env* env);
+
+    virtual void GenInitCode(Output* out, Env* env);
+    virtual void GenCleanUpCode(Output* out, Env* env);
+
+    virtual bool RequiresAnalyzerContext() const;
+
+protected:
+    int ValueVarType() const;
+    bool ToBeParsed() const;
+
+    bool DoTraverse(DataDepVisitor* visitor) override;
+
+protected:
+    FieldType tof_;
+    int flags_;
+    ID* id_;
+    Type* type_;
+    const ID* decl_id_;
+    string field_id_str_;
+    AttrList* attrs_;
+};
+
+#endif // pac_field_h

tools/binpac/src/pac_flow.cc

@@ -0,0 +1,260 @@
+#include "pac_flow.h"
+
+#include "pac_analyzer.h"
+#include "pac_conn.h"
+#include "pac_context.h"
+#include "pac_dataptr.h"
+#include "pac_dataunit.h"
+#include "pac_embedded.h"
+#include "pac_exception.h"
+#include "pac_expr.h"
+#include "pac_exttype.h"
+#include "pac_output.h"
+#include "pac_param.h"
+#include "pac_paramtype.h"
+#include "pac_type.h"
+#include "pac_varfield.h"
+
+FlowDecl::FlowDecl(ID* id, ParamList* params, AnalyzerElementList* elemlist) : AnalyzerDecl(id, FLOW, params) {
+    dataunit_ = nullptr;
+    conn_decl_ = nullptr;
+    flow_buffer_var_field_ = nullptr;
+    AddElements(elemlist);
+}
+
+FlowDecl::~FlowDecl() {
+    delete flow_buffer_var_field_;
+    delete dataunit_;
+}
+
+ParameterizedType* FlowDecl::flow_buffer_type_ = nullptr;
+
+ParameterizedType* FlowDecl::flow_buffer_type() {
+    if ( ! flow_buffer_type_ ) {
+        flow_buffer_type_ = new ParameterizedType(new ID(kFlowBufferClass), nullptr);
+    }
+    return flow_buffer_type_;
+}
+
+void FlowDecl::AddBaseClass(vector<string>* base_classes) const { base_classes->push_back("binpac::FlowAnalyzer"); }
+
+void FlowDecl::ProcessFlowElement(AnalyzerFlow* flow_elem) {
+    throw Exception(flow_elem, "flow should be defined in only a connection declaration");
+}
+
+void FlowDecl::ProcessDataUnitElement(AnalyzerDataUnit* dataunit_elem) {
+    if ( dataunit_ ) {
+        throw Exception(dataunit_elem, "dataunit already defined");
+    }
+    dataunit_ = dataunit_elem;
+
+    if ( dataunit_->type() == AnalyzerDataUnit::FLOWUNIT ) {
+        dataunit_->data_type()->MarkIncrementalInput();
+
+        flow_buffer_var_field_ = new PubVarField(flow_buffer_id->clone(), FlowDecl::flow_buffer_type()->Clone());
+        type_->AddField(flow_buffer_var_field_);
+
+        ASSERT(AnalyzerContextDecl::current_analyzer_context());
+        AnalyzerContextDecl::current_analyzer_context()->AddFlowBuffer();
+
+        // Add an argument to the context initiation
+        dataunit_->context_type()->AddParamArg(new Expr(flow_buffer_var_field_->id()->clone()));
+    }
+}
+
+void FlowDecl::Prepare() {
+    // Add the connection parameter
+    if ( ! conn_decl_ ) {
+        throw Exception(this, "no connection is not declared for the flow");
+    }
+
+    if ( ! params_ )
+        params_ = new ParamList();
+
+    params_->insert(params_->begin(), new Param(connection_id->clone(), conn_decl_->DataType()));
+
+    AnalyzerDecl::Prepare();
+
+    dataunit_->Prepare(env_);
+}
+
+void FlowDecl::GenPubDecls(Output* out_h, Output* out_cc) { AnalyzerDecl::GenPubDecls(out_h, out_cc); }
+
+void FlowDecl::GenPrivDecls(Output* out_h, Output* out_cc) {
+    // Declare the data unit
+    dataunit_->dataunit_var_field()->GenPrivDecls(out_h, env_);
+
+    // Declare the analyzer context
+    dataunit_->context_var_field()->GenPrivDecls(out_h, env_);
+
+    AnalyzerDecl::GenPrivDecls(out_h, out_cc);
+}
+
+void FlowDecl::GenInitCode(Output* out_cc) {
+    AnalyzerDecl::GenInitCode(out_cc);
+
+    out_cc->println("%s = nullptr;", env_->LValue(dataunit_id));
+    out_cc->println("%s = nullptr;", env_->LValue(analyzer_context_id));
+
+    if ( dataunit_->type() == AnalyzerDataUnit::FLOWUNIT ) {
+        flow_buffer_var_field_->type()->GenPreParsing(out_cc, env_);
+        env_->SetEvaluated(flow_buffer_var_field_->id());
+    }
+}
+
+void FlowDecl::GenCleanUpCode(Output* out_cc) {
+    GenDeleteDataUnit(out_cc);
+    AnalyzerDecl::GenCleanUpCode(out_cc);
+}
+
+void FlowDecl::GenEOFFunc(Output* out_h, Output* out_cc) {
+    string proto = strfmt("%s()", kFlowEOF);
+
+    out_h->println("void %s;", proto.c_str());
+
+    out_cc->println("void %s::%s {", class_name().c_str(), proto.c_str());
+    out_cc->inc_indent();
+
+    foreach (i, AnalyzerHelperList, eof_helpers_) {
+        (*i)->GenCode(nullptr, out_cc, this);
+    }
+
+    if ( dataunit_->type() == AnalyzerDataUnit::FLOWUNIT ) {
+        out_cc->println("%s->set_eof();", env_->LValue(flow_buffer_id));
+        out_cc->println("%s(nullptr, nullptr);", kNewData);
+    }
+
+    out_cc->dec_indent();
+    out_cc->println("}");
+}
+
+void FlowDecl::GenGapFunc(Output* out_h, Output* out_cc) {
+    string proto = strfmt("%s(int gap_length)", kFlowGap);
+
+    out_h->println("void %s;", proto.c_str());
+
+    out_cc->println("void %s::%s {", class_name().c_str(), proto.c_str());
+    out_cc->inc_indent();
+
+    if ( dataunit_->type() == AnalyzerDataUnit::FLOWUNIT ) {
+        out_cc->println("%s->NewGap(gap_length);", env_->LValue(flow_buffer_id));
+    }
+
+    out_cc->dec_indent();
+    out_cc->println("}");
+}
+
+void FlowDecl::GenProcessFunc(Output* out_h, Output* out_cc) {
+    env_->AddID(begin_of_data, TEMP_VAR, extern_type_const_byteptr);
+    env_->AddID(end_of_data, TEMP_VAR, extern_type_const_byteptr);
+
+    string proto = strfmt("%s(const_byteptr %s, const_byteptr %s)", kNewData, env_->LValue(begin_of_data),
+                          env_->LValue(end_of_data));
+
+    out_h->println("void %s override;", proto.c_str());
+
+    out_cc->println("void %s::%s {", class_name().c_str(), proto.c_str());
+    out_cc->inc_indent();
+
+    out_cc->println("try {");
+    out_cc->inc_indent();
+
+    env_->SetEvaluated(begin_of_data);
+    env_->SetEvaluated(end_of_data);
+
+    switch ( dataunit_->type() ) {
+        case AnalyzerDataUnit::DATAGRAM: GenCodeDatagram(out_cc); break;
+        case AnalyzerDataUnit::FLOWUNIT: GenCodeFlowUnit(out_cc); break;
+        default: ASSERT(0);
+    }
+
+    out_cc->dec_indent();
+
+    out_cc->println("} catch ( binpac::Exception const& e ) {");
+    out_cc->inc_indent();
+    GenCleanUpCode(out_cc);
+    if ( dataunit_->type() == AnalyzerDataUnit::FLOWUNIT ) {
+        out_cc->println("%s->DiscardData();", env_->LValue(flow_buffer_id));
+    }
+    out_cc->println("throw e;");
+    out_cc->dec_indent();
+    out_cc->println("}");
+
+    out_cc->dec_indent();
+    out_cc->println("}");
+    out_cc->println("");
+}
+
+void FlowDecl::GenNewDataUnit(Output* out_cc) {
+    Type* unit_datatype = dataunit_->data_type();
+    // dataunit_->data_type()->GenPreParsing(out_cc, env_);
+    dataunit_->GenNewDataUnit(out_cc, env_);
+    if ( unit_datatype->buffer_input() && unit_datatype->buffer_mode() == Type::BUFFER_BY_LENGTH ) {
+        out_cc->println("%s->NewFrame(0, false);", env_->LValue(flow_buffer_id));
+    }
+    dataunit_->GenNewContext(out_cc, env_);
+}
+
+void FlowDecl::GenDeleteDataUnit(Output* out_cc) {
+    // Do not just delete dataunit, because we may just want to Unref it.
+    // out_cc->println("delete %s;", env_->LValue(dataunit_id));
+    dataunit_->data_type()->GenCleanUpCode(out_cc, env_);
+    dataunit_->context_type()->GenCleanUpCode(out_cc, env_);
+}
+
+void FlowDecl::GenCodeFlowUnit(Output* out_cc) {
+    Type* unit_datatype = dataunit_->data_type();
+
+    out_cc->println("%s->NewData(%s, %s);", env_->LValue(flow_buffer_id), env_->RValue(begin_of_data),
+                    env_->RValue(end_of_data));
+
+    out_cc->println("while ( %s->data_available() && ", env_->LValue(flow_buffer_id));
+    out_cc->inc_indent();
+    out_cc->println("( !%s->have_pending_request() || %s->ready() ) ) {", env_->LValue(flow_buffer_id),
+                    env_->LValue(flow_buffer_id));
+
+    // Generate a new dataunit if necessary
+    out_cc->println("if ( ! %s ) {", env_->LValue(dataunit_id));
+    out_cc->inc_indent();
+    out_cc->println("BINPAC_ASSERT(!%s);", env_->LValue(analyzer_context_id));
+    GenNewDataUnit(out_cc);
+    out_cc->dec_indent();
+    out_cc->println("}");
+
+    DataPtr data(env_, nullptr, 0);
+    unit_datatype->GenParseCode(out_cc, env_, data, 0);
+
+    out_cc->println("if ( %s ) {", unit_datatype->parsing_complete(env_).c_str());
+    out_cc->inc_indent();
+    out_cc->println("// Clean up the flow unit after parsing");
+    GenDeleteDataUnit(out_cc);
+    // out_cc->println("BINPAC_ASSERT(%s == 0);", env_->LValue(dataunit_id));
+    out_cc->dec_indent();
+    out_cc->println("} else {");
+    out_cc->inc_indent();
+    out_cc->println("// Resume upon next input segment");
+    out_cc->println("BINPAC_ASSERT(!%s->ready());", env_->RValue(flow_buffer_id));
+    out_cc->println("break;");
+    out_cc->dec_indent();
+    out_cc->println("}");
+
+    out_cc->dec_indent();
+    out_cc->println("}");
+}
+
+void FlowDecl::GenCodeDatagram(Output* out_cc) {
+    Type* unit_datatype = dataunit_->data_type();
+    GenNewDataUnit(out_cc);
+
+    string parse_params = strfmt("%s, %s", env_->RValue(begin_of_data), env_->RValue(end_of_data));
+
+    if ( RequiresAnalyzerContext::compute(unit_datatype) ) {
+        parse_params += ", ";
+        parse_params += env_->RValue(analyzer_context_id);
+    }
+
+    DataPtr dataptr(env_, begin_of_data, 0);
+    unit_datatype->GenParseCode(out_cc, env_, dataptr, 0);
+
+    GenDeleteDataUnit(out_cc);
+}

tools/binpac/src/pac_flow.h

@@ -0,0 +1,46 @@
+#ifndef pac_flow_h
+#define pac_flow_h
+
+#include "pac_analyzer.h"
+
+class FlowDecl : public AnalyzerDecl {
+public:
+    FlowDecl(ID* flow_id, ParamList* params, AnalyzerElementList* elemlist);
+    ~FlowDecl() override;
+
+    void Prepare() override;
+
+    void set_conn_decl(ConnDecl* c) { conn_decl_ = c; }
+
+    static ParameterizedType* flow_buffer_type();
+
+protected:
+    void AddBaseClass(vector<string>* base_classes) const override;
+
+    void GenInitCode(Output* out_cc) override;
+    void GenCleanUpCode(Output* out_cc) override;
+    void GenProcessFunc(Output* out_h, Output* out_cc) override;
+    void GenEOFFunc(Output* out_h, Output* out_cc) override;
+    void GenGapFunc(Output* out_h, Output* out_cc) override;
+
+    void GenPubDecls(Output* out_h, Output* out_cc) override;
+    void GenPrivDecls(Output* out_h, Output* out_cc) override;
+
+    void ProcessFlowElement(AnalyzerFlow* flow_elem) override;
+    void ProcessDataUnitElement(AnalyzerDataUnit* dataunit_elem) override;
+
+private:
+    void GenNewDataUnit(Output* out_cc);
+    void GenDeleteDataUnit(Output* out_cc);
+    void GenCodeFlowUnit(Output* out_cc);
+    void GenCodeDatagram(Output* out_cc);
+
+    AnalyzerDataUnit* dataunit_;
+    ConnDecl* conn_decl_;
+
+    Field* flow_buffer_var_field_;
+
+    static ParameterizedType* flow_buffer_type_;
+};
+
+#endif // pac_flow_h

tools/binpac/src/pac_func.cc

@@ -0,0 +1,88 @@
+#include "pac_func.h"
+
+#include "pac_embedded.h"
+#include "pac_expr.h"
+#include "pac_output.h"
+#include "pac_param.h"
+#include "pac_type.h"
+
+Function::Function(ID* id, Type* type, ParamList* params)
+    : id_(id), type_(type), params_(params), expr_(nullptr), code_(nullptr) {
+    analyzer_decl_ = nullptr;
+    env_ = nullptr;
+}
+
+Function::~Function() {
+    delete id_;
+    delete type_;
+    delete_list(ParamList, params_);
+    delete env_;
+    delete expr_;
+    delete code_;
+}
+
+void Function::Prepare(Env* env) {
+    env->AddID(id_, FUNC_ID, type_);
+    env->SetEvaluated(id_);
+
+    env_ = new Env(env, this);
+
+    foreach (i, ParamList, params_) {
+        Param* p = *i;
+        env_->AddID(p->id(), FUNC_PARAM, p->type());
+        env_->SetEvaluated(p->id());
+    }
+}
+
+void Function::GenForwardDeclaration(Output* out_h) {
+    // Do nothing
+}
+
+void Function::GenCode(Output* out_h, Output* out_cc) {
+    out_h->println("%s %s(%s);", type_->DataTypeStr().c_str(), id_->Name(), ParamDecls(params_).c_str());
+
+    string class_str = "";
+    if ( analyzer_decl_ )
+        class_str = strfmt("%s::", analyzer_decl_->id()->Name());
+
+    string proto_str = strfmt("%s %s%s(%s)", type_->DataTypeStr().c_str(), class_str.c_str(), id_->Name(),
+                              ParamDecls(params_).c_str());
+
+    ASSERT(! (expr_ && code_));
+
+    if ( expr_ ) {
+        out_cc->println("%s {", proto_str.c_str());
+        out_cc->inc_indent();
+
+        out_cc->println("return static_cast<%s>(%s);", type_->DataTypeStr().c_str(), expr_->EvalExpr(out_cc, env_));
+
+        out_cc->dec_indent();
+        out_cc->println("}");
+    }
+
+    else if ( code_ ) {
+        out_cc->println("%s {", proto_str.c_str());
+        out_cc->inc_indent();
+
+        code_->GenCode(out_cc, env_);
+
+        out_cc->dec_indent();
+        out_cc->println("}");
+    }
+
+    out_cc->println("");
+}
+
+FuncDecl::FuncDecl(Function* function) : Decl(function->id()->clone(), FUNC), function_(function) {
+    function_->Prepare(global_env());
+}
+
+FuncDecl::~FuncDecl() { delete function_; }
+
+void FuncDecl::Prepare() {}
+
+void FuncDecl::GenForwardDeclaration(Output* out_h) { function_->GenForwardDeclaration(out_h); }
+
+void FuncDecl::GenCode(Output* out_h, Output* out_cc) { function_->GenCode(out_h, out_cc); }
+
+AnalyzerFunction::AnalyzerFunction(Function* function) : AnalyzerElement(FUNCTION), function_(function) {}

tools/binpac/src/pac_func.h

@@ -0,0 +1,65 @@
+#ifndef pac_func_h
+#define pac_func_h
+
+#include "pac_analyzer.h"
+#include "pac_decl.h"
+
+class Function : public Object {
+public:
+    Function(ID* id, Type* type, ParamList* params);
+    ~Function();
+
+    ID* id() const { return id_; }
+
+    AnalyzerDecl* analyzer_decl() const { return analyzer_decl_; }
+    void set_analyzer_decl(AnalyzerDecl* decl) { analyzer_decl_ = decl; }
+
+    Expr* expr() const { return expr_; }
+    void set_expr(Expr* expr) { expr_ = expr; }
+
+    EmbeddedCode* code() const { return code_; }
+    void set_code(EmbeddedCode* code) { code_ = code; }
+
+    void Prepare(Env* env);
+    void GenForwardDeclaration(Output* out_h);
+    void GenCode(Output* out_h, Output* out_cc);
+
+private:
+    Env* env_;
+
+    ID* id_;
+    Type* type_;
+    ParamList* params_;
+
+    AnalyzerDecl* analyzer_decl_;
+
+    Expr* expr_;
+    EmbeddedCode* code_;
+};
+
+class FuncDecl : public Decl {
+public:
+    FuncDecl(Function* function);
+    ~FuncDecl() override;
+
+    Function* function() const { return function_; }
+
+    void Prepare() override;
+    void GenForwardDeclaration(Output* out_h) override;
+    void GenCode(Output* out_h, Output* out_cc) override;
+
+private:
+    Function* function_;
+};
+
+class AnalyzerFunction : public AnalyzerElement {
+public:
+    AnalyzerFunction(Function* function);
+
+    Function* function() const { return function_; }
+
+private:
+    Function* function_;
+};
+
+#endif // pac_func_h

tools/binpac/src/pac_id.cc

@@ -0,0 +1,375 @@
+#include "pac_id.h"
+
+#include "pac_exception.h"
+#include "pac_expr.h"
+#include "pac_exttype.h"
+#include "pac_field.h"
+#include "pac_type.h"
+#include "pac_utils.h"
+
+const ID* default_value_var = nullptr;
+const ID* null_id = nullptr;
+const ID* null_byteseg_id = nullptr;
+const ID* null_decl_id = nullptr;
+const ID* begin_of_data = nullptr;
+const ID* end_of_data = nullptr;
+const ID* len_of_data = nullptr;
+const ID* byteorder_id = nullptr;
+const ID* bigendian_id = nullptr;
+const ID* littleendian_id = nullptr;
+const ID* unspecified_byteorder_id = nullptr;
+const ID* const_true_id = nullptr;
+const ID* const_false_id = nullptr;
+const ID* analyzer_context_id = nullptr;
+const ID* context_macro_id = nullptr;
+const ID* this_id = nullptr;
+const ID* sourcedata_id = nullptr;
+const ID* connection_id = nullptr;
+const ID* upflow_id = nullptr;
+const ID* downflow_id = nullptr;
+const ID* dataunit_id = nullptr;
+const ID* flow_buffer_id = nullptr;
+const ID* element_macro_id = nullptr;
+const ID* input_macro_id = nullptr;
+const ID* cxt_connection_id = nullptr;
+const ID* cxt_flow_id = nullptr;
+const ID* parsing_state_id = nullptr;
+const ID* buffering_state_id = nullptr;
+
+int ID::anonymous_id_seq = 0;
+
+ID* ID::NewAnonymousID(const string& prefix) {
+    ID* id = new ID(strfmt("%s%03d", prefix.c_str(), ++anonymous_id_seq));
+    id->anonymous_id_ = true;
+    return id;
+}
+
+IDRecord::IDRecord(Env* arg_env, const ID* arg_id, IDType arg_id_type)
+    : env(arg_env), id(arg_id), id_type(arg_id_type) {
+    eval = nullptr;
+    evaluated = in_evaluation = false;
+    setfunc = ""; // except for STATE_VAR
+    switch ( id_type ) {
+        case MEMBER_VAR:
+            rvalue = strfmt("%s()", id->Name());
+            lvalue = strfmt("%s_", id->Name());
+            break;
+        case PRIV_MEMBER_VAR:
+            rvalue = strfmt("%s_", id->Name());
+            lvalue = strfmt("%s_", id->Name());
+            break;
+        case UNION_VAR:
+            rvalue = strfmt("%s()", id->Name());
+            lvalue = strfmt("%s_", id->Name());
+            break;
+        case CONST:
+        case GLOBAL_VAR:
+            rvalue = strfmt("%s", id->Name());
+            lvalue = strfmt("%s", id->Name());
+            break;
+        case TEMP_VAR:
+            rvalue = strfmt("t_%s", id->Name());
+            lvalue = strfmt("t_%s", id->Name());
+            break;
+        case STATE_VAR:
+            rvalue = strfmt("%s()", id->Name());
+            lvalue = strfmt("%s_", id->Name());
+            break;
+        case MACRO:
+            rvalue = "@MACRO@";
+            lvalue = "@MACRO@";
+            break;
+        case FUNC_ID:
+            rvalue = strfmt("%s", id->Name());
+            lvalue = "@FUNC_ID@";
+            break;
+        case FUNC_PARAM:
+            rvalue = strfmt("%s", id->Name());
+            lvalue = "@FUNC_PARAM@";
+            break;
+    }
+
+    data_type = nullptr;
+    field = nullptr;
+    constant = constant_set = false;
+    macro = nullptr;
+}
+
+IDRecord::~IDRecord() {}
+
+void IDRecord::SetConstant(int c) {
+    ASSERT(id_type == CONST);
+    constant_set = true;
+    constant = c;
+}
+
+bool IDRecord::GetConstant(int* pc) const {
+    if ( constant_set )
+        *pc = constant;
+    return constant_set;
+}
+
+void IDRecord::SetMacro(Expr* e) {
+    ASSERT(id_type == MACRO);
+    macro = e;
+}
+
+Expr* IDRecord::GetMacro() const {
+    ASSERT(id_type == MACRO);
+    return macro;
+}
+
+void IDRecord::SetEvaluated(bool v) {
+    if ( v )
+        ASSERT(! evaluated);
+    evaluated = v;
+}
+
+void IDRecord::Evaluate(Output* out, Env* env) {
+    if ( evaluated )
+        return;
+
+    if ( ! out )
+        throw ExceptionIDNotEvaluated(id);
+
+    if ( ! eval )
+        throw Exception(id, "no evaluation method");
+
+    if ( in_evaluation )
+        throw ExceptionCyclicDependence(id);
+
+    in_evaluation = true;
+    eval->GenEval(out, env);
+    in_evaluation = false;
+
+    evaluated = true;
+}
+
+const char* IDRecord::RValue() const {
+    if ( id_type == MACRO )
+        return macro->EvalExpr(nullptr, env);
+
+    if ( id_type == TEMP_VAR && ! evaluated )
+        throw ExceptionIDNotEvaluated(id);
+
+    return rvalue.c_str();
+}
+
+const char* IDRecord::LValue() const {
+    ASSERT(id_type != MACRO && id_type != FUNC_ID);
+    return lvalue.c_str();
+}
+
+Env::Env(Env* parent_env, Object* context_object) : parent(parent_env), context_object_(context_object) {
+    allow_undefined_id_ = false;
+    in_branch_ = false;
+}
+
+Env::~Env() {
+    for ( id_map_t::iterator it = id_map.begin(); it != id_map.end(); ++it ) {
+        delete it->second;
+        it->second = 0;
+    }
+}
+
+void Env::AddID(const ID* id, IDType id_type, Type* data_type) {
+    DEBUG_MSG("To add ID `%s'...\n", id->Name());
+    id_map_t::iterator it = id_map.find(id);
+    if ( it != id_map.end() ) {
+        DEBUG_MSG("Duplicate definition: `%s'\n", it->first->Name());
+        throw ExceptionIDRedefinition(id);
+    }
+    id_map[id] = new IDRecord(this, id, id_type);
+    // TODO: figure out when data_type must be non-NULL
+    // ASSERT(data_type);
+    SetDataType(id, data_type);
+}
+
+void Env::AddConstID(const ID* id, const int c, Type* type) {
+    if ( ! type )
+        type = extern_type_int;
+    AddID(id, CONST, type);
+    SetConstant(id, c);
+    SetEvaluated(id); // a constant is always evaluated
+}
+
+void Env::AddMacro(const ID* id, Expr* macro) {
+    AddID(id, MACRO, macro->DataType(this));
+    SetMacro(id, macro);
+    SetEvaluated(id);
+}
+
+ID* Env::AddTempID(Type* type) {
+    ID* id = ID::NewAnonymousID("t_var_");
+    AddID(id, TEMP_VAR, type);
+    return id;
+}
+
+IDRecord* Env::lookup(const ID* id, bool recursive, bool raise_exception) const {
+    ASSERT(id);
+
+    id_map_t::const_iterator it = id_map.find(id);
+    if ( it != id_map.end() )
+        return it->second;
+
+    if ( recursive && parent )
+        return parent->lookup(id, recursive, raise_exception);
+
+    if ( raise_exception )
+        throw ExceptionIDNotFound(id);
+    else
+        return nullptr;
+}
+
+IDType Env::GetIDType(const ID* id) const { return lookup(id, true, true)->GetType(); }
+
+const char* Env::RValue(const ID* id) const {
+    IDRecord* r = lookup(id, true, false);
+    if ( r )
+        return r->RValue();
+    else {
+        if ( allow_undefined_id() )
+            return id->Name();
+        else
+            throw ExceptionIDNotFound(id);
+    }
+}
+
+const char* Env::LValue(const ID* id) const { return lookup(id, true, true)->LValue(); }
+
+void Env::SetEvalMethod(const ID* id, Evaluatable* eval) { lookup(id, true, true)->SetEvalMethod(eval); }
+
+void Env::Evaluate(Output* out, const ID* id) {
+    IDRecord* r = lookup(id, true, ! allow_undefined_id());
+    if ( r )
+        r->Evaluate(out, this);
+}
+
+bool Env::Evaluated(const ID* id) const {
+    IDRecord* r = lookup(id, true, ! allow_undefined_id());
+    if ( r )
+        return r->Evaluated();
+    else
+        // Assume undefined variables are already evaluated
+        return true;
+}
+
+void Env::SetEvaluated(const ID* id, bool v) {
+    if ( in_branch() ) {
+        Field* f = GetField(id);
+        if ( f && f->tof() == LET_FIELD ) {
+            throw Exception(context_object_, strfmt("INTERNAL ERROR: "
+                                                    "evaluating let field '%s' in a branch! "
+                                                    "To work around this problem, "
+                                                    "add '&requires(%s)' to the case type. "
+                                                    "Sorry for the inconvenience.\n",
+                                                    id->Name(), id->Name()));
+            ASSERT(0);
+        }
+    }
+
+    IDRecord* r = lookup(id, false, false);
+    if ( r )
+        r->SetEvaluated(v);
+    else if ( parent )
+        parent->SetEvaluated(id, v);
+    else
+        throw ExceptionIDNotFound(id);
+}
+
+void Env::SetField(const ID* id, Field* field) { lookup(id, false, true)->SetField(field); }
+
+Field* Env::GetField(const ID* id) const { return lookup(id, true, true)->GetField(); }
+
+void Env::SetDataType(const ID* id, Type* type) { lookup(id, true, true)->SetDataType(type); }
+
+Type* Env::GetDataType(const ID* id) const {
+    IDRecord* r = lookup(id, true, false);
+    if ( r )
+        return r->GetDataType();
+    else
+        return nullptr;
+}
+
+string Env::DataTypeStr(const ID* id) const {
+    Type* type = GetDataType(id);
+    if ( ! type )
+        throw Exception(id, "data type not defined");
+    return type->DataTypeStr();
+}
+
+void Env::SetConstant(const ID* id, int constant) { lookup(id, false, true)->SetConstant(constant); }
+
+bool Env::GetConstant(const ID* id, int* pc) const {
+    ASSERT(pc);
+    // lookup without raising exception
+    IDRecord* r = lookup(id, true, false);
+    if ( r )
+        return r->GetConstant(pc);
+    else
+        return false;
+}
+
+void Env::SetMacro(const ID* id, Expr* macro) { lookup(id, true, true)->SetMacro(macro); }
+
+Expr* Env::GetMacro(const ID* id) const { return lookup(id, true, true)->GetMacro(); }
+
+void init_builtin_identifiers() {
+    default_value_var = new ID("val");
+    null_id = new ID("NULL");
+    null_byteseg_id = new ID("null_byteseg");
+    begin_of_data = new ID("begin_of_data");
+    end_of_data = new ID("end_of_data");
+    len_of_data = new ID("length_of_data");
+    byteorder_id = new ID("byteorder");
+    bigendian_id = new ID("bigendian");
+    littleendian_id = new ID("littleendian");
+    unspecified_byteorder_id = new ID("unspecified_byteorder");
+    const_true_id = new ID("true");
+    const_false_id = new ID("false");
+    analyzer_context_id = new ID("context");
+    this_id = new ID("this");
+    sourcedata_id = new ID("sourcedata");
+    connection_id = new ID("connection");
+    upflow_id = new ID("upflow");
+    downflow_id = new ID("downflow");
+    dataunit_id = new ID("dataunit");
+    flow_buffer_id = new ID("flow_buffer");
+    element_macro_id = new ID("$element");
+    input_macro_id = new ID("$input");
+    context_macro_id = new ID("$context");
+    parsing_state_id = new ID("parsing_state");
+    buffering_state_id = new ID("buffering_state");
+
+    null_decl_id = new ID("<null-decl>");
+    current_decl_id = null_decl_id;
+}
+
+Env* global_env() {
+    static Env* the_global_env = nullptr;
+
+    if ( ! the_global_env ) {
+        the_global_env = new Env(nullptr, nullptr);
+
+        // These two are defined in binpac.h, so we do not need to
+        // generate code for them.
+        the_global_env->AddConstID(bigendian_id, 0);
+        the_global_env->AddConstID(littleendian_id, 1);
+        the_global_env->AddConstID(unspecified_byteorder_id, -1);
+        the_global_env->AddConstID(const_false_id, 0);
+        the_global_env->AddConstID(const_true_id, 1);
+        // A hack for ID "this"
+        the_global_env->AddConstID(this_id, 0);
+        the_global_env->AddConstID(null_id, 0, extern_type_nullptr);
+
+#if 0
+		the_global_env->AddID(null_byteseg_id,
+			GLOBAL_VAR,
+			extern_type_const_byteseg);
+#endif
+    }
+
+    return the_global_env;
+}
+
+string set_function(const ID* id) { return strfmt("set_%s", id->Name()); }

tools/binpac/src/pac_id.h

@@ -0,0 +1,232 @@
+#ifndef pac_id_h
+#define pac_id_h
+
+#include <map>
+#include <string>
+using namespace std;
+
+#include "pac_common.h"
+#include "pac_dbg.h"
+#include "pac_utils.h"
+
+// Classes handling identifiers.
+//
+// ID -- name and location of definition of an ID
+//
+// IDRecord -- association of an ID, its definition type (const, global, temp,
+//   member, or union member), and its evaluation method.
+//
+// Evaluatable -- interface for a variable or a field that needs be evaluated
+//   before referenced.
+//
+// Env -- a mapping from ID names to their L/R-value expressions and evaluation
+//   methods.
+
+enum IDType {
+    CONST,
+    GLOBAL_VAR,
+    TEMP_VAR,
+    MEMBER_VAR,
+    PRIV_MEMBER_VAR,
+    UNION_VAR,
+    STATE_VAR,
+    MACRO,
+    FUNC_ID,
+    FUNC_PARAM,
+};
+
+class ID;
+class IDRecord;
+class Env;
+class Evaluatable;
+
+class ID : public Object {
+public:
+    ID(string arg_name) : name(arg_name), anonymous_id_(false) { locname = nfmt("%s:%s", Location(), Name()); }
+    ~ID() { delete[] locname; }
+
+    bool operator==(ID const& x) const { return name == x.Name(); }
+
+    const char* Name() const { return name.c_str(); }
+    const char* LocName() const { return locname; }
+    bool is_anonymous() const { return anonymous_id_; }
+
+    ID* clone() const { return new ID(Name()); }
+
+protected:
+    string name;
+    bool anonymous_id_;
+    char* locname;
+    friend class ID_ptr_cmp;
+
+public:
+    static ID* NewAnonymousID(const string& prefix);
+
+private:
+    static int anonymous_id_seq;
+};
+
+// A comparison operator for pointers to ID's.
+class ID_ptr_cmp {
+public:
+    bool operator()(const ID* const& id1, const ID* const& id2) const {
+        ASSERT(id1);
+        ASSERT(id2);
+        return id1->name < id2->name;
+    }
+};
+
+class IDRecord {
+public:
+    IDRecord(Env* env, const ID* id, IDType id_type);
+    ~IDRecord();
+
+    IDType GetType() const { return id_type; }
+
+    void SetDataType(Type* type) { data_type = type; }
+    Type* GetDataType() const { return data_type; }
+
+    void SetEvalMethod(Evaluatable* arg_eval) { eval = arg_eval; }
+    void Evaluate(Output* out, Env* env);
+    void SetEvaluated(bool v);
+    bool Evaluated() const { return evaluated; }
+
+    void SetField(Field* f) { field = f; }
+    Field* GetField() const { return field; }
+
+    void SetConstant(int c);
+    bool GetConstant(int* pc) const;
+
+    void SetMacro(Expr* expr);
+    Expr* GetMacro() const;
+
+    const char* RValue() const;
+    const char* LValue() const;
+
+protected:
+    Env* env;
+    const ID* id;
+    IDType id_type;
+
+    string rvalue;
+    string lvalue;
+    string setfunc;
+
+    Type* data_type;
+
+    Field* field;
+
+    int constant;
+    bool constant_set;
+
+    Expr* macro;
+
+    bool evaluated;
+    bool in_evaluation; // to detect cyclic dependence
+    Evaluatable* eval;
+};
+
+class Evaluatable {
+public:
+    virtual ~Evaluatable() {}
+    virtual void GenEval(Output* out, Env* env) = 0;
+};
+
+class Env {
+public:
+    Env(Env* parent_env, Object* context_object);
+    ~Env();
+
+    bool allow_undefined_id() const { return allow_undefined_id_; }
+    void set_allow_undefined_id(bool x) { allow_undefined_id_ = x; }
+
+    bool in_branch() const { return in_branch_; }
+    void set_in_branch(bool x) { in_branch_ = x; }
+
+    void AddID(const ID* id, IDType id_type, Type* type);
+    void AddConstID(const ID* id, const int c, Type* type = 0);
+    void AddMacro(const ID* id, Expr* expr);
+
+    // Generate a temp ID with a unique name
+    ID* AddTempID(Type* type);
+
+    IDType GetIDType(const ID* id) const;
+    const char* RValue(const ID* id) const;
+    const char* LValue(const ID* id) const;
+    // const char *SetFunc(const ID *id) const;
+
+    // Set evaluation method for the ID
+    void SetEvalMethod(const ID* id, Evaluatable* eval);
+
+    // Evaluate the ID according to the evaluation method. It
+    // assumes the ID has an evaluation emthod. It does nothing
+    // if the ID has already been evaluated.
+    void Evaluate(Output* out, const ID* id);
+
+    // Whether the ID has already been evaluated.
+    bool Evaluated(const ID* id) const;
+
+    // Set the ID as evaluated (or not).
+    void SetEvaluated(const ID* id, bool v = true);
+
+    void SetField(const ID* id, Field* field);
+    Field* GetField(const ID* id) const;
+
+    bool GetConstant(const ID* id, int* pc) const;
+
+    Expr* GetMacro(const ID* id) const;
+
+    Type* GetDataType(const ID* id) const;
+
+    string DataTypeStr(const ID* id) const;
+
+protected:
+    IDRecord* lookup(const ID* id, bool recursive, bool raise_exception) const;
+
+    void SetDataType(const ID* id, Type* type);
+    void SetConstant(const ID* id, int constant);
+    void SetMacro(const ID* id, Expr* macro);
+
+private:
+    Env* parent;
+    Object* context_object_;
+    typedef map<const ID*, IDRecord*, ID_ptr_cmp> id_map_t;
+    id_map_t id_map;
+    bool allow_undefined_id_;
+    bool in_branch_;
+};
+
+extern const ID* default_value_var;
+extern const ID* null_id;
+extern const ID* null_byteseg_id;
+extern const ID* begin_of_data;
+extern const ID* end_of_data;
+extern const ID* len_of_data;
+extern const ID* byteorder_id;
+extern const ID* bigendian_id;
+extern const ID* littleendian_id;
+extern const ID* unspecified_byteorder_id;
+extern const ID* analyzer_context_id;
+extern const ID* context_macro_id;
+extern const ID* this_id;
+extern const ID* sourcedata_id;
+// extern const ID *sourcedata_begin_id;
+// extern const ID *sourcedata_end_id;
+extern const ID* connection_id;
+extern const ID* upflow_id;
+extern const ID* downflow_id;
+extern const ID* dataunit_id;
+extern const ID* flow_buffer_id;
+extern const ID* element_macro_id;
+extern const ID* cxt_connection_id;
+extern const ID* cxt_flow_id;
+extern const ID* input_macro_id;
+extern const ID* parsing_state_id;
+extern const ID* buffering_state_id;
+
+extern void init_builtin_identifiers();
+extern Env* global_env();
+
+extern string set_function(const ID* id);
+
+#endif // pac_id_h

tools/binpac/src/pac_inputbuf.cc

@@ -0,0 +1,33 @@
+#include "pac_inputbuf.h"
+
+#include "pac_expr.h"
+#include "pac_exttype.h"
+#include "pac_id.h"
+#include "pac_output.h"
+#include "pac_type.h"
+
+InputBuffer::InputBuffer(Expr* expr) : DataDepElement(INPUT_BUFFER), expr_(expr) {}
+
+bool InputBuffer::DoTraverse(DataDepVisitor* visitor) {
+    if ( expr_ && ! expr_->Traverse(visitor) )
+        return false;
+    return true;
+}
+
+bool InputBuffer::RequiresAnalyzerContext() const { return expr_->RequiresAnalyzerContext(); }
+
+DataPtr InputBuffer::GenDataBeginEnd(Output* out_cc, Env* env) {
+    env->AddID(begin_of_data, TEMP_VAR, extern_type_const_byteptr);
+    env->AddID(end_of_data, TEMP_VAR, extern_type_const_byteptr);
+
+    out_cc->println("%s %s;", extern_type_const_byteptr->DataTypeStr().c_str(), env->LValue(begin_of_data));
+    out_cc->println("%s %s;", extern_type_const_byteptr->DataTypeStr().c_str(), env->LValue(end_of_data));
+
+    out_cc->println("get_pointers(%s, &%s, &%s);", expr_->EvalExpr(out_cc, env), env->LValue(begin_of_data),
+                    env->LValue(end_of_data));
+
+    env->SetEvaluated(begin_of_data);
+    env->SetEvaluated(end_of_data);
+
+    return DataPtr(env, begin_of_data, 0);
+}

tools/binpac/src/pac_inputbuf.h

@@ -0,0 +1,23 @@
+#ifndef pac_inputbuf_h
+#define pac_inputbuf_h
+
+#include "pac_datadep.h"
+#include "pac_dataptr.h"
+
+class Expr;
+
+class InputBuffer : public Object, public DataDepElement {
+public:
+    InputBuffer(Expr* expr);
+
+    bool RequiresAnalyzerContext() const;
+    DataPtr GenDataBeginEnd(Output* out_cc, Env* env);
+
+protected:
+    bool DoTraverse(DataDepVisitor* visitor) override;
+
+private:
+    Expr* expr_;
+};
+
+#endif // pac_inputbuf_h

tools/binpac/src/pac_let.cc

@@ -0,0 +1,121 @@
+#include "pac_let.h"
+
+#include "pac_expr.h"
+#include "pac_exttype.h"
+#include "pac_output.h"
+#include "pac_type.h"
+
+namespace {
+
+void GenLetEval(const ID* id, Expr* expr, string prefix, Output* out, Env* env) {}
+
+} // namespace
+
+LetField::LetField(ID* id, Type* type, Expr* expr)
+    : Field(LET_FIELD, TYPE_NOT_TO_BE_PARSED | CLASS_MEMBER | PUBLIC_READABLE, id, type), expr_(expr) {
+    ASSERT(expr_);
+}
+
+LetField::~LetField() { delete expr_; }
+
+bool LetField::DoTraverse(DataDepVisitor* visitor) { return Field::DoTraverse(visitor) && expr()->Traverse(visitor); }
+
+bool LetField::RequiresAnalyzerContext() const {
+    return Field::RequiresAnalyzerContext() || (expr() && expr()->RequiresAnalyzerContext());
+}
+
+void LetField::Prepare(Env* env) {
+    if ( ! type_ ) {
+        ASSERT(expr_);
+        type_ = expr_->DataType(env);
+        if ( type_ )
+            type_ = type_->Clone();
+        else
+            type_ = extern_type_int->Clone();
+
+        foreach (i, AttrList, attrs_)
+            ProcessAttr(*i);
+    }
+
+    Field::Prepare(env);
+    env->SetEvalMethod(id_, this);
+}
+
+void LetField::GenInitCode(Output* out_cc, Env* env) {
+    int v;
+    if ( expr_ && expr_->ConstFold(env, &v) ) {
+        DEBUG_MSG("Folding const for `%s'\n", id_->Name());
+        GenEval(out_cc, env);
+    }
+    else
+        type_->GenInitCode(out_cc, env);
+}
+
+void LetField::GenParseCode(Output* out_cc, Env* env) {
+    if ( env->Evaluated(id_) )
+        return;
+
+    if ( type_->attr_if_expr() ) {
+        // A conditional field
+
+        env->Evaluate(out_cc, type_->has_value_var());
+
+        // force evaluation of IDs contained in this expr
+        expr()->ForceIDEval(out_cc, env);
+
+        out_cc->println("if ( %s ) {", env->RValue(type_->has_value_var()));
+        out_cc->inc_indent();
+    }
+
+    out_cc->println("%s = %s;", env->LValue(id_), expr()->EvalExpr(out_cc, env));
+    if ( ! env->Evaluated(id_) )
+        env->SetEvaluated(id_);
+
+    if ( type_->attr_if_expr() ) {
+        out_cc->dec_indent();
+        out_cc->println("}");
+    }
+}
+
+void LetField::GenEval(Output* out_cc, Env* env) { GenParseCode(out_cc, env); }
+
+LetDecl::LetDecl(ID* id, Type* type, Expr* expr) : Decl(id, LET), type_(type), expr_(expr) {
+    if ( ! type_ ) {
+        ASSERT(expr_);
+        type_ = expr_->DataType(global_env());
+        if ( type_ )
+            type_ = type_->Clone();
+        else
+            type_ = extern_type_int->Clone();
+    }
+
+    Env* env = global_env();
+    int c;
+    if ( expr_ && expr_->ConstFold(env, &c) )
+        env->AddConstID(id_, c, type);
+    else
+        env->AddID(id_, GLOBAL_VAR, type_);
+}
+
+LetDecl::~LetDecl() {
+    delete type_;
+    delete expr_;
+}
+
+void LetDecl::Prepare() {}
+
+void LetDecl::GenForwardDeclaration(Output* out_h) {}
+
+void LetDecl::GenCode(Output* out_h, Output* out_cc) {
+    out_h->println("extern %s const %s;", type_->DataTypeStr().c_str(), global_env()->RValue(id_));
+    GenEval(out_cc, global_env());
+}
+
+void LetDecl::GenEval(Output* out_cc, Env* /* env */) {
+    Env* env = global_env();
+    string tmp = strfmt("%s const", type_->DataTypeStr().c_str());
+    out_cc->println("%s %s = %s;", tmp.c_str(), env->LValue(id_), expr_->EvalExpr(out_cc, env));
+
+    if ( ! env->Evaluated(id_) )
+        env->SetEvaluated(id_);
+}

tools/binpac/src/pac_let.h

@@ -0,0 +1,46 @@
+#ifndef pac_let_h
+#define pac_let_h
+
+#include "pac_decl.h"
+#include "pac_field.h"
+
+class LetField : public Field, Evaluatable {
+public:
+    LetField(ID* arg_id, Type* type, Expr* arg_expr);
+    ~LetField() override;
+
+    Expr* expr() const { return expr_; }
+
+    void Prepare(Env* env) override;
+
+    void GenInitCode(Output* out, Env* env) override;
+    void GenParseCode(Output* out, Env* env);
+    void GenEval(Output* out, Env* env) override;
+
+    bool RequiresAnalyzerContext() const override;
+
+protected:
+    bool DoTraverse(DataDepVisitor* visitor) override;
+
+protected:
+    Expr* expr_;
+};
+
+class LetDecl : public Decl, Evaluatable {
+public:
+    LetDecl(ID* id, Type* type, Expr* expr);
+    ~LetDecl() override;
+
+    Expr* expr() const { return expr_; }
+
+    void Prepare() override;
+    void GenForwardDeclaration(Output* out_h) override;
+    void GenCode(Output* out_h, Output* out_cc) override;
+    void GenEval(Output* out, Env* env) override;
+
+private:
+    Type* type_;
+    Expr* expr_;
+};
+
+#endif // pac_let_h

tools/binpac/src/pac_main.cc

@@ -0,0 +1,262 @@
+#include <ctype.h>
+#include <unistd.h>
+
+#include "config.h"
+#include "pac_common.h"
+#include "pac_decl.h"
+#include "pac_exception.h"
+#include "pac_exttype.h"
+#include "pac_id.h"
+#include "pac_output.h"
+#include "pac_parse.h"
+#include "pac_type.h"
+#include "pac_utils.h"
+
+extern int yydebug;
+extern int yyparse();
+extern void switch_to_file(FILE* fp_input);
+string input_filename;
+
+bool FLAGS_pac_debug = false;
+bool FLAGS_quiet = false;
+string FLAGS_output_directory;
+vector<string> FLAGS_include_directories;
+
+Output* header_output = nullptr;
+Output* source_output = nullptr;
+
+void add_to_include_directories(string dirs) {
+    unsigned int dir_begin = 0, dir_end;
+    while ( dir_begin < dirs.length() ) {
+        for ( dir_end = dir_begin; dir_end < dirs.length(); ++dir_end )
+            if ( dirs[dir_end] == ':' )
+                break;
+
+        string dir = dirs.substr(dir_begin, dir_end - dir_begin);
+
+        // Add a trailing '/' if necessary
+        if ( dir.length() > 0 && *(dir.end() - 1) != '/' )
+            dir += '/';
+
+        FLAGS_include_directories.push_back(std::move(dir));
+        dir_begin = dir_end + 1;
+    }
+}
+
+void pac_init() {
+    init_builtin_identifiers();
+    Type::init();
+}
+
+void insert_comments(Output* out, const char* source_filename) {
+    out->println("// This file is automatically generated from %s.\n", source_filename);
+}
+
+void insert_basictype_defs(Output* out) {
+    out->println("#ifndef pac_type_defs");
+    out->println("#define pac_type_defs");
+    out->println("");
+    out->println("typedef char int8;");
+    out->println("typedef short int16;");
+    out->println("typedef long int32;");
+    out->println("typedef long long int64;");
+
+    out->println("typedef unsigned char uint8;");
+    out->println("typedef unsigned short uint16;");
+    out->println("typedef unsigned long uint32;");
+    out->println("typedef unsigned long long uint64;");
+
+    out->println("");
+    out->println("#endif /* pac_type_defs */");
+    out->println("");
+}
+
+void insert_byteorder_macros(Output* out) {
+    out->println("#define FixByteOrder16(x)	(byteorder == HOST_BYTEORDER ? (x) : pac_swap16(x))");
+    out->println("#define FixByteOrder32(x)	(byteorder == HOST_BYTEORDER ? (x) : pac_swap32(x))");
+    out->println("#define FixByteOrder64(x)	(byteorder == HOST_BYTEORDER ? (x) : pac_swap64(x))");
+    out->println("");
+}
+
+const char* to_id(const char* s) {
+    static char t[1024];
+    int i;
+    for ( i = 0; s[i] && i < (int)sizeof(t) - 1; ++i )
+        t[i] = isalnum(s[i]) ? s[i] : '_';
+    if ( isdigit(t[0]) )
+        t[0] = '_';
+    t[i] = '\0';
+    return t;
+}
+
+int compile(const char* filename) {
+    FILE* fp_input = fopen(filename, "r");
+    if ( ! fp_input ) {
+        string tmp = strfmt("Error in opening %s", filename);
+        perror(tmp.c_str());
+        return -1;
+    }
+    input_filename = filename;
+
+    string basename;
+
+    if ( ! FLAGS_output_directory.empty() ) {
+        // Strip leading directories of filename
+        const char* last_slash = strrchr(filename, '/');
+        if ( last_slash )
+            basename = last_slash + 1;
+        else
+            basename = filename;
+        basename = FLAGS_output_directory + "/" + basename;
+    }
+    else
+        basename = filename;
+
+    // If the file name ends with ".pac"
+    if ( basename.length() > 4 && basename.substr(basename.length() - 4) == ".pac" ) {
+        basename = basename.substr(0, basename.length() - 4);
+    }
+
+    basename += "_pac";
+
+    DEBUG_MSG("Output file: %s.{h,cc}\n", basename.c_str());
+
+    int ret = 0;
+
+    try {
+        switch_to_file(fp_input);
+        if ( yyparse() )
+            return 1;
+
+        Output out_h(strfmt("%s.h", basename.c_str()));
+        Output out_cc(strfmt("%s.cc", basename.c_str()));
+
+        header_output = &out_h;
+        source_output = &out_cc;
+
+        insert_comments(&out_h, filename);
+        insert_comments(&out_cc, filename);
+
+        const char* filename_id = to_id(filename);
+
+        out_h.println("#ifndef %s_h", filename_id);
+        out_h.println("#define %s_h", filename_id);
+        out_h.println("");
+        out_h.println("#include <vector>");
+        out_h.println("");
+        out_h.println("#include \"binpac.h\"");
+        out_h.println("");
+
+        out_cc.println("");
+        out_cc.println("#ifdef __clang__");
+        out_cc.println("#pragma clang diagnostic ignored \"-Wparentheses-equality\"");
+        out_cc.println("#endif");
+        out_cc.println("");
+
+        out_cc.println("#include \"%s.h\"\n", basename.c_str());
+
+        Decl::ProcessDecls(&out_h, &out_cc);
+
+        out_h.println("#endif /* %s_h */", filename_id);
+    } catch ( OutputException& e ) {
+        fprintf(stderr, "Error in compiling %s: %s\n", filename, e.errmsg());
+        ret = 1;
+    } catch ( Exception& e ) {
+        fprintf(stderr, "%s\n", e.msg());
+        exit(1);
+    }
+
+    header_output = nullptr;
+    source_output = nullptr;
+    input_filename = "";
+    fclose(fp_input);
+
+    return ret;
+}
+
+void usage() {
+#ifdef BINPAC_VERSION
+    fprintf(stderr, "binpac version %s\n", BINPAC_VERSION);
+#endif
+    fprintf(stderr, "usage: binpac [options] <pac files>\n");
+    fprintf(stderr, "     <pac files>           | pac-language input files\n");
+    fprintf(stderr, "     -d <dir>              | use given directory for compiler output\n");
+    fprintf(stderr, "     -D                    | enable debugging output\n");
+    fprintf(stderr, "     -q                    | stay quiet\n");
+    fprintf(stderr, "     -h                    | show command line help\n");
+    fprintf(stderr, "     -I <dir>              | include <dir> in input file search path\n");
+    exit(1);
+}
+
+// GCC uses __SANITIZE_ADDRESS__, Clang uses __has_feature
+#if defined(__SANITIZE_ADDRESS__)
+#define USING_ASAN
+#endif
+
+#if defined(__has_feature)
+#if __has_feature(address_sanitizer)
+#define USING_ASAN
+#endif
+#endif
+
+// FreeBSD doesn't support LeakSanitizer
+#if defined(USING_ASAN) && ! defined(__FreeBSD__)
+#include <sanitizer/lsan_interface.h>
+#define BINPAC_LSAN_DISABLE() __lsan_disable()
+#else
+#define BINPAC_LSAN_DISABLE()
+#endif
+
+int main(int argc, char* argv[]) {
+    // We generally do not care at all if binpac is leaking and other
+    // projects that use it, like Zeek, only have their build tripped up
+    // by the default behavior of LSAN to treat leaks as errors.
+    BINPAC_LSAN_DISABLE();
+
+#ifdef HAVE_MALLOC_OPTIONS
+    extern char* malloc_options;
+#endif
+    int o;
+    while ( (o = getopt(argc, argv, "DqI:d:h")) != -1 ) {
+        switch ( o ) {
+            case 'D': yydebug = 1; FLAGS_pac_debug = true;
+#ifdef HAVE_MALLOC_OPTIONS
+                malloc_options = "A";
+#endif
+                break;
+
+            case 'q': FLAGS_quiet = true; break;
+
+            case 'I':
+                // Add to FLAGS_include_directories
+                add_to_include_directories(optarg);
+                break;
+
+            case 'd': FLAGS_output_directory = optarg; break;
+
+            case 'h': usage(); break;
+        }
+    }
+
+    // Strip the trailing '/'s
+    while ( ! FLAGS_output_directory.empty() && *(FLAGS_output_directory.end() - 1) == '/' ) {
+        FLAGS_output_directory.erase(FLAGS_output_directory.end() - 1);
+    }
+
+    // Add the current directory to FLAGS_include_directories
+    add_to_include_directories(".");
+
+    pac_init();
+
+    argc -= optind;
+    argv += optind;
+    if ( argc == 0 )
+        compile("-");
+
+    int ret = 0;
+    for ( int i = 0; i < argc; ++i )
+        if ( compile(argv[i]) )
+            ret = 1;
+
+    return ret;
+}

tools/binpac/src/pac_nullptr.h

@@ -0,0 +1,14 @@
+#ifndef pac_nullptr_h
+#define pac_nullptr_h
+
+#include "pac_common.h"
+
+class Nullptr : public Object {
+public:
+    const char* Str() const { return s.c_str(); }
+
+protected:
+    const string s = "nullptr";
+};
+
+#endif // pac_nullptr_h

tools/binpac/src/pac_number.h

@@ -0,0 +1,18 @@
+#ifndef pac_number_h
+#define pac_number_h
+
+#include "pac_common.h"
+
+class Number : public Object {
+public:
+    Number(int arg_n) : s(strfmt("%d", arg_n)), n(arg_n) {}
+    Number(const char* arg_s, int arg_n) : s(arg_s), n(arg_n) {}
+    const char* Str() const { return s.c_str(); }
+    int Num() const { return n; }
+
+protected:
+    const string s;
+    const int n;
+};
+
+#endif // pac_number_h

tools/binpac/src/pac_output.cc

@@ -0,0 +1,76 @@
+#include "pac_output.h"
+
+#include <errno.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "pac_utils.h"
+
+OutputException::OutputException(const char* arg_msg) { msg = arg_msg; }
+
+OutputException::~OutputException() {}
+
+Output::Output(string filename) {
+    fp = fopen(filename.c_str(), "w");
+    if ( ! fp )
+        throw OutputException(strerror(errno));
+    indent_ = 0;
+}
+
+Output::~Output() {
+    if ( fp )
+        fclose(fp);
+}
+
+int Output::print(const char* fmt, va_list ap) {
+    int r = vfprintf(fp, fmt, ap);
+    if ( r == -1 )
+        throw OutputException(strerror(errno));
+    return r;
+}
+
+int Output::print(const char* fmt, ...) {
+    va_list ap;
+    va_start(ap, fmt);
+    int r = -1;
+
+    try {
+        r = print(fmt, ap);
+    }
+
+    catch ( ... ) {
+        va_end(ap);
+        throw;
+    }
+
+    va_end(ap);
+    return r;
+}
+
+int Output::println(const char* fmt, ...) {
+    if ( strlen(fmt) == 0 ) {
+        fprintf(fp, "\n");
+        return 0;
+    }
+
+    for ( int i = 0; i < indent(); ++i )
+        fprintf(fp, "    ");
+
+    va_list ap;
+    va_start(ap, fmt);
+    int r = -1;
+
+    try {
+        r = print(fmt, ap);
+    }
+
+    catch ( ... ) {
+        va_end(ap);
+        throw;
+    }
+
+    va_end(ap);
+    fprintf(fp, "\n");
+    return r;
+}

tools/binpac/src/pac_output.h

@@ -0,0 +1,40 @@
+#ifndef pac_output_h
+#define pac_output_h
+
+#include <stdarg.h>
+#include <stdio.h>
+#include <string>
+
+using namespace std;
+
+class OutputException {
+public:
+    OutputException(const char* arg_msg);
+    ~OutputException();
+    const char* errmsg() const { return msg.c_str(); }
+
+protected:
+    string msg;
+};
+
+class Output {
+public:
+    Output(string filename);
+    ~Output();
+
+    int println(const char* fmt, ...);
+    int print(const char* fmt, ...);
+
+    int indent() const { return indent_; }
+
+    void inc_indent() { ++indent_; }
+    void dec_indent() { --indent_; }
+
+protected:
+    int print(const char* fmt, va_list ap);
+
+    FILE* fp;
+    int indent_;
+};
+
+#endif /* pac_output_h */

tools/binpac/src/pac_param.cc

@@ -0,0 +1,53 @@
+#include "pac_param.h"
+
+#include "pac_decl.h"
+#include "pac_exttype.h"
+#include "pac_field.h"
+#include "pac_id.h"
+#include "pac_output.h"
+#include "pac_type.h"
+#include "pac_utils.h"
+
+Param::Param(ID* id, Type* type) : id_(id), type_(type) {
+    if ( ! type_ )
+        type_ = extern_type_int->Clone();
+
+    decl_str_ = strfmt("%s %s", type_->DataTypeConstRefStr().c_str(), id_->Name());
+
+    param_field_ = new ParamField(this);
+}
+
+Param::~Param() {}
+
+const string& Param::decl_str() const {
+    ASSERT(! decl_str_.empty());
+    return decl_str_;
+}
+
+string ParamDecls(ParamList* params) {
+    string param_decls;
+
+    int first = 1;
+    foreach (i, ParamList, params) {
+        Param* p = *i;
+        const char* decl_str = p->decl_str().c_str();
+        if ( first )
+            first = 0;
+        else
+            param_decls += ", ";
+        param_decls += decl_str;
+    }
+    return param_decls;
+}
+
+ParamField::ParamField(const Param* param)
+    : Field(PARAM_FIELD, TYPE_NOT_TO_BE_PARSED | CLASS_MEMBER | PUBLIC_READABLE, param->id(), param->type()) {}
+
+void ParamField::GenInitCode(Output* out_cc, Env* env) {
+    out_cc->println("%s = %s;", env->LValue(id()), id()->Name());
+    env->SetEvaluated(id());
+}
+
+void ParamField::GenCleanUpCode(Output* out_cc, Env* env) {
+    // Do nothing
+}

tools/binpac/src/pac_param.h

@@ -0,0 +1,46 @@
+#ifndef pac_param_h
+#define pac_param_h
+
+#include "pac_common.h"
+#include "pac_field.h"
+
+class Param : public Object {
+public:
+    Param(ID* id, Type* type);
+    ~Param();
+
+    ID* id() const { return id_; }
+    Type* type() const { return type_; }
+    const string& decl_str() const;
+    Field* param_field() const { return param_field_; }
+
+private:
+    ID* id_;
+    Type* type_;
+    string decl_str_;
+    Field* param_field_;
+};
+
+class ParamField : public Field {
+public:
+    ParamField(const Param* param);
+
+    void GenInitCode(Output* out, Env* env) override;
+    void GenCleanUpCode(Output* out, Env* env) override;
+};
+
+// Returns the string with a list of param declarations separated by ','.
+string ParamDecls(ParamList* params);
+
+#if 0
+// Generate assignments to parameters, in the form of "%s_ = %s;" % (id, id).
+void GenParamAssignments(ParamList *params, Output *out_cc, Env *env);
+
+// Generate public access methods to parameter members.
+void GenParamPubDecls(ParamList *params, Output *out_h, Env *env);
+
+// Generate private definitions of parameter members.
+void GenParamPrivDecls(ParamList *params, Output *out_h, Env *env);
+#endif
+
+#endif // pac_param_h

tools/binpac/src/pac_paramtype.cc

@@ -0,0 +1,221 @@
+#include "pac_paramtype.h"
+
+#include "pac_context.h"
+#include "pac_dataptr.h"
+#include "pac_exception.h"
+#include "pac_expr.h"
+#include "pac_output.h"
+#include "pac_typedecl.h"
+
+ParameterizedType::ParameterizedType(ID* type_id, ExprList* args)
+    : Type(PARAMETERIZED), type_id_(type_id), args_(args) {
+    checking_requires_analyzer_context_ = false;
+}
+
+ParameterizedType::~ParameterizedType() {}
+
+string ParameterizedType::EvalMember(const ID* member_id) const {
+    Type* ty = ReferredDataType(true);
+    return strfmt("->%s", ty->env()->RValue(member_id));
+}
+
+string ParameterizedType::class_name() const { return type_id_->Name(); }
+
+Type* ParameterizedType::DoClone() const { return new ParameterizedType(type_id_->clone(), args_); }
+
+void ParameterizedType::AddParamArg(Expr* arg) { args_->push_back(arg); }
+
+bool ParameterizedType::DefineValueVar() const { return true; }
+
+string ParameterizedType::DataTypeStr() const { return strfmt("%s*", type_id_->Name()); }
+
+Type* ParameterizedType::MemberDataType(const ID* member_id) const {
+    Type* ref_type = TypeDecl::LookUpType(type_id_);
+    if ( ! ref_type )
+        return nullptr;
+    return ref_type->MemberDataType(member_id);
+}
+
+Type* ParameterizedType::ReferredDataType(bool throw_exception) const {
+    Type* type = TypeDecl::LookUpType(type_id_);
+    if ( ! type ) {
+        DEBUG_MSG("WARNING: cannot find referenced type for %s\n", type_id_->Name());
+        if ( throw_exception )
+            throw ExceptionIDNotFound(type_id_);
+    }
+    return type;
+}
+
+int ParameterizedType::StaticSize(Env* env) const { return ReferredDataType(true)->StaticSize(env); }
+
+void ParameterizedType::DoMarkIncrementalInput() {
+    Type* ty = ReferredDataType(true);
+
+    ty->MarkIncrementalInput();
+
+    buffer_input_ = ty->buffer_input();
+    incremental_parsing_ = ty->incremental_parsing();
+}
+
+Type::BufferMode ParameterizedType::buffer_mode() const {
+    // Note that the precedence is on attributes (&oneline or &length)
+    // specified on the parameterized type directly than on the type
+    // declaration.
+    //
+    // If both &oneline and &length are specified at the same place,
+    // use &length.
+    //
+    BufferMode mode = Type::buffer_mode();
+    Type* ty = ReferredDataType(true);
+
+    if ( mode != NOT_BUFFERABLE )
+        return mode;
+    else if ( ty->BufferableByLength() )
+        return BUFFER_BY_LENGTH;
+    else if ( ty->BufferableByLine() )
+        return BUFFER_BY_LINE;
+
+    return NOT_BUFFERABLE;
+}
+
+bool ParameterizedType::ByteOrderSensitive() const { return ReferredDataType(true)->RequiresByteOrder(); }
+
+bool ParameterizedType::DoTraverse(DataDepVisitor* visitor) {
+    if ( ! Type::DoTraverse(visitor) )
+        return false;
+
+    foreach (i, ExprList, args_)
+        if ( ! (*i)->Traverse(visitor) )
+            return false;
+
+    Type* ty = ReferredDataType(false);
+    if ( ty && ! ty->Traverse(visitor) )
+        return false;
+
+    return true;
+}
+
+bool ParameterizedType::RequiresAnalyzerContext() {
+    if ( checking_requires_analyzer_context_ )
+        return false;
+    checking_requires_analyzer_context_ = true;
+
+    bool ret = false;
+    // If any argument expression refers to analyzer context
+    foreach (i, ExprList, args_)
+        if ( (*i)->RequiresAnalyzerContext() ) {
+            ret = true;
+            break;
+        }
+    ret = ret || Type::RequiresAnalyzerContext();
+
+    if ( ! ret ) {
+        Type* ty = ReferredDataType(false);
+        if ( ty )
+            ret = ty->RequiresAnalyzerContext();
+    }
+
+    checking_requires_analyzer_context_ = false;
+    return ret;
+}
+
+void ParameterizedType::GenInitCode(Output* out_cc, Env* env) {
+    ASSERT(persistent());
+    out_cc->println("%s = nullptr;", env->LValue(value_var()));
+    Type::GenInitCode(out_cc, env);
+}
+
+void ParameterizedType::GenCleanUpCode(Output* out_cc, Env* env) {
+    Type* ty = ReferredDataType(false);
+    if ( ty && ty->attr_refcount() )
+        out_cc->println("Unref(%s);", lvalue());
+    else
+        out_cc->println("delete %s;", lvalue());
+    out_cc->println("%s = nullptr;", lvalue());
+    Type::GenCleanUpCode(out_cc, env);
+}
+
+string ParameterizedType::EvalParameters(Output* out_cc, Env* env) const {
+    string arg_str;
+
+    int first = 1;
+    foreach (i, ExprList, args_) {
+        Expr* e = *i;
+        if ( first )
+            first = 0;
+        else
+            arg_str += ", ";
+        arg_str += e->EvalExpr(out_cc, env);
+    }
+
+    return arg_str;
+}
+
+void ParameterizedType::GenNewInstance(Output* out_cc, Env* env) {
+    out_cc->println("%s = new %s(%s);", lvalue(), type_id_->Name(), EvalParameters(out_cc, env).c_str());
+}
+
+void ParameterizedType::DoGenParseCode(Output* out_cc, Env* env, const DataPtr& data, int flags) {
+    DEBUG_MSG("DoGenParseCode for %s\n", type_id_->Name());
+
+    Type* ref_type = ReferredDataType(true);
+
+    const char* parse_func;
+    string parse_params;
+
+    if ( buffer_mode() == BUFFER_NOTHING ) {
+        ASSERT(! ref_type->incremental_input());
+        parse_func = kParseFuncWithoutBuffer;
+        parse_params = "nullptr, nullptr";
+    }
+    else if ( ref_type->incremental_input() ) {
+        parse_func = kParseFuncWithBuffer;
+        parse_params = env->RValue(flow_buffer_id);
+    }
+    else {
+        parse_func = kParseFuncWithoutBuffer;
+        parse_params = strfmt("%s, %s", data.ptr_expr(), env->RValue(end_of_data));
+    }
+
+    if ( RequiresAnalyzerContext::compute(ref_type) ) {
+        parse_params += strfmt(", %s", env->RValue(analyzer_context_id));
+    }
+
+    if ( ref_type->RequiresByteOrder() ) {
+        env->Evaluate(out_cc, byteorder_id);
+        parse_params += strfmt(", %s", env->RValue(byteorder_id));
+    }
+
+    string call_parse_func = strfmt("%s->%s(%s)",
+                                    lvalue(), // parse() needs an LValue
+                                    parse_func, parse_params.c_str());
+
+    if ( incremental_input() ) {
+        if ( buffer_mode() == BUFFER_NOTHING ) {
+            out_cc->println("%s;", call_parse_func.c_str());
+            out_cc->println("%s = true;", env->LValue(parsing_complete_var()));
+        }
+        else {
+            ASSERT(parsing_complete_var());
+            out_cc->println("%s = %s;", env->LValue(parsing_complete_var()), call_parse_func.c_str());
+
+            // parsing_complete_var might have been already
+            // evaluated when set to false
+            if ( ! env->Evaluated(parsing_complete_var()) )
+                env->SetEvaluated(parsing_complete_var());
+        }
+    }
+    else {
+        if ( AddSizeVar(out_cc, env) ) {
+            out_cc->println("%s = %s;", env->LValue(size_var()), call_parse_func.c_str());
+            env->SetEvaluated(size_var());
+        }
+        else {
+            out_cc->println("%s;", call_parse_func.c_str());
+        }
+    }
+}
+
+void ParameterizedType::GenDynamicSize(Output* out_cc, Env* env, const DataPtr& data) {
+    GenParseCode(out_cc, env, data, 0);
+}

tools/binpac/src/pac_paramtype.h

@@ -0,0 +1,60 @@
+#ifndef pac_paramtype_h
+#define pac_paramtype_h
+
+#include "pac_type.h"
+
+// An instantiated type: ID + expression list
+class ParameterizedType : public Type {
+public:
+    ParameterizedType(ID* type_id, ExprList* args);
+    ~ParameterizedType() override;
+
+    Type* clone() const;
+
+    string EvalMember(const ID* member_id) const override;
+    // Env *member_env() const;
+
+    void AddParamArg(Expr* arg);
+
+    bool DefineValueVar() const override;
+    string DataTypeStr() const override;
+    string DefaultValue() const override { return "0"; }
+    Type* MemberDataType(const ID* member_id) const override;
+
+    // "throw_exception" specifies whether to throw an exception
+    // if the referred data type is not found
+    Type* ReferredDataType(bool throw_exception) const;
+
+    void GenCleanUpCode(Output* out, Env* env) override;
+
+    int StaticSize(Env* env) const override;
+
+    bool IsPointerType() const override { return true; }
+
+    bool ByteOrderSensitive() const override;
+    bool RequiresAnalyzerContext() override;
+
+    void GenInitCode(Output* out_cc, Env* env) override;
+
+    string class_name() const;
+    string EvalParameters(Output* out_cc, Env* env) const;
+
+    BufferMode buffer_mode() const override;
+
+protected:
+    void GenNewInstance(Output* out, Env* env) override;
+
+    bool DoTraverse(DataDepVisitor* visitor) override;
+    Type* DoClone() const override;
+    void DoMarkIncrementalInput() override;
+
+private:
+    ID* type_id_;
+    ExprList* args_;
+    bool checking_requires_analyzer_context_;
+
+    void DoGenParseCode(Output* out, Env* env, const DataPtr& data, int flags) override;
+    void GenDynamicSize(Output* out, Env* env, const DataPtr& data) override;
+};
+
+#endif // pac_paramtype_h

tools/binpac/src/pac_primitive.cc

@@ -0,0 +1,30 @@
+#include "pac_primitive.h"
+
+#include "pac_dbg.h"
+#include "pac_expr.h"
+#include "pac_id.h"
+#include "pac_type.h"
+
+string PPVal::ToCode(Env* env) {
+    ASSERT(expr_);
+    return string(expr_->EvalExpr(nullptr, env));
+}
+
+string PPSet::ToCode(Env* env) {
+    ASSERT(expr_);
+    return expr_->SetFunc(nullptr, env);
+}
+
+string PPType::ToCode(Env* env) {
+    Type* type = expr_->DataType(env);
+    return type->DataTypeStr();
+}
+
+string PPConstDef::ToCode(Env* env) {
+    Type* type = expr_->DataType(env);
+    env->AddID(id_, TEMP_VAR, type);
+    env->SetEvaluated(id_);
+
+    string type_str = type->DataTypeStr();
+    return strfmt("%s %s = %s", type_str.c_str(), env->LValue(id_), expr_->EvalExpr(nullptr, env));
+}

tools/binpac/src/pac_primitive.h

@@ -0,0 +1,67 @@
+#ifndef pac_primitive_h
+#define pac_primitive_h
+
+#include "pac_common.h"
+
+class PacPrimitive {
+public:
+    enum PrimitiveType { VAL, SET, TYPE, CONST_DEF };
+
+    explicit PacPrimitive(PrimitiveType type) : type_(type) {}
+    virtual ~PacPrimitive() {}
+
+    PrimitiveType type() const { return type_; }
+
+    virtual string ToCode(Env* env) = 0;
+
+private:
+    PrimitiveType type_;
+};
+
+class PPVal : public PacPrimitive {
+public:
+    PPVal(Expr* expr) : PacPrimitive(VAL), expr_(expr) {}
+    Expr* expr() const { return expr_; }
+
+    string ToCode(Env* env) override;
+
+private:
+    Expr* expr_;
+};
+
+class PPSet : public PacPrimitive {
+public:
+    PPSet(Expr* expr) : PacPrimitive(SET), expr_(expr) {}
+    Expr* expr() const { return expr_; }
+
+    string ToCode(Env* env) override;
+
+private:
+    Expr* expr_;
+};
+
+class PPType : public PacPrimitive {
+public:
+    PPType(Expr* expr) : PacPrimitive(TYPE), expr_(expr) {}
+    Expr* expr() const { return expr_; }
+
+    string ToCode(Env* env) override;
+
+private:
+    Expr* expr_;
+};
+
+class PPConstDef : public PacPrimitive {
+public:
+    PPConstDef(const ID* id, Expr* expr) : PacPrimitive(CONST_DEF), id_(id), expr_(expr) {}
+    const ID* id() const { return id_; }
+    Expr* expr() const { return expr_; }
+
+    string ToCode(Env* env) override;
+
+private:
+    const ID* id_;
+    Expr* expr_;
+};
+
+#endif // pac_primitive_h

tools/binpac/src/pac_record.cc

@@ -0,0 +1,566 @@
+#include "pac_record.h"
+
+#include "pac_attr.h"
+#include "pac_dataptr.h"
+#include "pac_exception.h"
+#include "pac_expr.h"
+#include "pac_exttype.h"
+#include "pac_field.h"
+#include "pac_output.h"
+#include "pac_type.h"
+#include "pac_typedecl.h"
+#include "pac_utils.h"
+#include "pac_varfield.h"
+
+RecordType::RecordType(RecordFieldList* record_fields) : Type(RECORD) {
+    // Here we assume that the type is a standalone type.
+    value_var_ = nullptr;
+
+    // Put all fields in fields_
+    foreach (i, RecordFieldList, record_fields)
+        AddField(*i);
+
+    // Put RecordField's in record_fields_
+    record_fields_ = record_fields;
+
+    parsing_dataptr_var_field_ = nullptr;
+}
+
+RecordType::~RecordType() {
+    // Do not delete_list(RecordFieldList, record_fields_)
+    // because the fields are also in fields_.
+    delete record_fields_;
+    delete parsing_dataptr_var_field_;
+}
+
+const ID* RecordType::parsing_dataptr_var() const {
+    return parsing_dataptr_var_field_ ? parsing_dataptr_var_field_->id() : nullptr;
+}
+
+bool RecordType::DefineValueVar() const { return false; }
+
+string RecordType::DataTypeStr() const {
+    ASSERT(type_decl());
+    return strfmt("%s*", type_decl()->class_name().c_str());
+}
+
+void RecordType::Prepare(Env* env, int flags) {
+    ASSERT(flags & TO_BE_PARSED);
+
+    RecordField* prev = nullptr;
+    int offset = 0;
+    int seq = 0;
+    foreach (i, RecordFieldList, record_fields_) {
+        RecordField* f = *i;
+        f->set_record_type(this);
+        f->set_prev(prev);
+        if ( prev )
+            prev->set_next(f);
+        prev = f;
+        if ( offset >= 0 ) {
+            f->set_static_offset(offset);
+            int w = f->StaticSize(env, offset);
+            if ( w < 0 )
+                offset = -1;
+            else
+                offset += w;
+        }
+        ++seq;
+        f->set_parsing_state_seq(seq);
+    }
+
+    if ( incremental_parsing() ) {
+#if 0
+		ASSERT(! parsing_state_var_field_);
+		ID *parsing_state_var_id = new ID("parsing_state");
+		parsing_state_var_field_ = new PrivVarField(
+			parsing_state_var_id, extern_type_int->Clone());
+		AddField(parsing_state_var_field_);
+
+		ID *parsing_dataptr_var_id = new ID("parsing_dataptr");
+		parsing_dataptr_var_field_ = new TempVarField(
+			parsing_dataptr_var_id, extern_type_const_byteptr->Clone());
+		parsing_dataptr_var_field_->Prepare(env);
+#endif
+    }
+
+    Type::Prepare(env, flags);
+}
+
+void RecordType::GenPubDecls(Output* out_h, Env* env) { Type::GenPubDecls(out_h, env); }
+
+void RecordType::GenPrivDecls(Output* out_h, Env* env) { Type::GenPrivDecls(out_h, env); }
+
+void RecordType::GenInitCode(Output* out_cc, Env* env) { Type::GenInitCode(out_cc, env); }
+
+void RecordType::GenCleanUpCode(Output* out_cc, Env* env) { Type::GenCleanUpCode(out_cc, env); }
+
+void RecordType::DoGenParseCode(Output* out_cc, Env* env, const DataPtr& data, int flags) {
+    if ( ! incremental_input() && StaticSize(env) >= 0 )
+        GenBoundaryCheck(out_cc, env, data);
+
+    if ( incremental_parsing() ) {
+        out_cc->println("// NOLINTBEGIN(bugprone-branch-clone)");
+        out_cc->println("switch ( %s ) {", env->LValue(parsing_state_id));
+
+        out_cc->println("case 0:");
+        out_cc->inc_indent();
+        foreach (i, RecordFieldList, record_fields_) {
+            RecordField* f = *i;
+            f->GenParseCode(out_cc, env);
+            out_cc->println("");
+        }
+        out_cc->println("");
+        out_cc->println("%s = true;", env->LValue(parsing_complete_var()));
+        out_cc->dec_indent();
+        out_cc->println("}");
+        out_cc->println("// NOLINTEND(bugprone-branch-clone)");
+    }
+    else {
+        ASSERT(data.id() == begin_of_data && data.offset() == 0);
+        foreach (i, RecordFieldList, record_fields_) {
+            RecordField* f = *i;
+            f->GenParseCode(out_cc, env);
+            out_cc->println("");
+        }
+        if ( incremental_input() ) {
+            ASSERT(parsing_complete_var());
+            out_cc->println("%s = true;", env->LValue(parsing_complete_var()));
+        }
+    }
+
+    if ( ! incremental_input() && AddSizeVar(out_cc, env) ) {
+        const DataPtr& end_of_record_dataptr = record_fields_->back()->getFieldEnd(out_cc, env);
+
+        out_cc->println("%s = %s - %s;", env->LValue(size_var()), end_of_record_dataptr.ptr_expr(),
+                        env->RValue(begin_of_data));
+        env->SetEvaluated(size_var());
+    }
+
+    if ( ! boundary_checked() ) {
+        RecordField* last_field = record_fields_->back();
+        if ( ! last_field->BoundaryChecked() )
+            GenBoundaryCheck(out_cc, env, data);
+    }
+}
+
+void RecordType::GenDynamicSize(Output* out_cc, Env* env, const DataPtr& data) { GenParseCode(out_cc, env, data, 0); }
+
+int RecordType::StaticSize(Env* env) const {
+    int tot_w = 0;
+    foreach (i, RecordFieldList, record_fields_) {
+        RecordField* f = *i;
+        int w = f->StaticSize(env, tot_w);
+        if ( w < 0 )
+            return -1;
+        tot_w += w;
+    }
+    return tot_w;
+}
+
+void RecordType::SetBoundaryChecked() {
+    Type::SetBoundaryChecked();
+
+    if ( StaticSize(env()) < 0 || attr_length_expr_ )
+        // Don't assume sufficient bounds checking has been done on fields
+        // if the record is of variable size or if its size is set from &length
+        // (whose value is not necessarily trustworthy).
+        return;
+
+    foreach (i, RecordFieldList, record_fields_) {
+        RecordField* f = *i;
+        f->SetBoundaryChecked();
+    }
+}
+
+void RecordType::DoMarkIncrementalInput() {
+    foreach (i, RecordFieldList, record_fields_) {
+        RecordField* f = *i;
+        f->type()->MarkIncrementalInput();
+    }
+}
+
+bool RecordType::DoTraverse(DataDepVisitor* visitor) { return Type::DoTraverse(visitor); }
+
+bool RecordType::ByteOrderSensitive() const {
+    foreach (i, RecordFieldList, record_fields_) {
+        RecordField* f = *i;
+        if ( f->RequiresByteOrder() )
+            return true;
+    }
+    return false;
+}
+
+RecordField::RecordField(FieldType tof, ID* id, Type* type)
+    : Field(tof, TYPE_TO_BE_PARSED | CLASS_MEMBER | PUBLIC_READABLE, id, type) {
+    begin_of_field_dataptr = nullptr;
+    end_of_field_dataptr = nullptr;
+    field_size_expr = nullptr;
+    field_offset_expr = nullptr;
+    end_of_field_dataptr_var = nullptr;
+    record_type_ = nullptr;
+    prev_ = nullptr;
+    next_ = nullptr;
+    static_offset_ = -1;
+    parsing_state_seq_ = 0;
+    boundary_checked_ = false;
+}
+
+RecordField::~RecordField() {
+    delete begin_of_field_dataptr;
+    delete end_of_field_dataptr;
+    delete[] field_size_expr;
+    delete[] field_offset_expr;
+    delete end_of_field_dataptr_var;
+}
+
+const DataPtr& RecordField::getFieldBegin(Output* out_cc, Env* env) {
+    if ( prev() )
+        return prev()->getFieldEnd(out_cc, env);
+    else {
+        // The first field
+        if ( ! begin_of_field_dataptr ) {
+            begin_of_field_dataptr = new DataPtr(env, begin_of_data, 0);
+        }
+        return *begin_of_field_dataptr;
+    }
+}
+
+const DataPtr& RecordField::getFieldEnd(Output* out_cc, Env* env) {
+    if ( end_of_field_dataptr )
+        return *end_of_field_dataptr;
+
+    const DataPtr& begin_ptr = getFieldBegin(out_cc, env);
+
+    if ( record_type()->incremental_parsing() ) {
+        ASSERT(0);
+        if ( ! end_of_field_dataptr ) {
+            const ID* dataptr_var = record_type()->parsing_dataptr_var();
+            ASSERT(dataptr_var);
+
+            end_of_field_dataptr = new DataPtr(env, dataptr_var, 0);
+        }
+    }
+    else {
+        int field_offset;
+        if ( begin_ptr.id() == begin_of_data )
+            field_offset = begin_ptr.offset();
+        else
+            field_offset = -1; // unknown
+
+        int field_size = StaticSize(env, field_offset);
+        if ( field_size >= 0 ) // can be statically determinted
+        {
+            end_of_field_dataptr = new DataPtr(env, begin_ptr.id(), begin_ptr.offset() + field_size);
+        }
+        else {
+            // If not, we add a variable for the offset after the field
+            end_of_field_dataptr_var = new ID(strfmt("dataptr_after_%s", id()->Name()));
+            env->AddID(end_of_field_dataptr_var, TEMP_VAR, extern_type_const_byteptr);
+
+            GenFieldEnd(out_cc, env, begin_ptr);
+
+            end_of_field_dataptr = new DataPtr(env, end_of_field_dataptr_var, 0);
+        }
+    }
+
+    return *end_of_field_dataptr;
+}
+
+const char* RecordField::FieldSize(Output* out_cc, Env* env) {
+    if ( field_size_expr )
+        return field_size_expr;
+
+    const DataPtr& begin = getFieldBegin(out_cc, env);
+    const DataPtr& end = getFieldEnd(out_cc, env);
+    if ( begin.id() == end.id() )
+        field_size_expr = nfmt("%d", end.offset() - begin.offset());
+    else
+        field_size_expr = nfmt("(%s - %s)", end.ptr_expr(), begin.ptr_expr());
+    return field_size_expr;
+}
+
+const char* RecordField::FieldOffset(Output* out_cc, Env* env) {
+    if ( field_offset_expr )
+        return field_offset_expr;
+
+    const DataPtr& begin = getFieldBegin(out_cc, env);
+    if ( begin.id() == begin_of_data )
+        field_offset_expr = nfmt("%d", begin.offset());
+    else
+        field_offset_expr = nfmt("(%s - %s)", begin.ptr_expr(), env->RValue(begin_of_data));
+    return field_offset_expr;
+}
+
+// The reasoning behind AttemptBoundaryCheck is: "If my next field
+// can check its boundary, then I don't have to check mine, and it
+// will save me a boundary-check."
+bool RecordField::AttemptBoundaryCheck(Output* out_cc, Env* env) {
+    if ( boundary_checked_ )
+        return true;
+
+    // If I do not even know my size till I parse the data, my
+    // next field won't be able to check its boundary now.
+
+    const DataPtr& begin = getFieldBegin(out_cc, env);
+    if ( StaticSize(env, begin.AbsOffset(begin_of_data)) < 0 )
+        return false;
+
+    // Now we ask the next field to check its boundary.
+    if ( next() && next()->AttemptBoundaryCheck(out_cc, env) ) {
+        // If it works, we are all set
+        SetBoundaryChecked();
+        return true;
+    }
+    else
+        // If it fails, then I can still try to do it by myself
+        return GenBoundaryCheck(out_cc, env);
+}
+
+RecordDataField::RecordDataField(ID* id, Type* type) : RecordField(RECORD_FIELD, id, type) { ASSERT(type_); }
+
+RecordDataField::~RecordDataField() {}
+
+void RecordDataField::Prepare(Env* env) {
+    Field::Prepare(env);
+    env->SetEvalMethod(id_, this);
+    env->SetField(id_, this);
+}
+
+void RecordDataField::GenParseCode(Output* out_cc, Env* env) {
+    if ( env->Evaluated(id()) )
+        return;
+
+    // Always evaluate record fields in order if parsing
+    // is incremental.
+    if ( record_type()->incremental_parsing() && prev() )
+        prev()->GenParseCode(out_cc, env);
+
+    DataPtr data(env, nullptr, 0);
+    if ( ! record_type()->incremental_parsing() ) {
+        data = getFieldBegin(out_cc, env);
+
+        Expr* len_expr = record_type()->attr_length_expr();
+        int len;
+
+        if ( ! record_type()->buffer_input() || (len_expr && len_expr->ConstFold(env, &len)) )
+            AttemptBoundaryCheck(out_cc, env);
+    }
+
+    out_cc->println("// Parse \"%s\"", id_->Name());
+#if 0
+	out_cc->println("DEBUG_MSG(\"%%.6f Parse %s\\n\", network_time());",
+		id_->Name());
+#endif
+    type_->GenPreParsing(out_cc, env);
+    if ( type_->incremental_input() ) {
+        // The enclosing record type must be incrementally parsed
+        out_cc->println("%s = %d;", env->LValue(parsing_state_id), parsing_state_seq());
+        out_cc->println("/* fall through */");
+        out_cc->dec_indent();
+        out_cc->println("case %d:", parsing_state_seq());
+        out_cc->println("{");
+        out_cc->inc_indent();
+    }
+
+    type_->GenParseCode(out_cc, env, data, 0);
+
+    if ( record_type()->incremental_parsing() ) {
+        ASSERT(type_->incremental_input());
+
+        out_cc->println("if ( ! (%s) )", type_->parsing_complete(env).c_str());
+        out_cc->inc_indent();
+        out_cc->println("goto %s;", kNeedMoreData);
+        out_cc->dec_indent();
+    }
+
+    if ( record_type()->incremental_parsing() ) {
+#if 0
+		const ID *dataptr_var =
+			record_type()->parsing_dataptr_var();
+		ASSERT(dataptr_var);
+		out_cc->println("%s += (%s);",
+			env->LValue(dataptr_var),
+			type_->DataSize(out_cc, env, data).c_str());
+#endif
+        out_cc->println("}");
+    }
+
+    SetBoundaryChecked();
+}
+
+void RecordDataField::GenEval(Output* out_cc, Env* env) { GenParseCode(out_cc, env); }
+
+void RecordDataField::GenFieldEnd(Output* out_cc, Env* env, const DataPtr& field_begin) {
+    out_cc->println("const_byteptr const %s = %s + (%s);", env->LValue(end_of_field_dataptr_var),
+                    field_begin.ptr_expr(), type_->DataSize(out_cc, env, field_begin).c_str());
+    env->SetEvaluated(end_of_field_dataptr_var);
+
+    out_cc->println("BINPAC_ASSERT(%s <= %s);", env->RValue(end_of_field_dataptr_var), env->RValue(end_of_data));
+}
+
+void RecordDataField::SetBoundaryChecked() {
+    RecordField::SetBoundaryChecked();
+    type_->SetBoundaryChecked();
+}
+
+bool RecordDataField::GenBoundaryCheck(Output* out_cc, Env* env) {
+    if ( boundary_checked_ )
+        return true;
+
+    type_->GenBoundaryCheck(out_cc, env, getFieldBegin(out_cc, env));
+
+    SetBoundaryChecked();
+    return true;
+}
+
+bool RecordDataField::DoTraverse(DataDepVisitor* visitor) { return Field::DoTraverse(visitor); }
+
+bool RecordDataField::RequiresAnalyzerContext() const {
+    return Field::RequiresAnalyzerContext() || type()->RequiresAnalyzerContext();
+}
+
+RecordPaddingField::RecordPaddingField(ID* id, PaddingType ptype, Expr* expr)
+    : RecordField(PADDING_FIELD, id, nullptr), ptype_(ptype), expr_(expr) {
+    wordsize_ = -1;
+}
+
+RecordPaddingField::~RecordPaddingField() {}
+
+void RecordPaddingField::Prepare(Env* env) {
+    Field::Prepare(env);
+    if ( ptype_ == PAD_TO_NEXT_WORD ) {
+        if ( ! expr_->ConstFold(env, &wordsize_) )
+            throw ExceptionPaddingError(this, strfmt("padding word size not a constant"));
+    }
+}
+
+void RecordPaddingField::GenParseCode(Output* out_cc, Env* env) {
+    // Always evaluate record fields in order if parsing
+    // is incremental.
+    if ( record_type()->incremental_parsing() && prev() )
+        prev()->GenParseCode(out_cc, env);
+}
+
+int RecordPaddingField::StaticSize(Env* env, int offset) const {
+    int length;
+    int target_offset;
+    int offset_in_word;
+
+    switch ( ptype_ ) {
+        case PAD_BY_LENGTH: return expr_->ConstFold(env, &length) ? length : -1;
+
+        case PAD_TO_OFFSET:
+            // If the current offset cannot be statically
+            // determined, we need to Generate code to
+            // check the offset
+            if ( offset == -1 )
+                return -1;
+
+            if ( ! expr_->ConstFold(env, &target_offset) )
+                return -1;
+
+            // If both the current and target offsets
+            // can be statically computed, we can get its
+            // static size
+            if ( offset > target_offset )
+                throw ExceptionPaddingError(this, strfmt("current offset = %d, "
+                                                         "target offset = %d",
+                                                         offset, target_offset));
+            return target_offset - offset;
+
+        case PAD_TO_NEXT_WORD:
+            if ( offset == -1 || wordsize_ == -1 )
+                return -1;
+
+            offset_in_word = offset % wordsize_;
+            return (offset_in_word == 0) ? 0 : wordsize_ - offset_in_word;
+    }
+
+    return -1;
+}
+
+void RecordPaddingField::GenFieldEnd(Output* out_cc, Env* env, const DataPtr& field_begin) {
+    ASSERT(! env->Evaluated(end_of_field_dataptr_var));
+
+    char* padding_var;
+    switch ( ptype_ ) {
+        case PAD_BY_LENGTH:
+            out_cc->println("if ( (%s) < 0 ) { // check for negative pad length", expr_->EvalExpr(out_cc, env));
+            out_cc->inc_indent();
+            out_cc->println("throw binpac::ExceptionInvalidStringLength(\"%s\", %s);", Location(),
+                            expr_->EvalExpr(out_cc, env));
+            out_cc->dec_indent();
+            out_cc->println("}");
+            out_cc->println("");
+
+            out_cc->println("const_byteptr const %s = %s + (%s);", env->LValue(end_of_field_dataptr_var),
+                            field_begin.ptr_expr(), expr_->EvalExpr(out_cc, env));
+
+            out_cc->println("// Checking out-of-bound padding for \"%s\"", field_id_str_.c_str());
+            out_cc->println("if ( %s > %s || %s < %s ) {", env->LValue(end_of_field_dataptr_var),
+                            env->RValue(end_of_data), env->LValue(end_of_field_dataptr_var), field_begin.ptr_expr());
+            out_cc->inc_indent();
+            out_cc->println("throw binpac::ExceptionOutOfBound(\"%s\",", field_id_str_.c_str());
+            out_cc->println("	(%s), ", expr_->EvalExpr(out_cc, env));
+            out_cc->println("	(%s) - (%s));", env->RValue(end_of_data), env->LValue(end_of_field_dataptr_var));
+            out_cc->dec_indent();
+            out_cc->println("}");
+            out_cc->println("");
+            break;
+
+        case PAD_TO_OFFSET:
+            out_cc->println("const_byteptr %s = %s + (%s);", env->LValue(end_of_field_dataptr_var),
+                            env->RValue(begin_of_data), expr_->EvalExpr(out_cc, env));
+            out_cc->println("if ( %s < %s ) {", env->LValue(end_of_field_dataptr_var), field_begin.ptr_expr());
+            out_cc->inc_indent();
+            out_cc->println("// throw binpac::ExceptionInvalidOffset(\"%s\", %s - %s, %s);", id_->LocName(),
+                            field_begin.ptr_expr(), env->RValue(begin_of_data), expr_->EvalExpr(out_cc, env));
+            out_cc->println("%s = %s;", env->LValue(end_of_field_dataptr_var), field_begin.ptr_expr());
+            out_cc->dec_indent();
+            out_cc->println("}");
+            out_cc->println("if ( %s > %s ) {", env->LValue(end_of_field_dataptr_var), env->RValue(end_of_data));
+            out_cc->inc_indent();
+            out_cc->println("throw binpac::ExceptionOutOfBound(\"%s\",", field_id_str_.c_str());
+            out_cc->println("	(%s), ", expr_->EvalExpr(out_cc, env));
+            out_cc->println("	(%s) - (%s));", env->RValue(end_of_data), env->LValue(end_of_field_dataptr_var));
+            out_cc->dec_indent();
+            out_cc->println("}");
+            break;
+
+        case PAD_TO_NEXT_WORD:
+            padding_var = nfmt("%s__size", id()->Name());
+            out_cc->println("int %s = (%s - %s) %% %d;", padding_var, field_begin.ptr_expr(),
+                            env->RValue(begin_of_data), wordsize_);
+            out_cc->println("%s = (%s == 0) ? 0 : %d - %s;", padding_var, padding_var, wordsize_, padding_var);
+            out_cc->println("const_byteptr const %s = %s + %s;", env->LValue(end_of_field_dataptr_var),
+                            field_begin.ptr_expr(), padding_var);
+            delete[] padding_var;
+            break;
+    }
+
+    env->SetEvaluated(end_of_field_dataptr_var);
+}
+
+bool RecordPaddingField::GenBoundaryCheck(Output* out_cc, Env* env) {
+    if ( boundary_checked_ )
+        return true;
+
+    const DataPtr& begin = getFieldBegin(out_cc, env);
+
+    char* size;
+    int ss = StaticSize(env, begin.AbsOffset(begin_of_data));
+    ASSERT(ss >= 0);
+    size = nfmt("%d", ss);
+
+    begin.GenBoundaryCheck(out_cc, env, size, field_id_str_.c_str());
+
+    delete[] size;
+
+    SetBoundaryChecked();
+    return true;
+}
+
+bool RecordPaddingField::DoTraverse(DataDepVisitor* visitor) {
+    return Field::DoTraverse(visitor) && (! expr_ || expr_->Traverse(visitor));
+}

tools/binpac/src/pac_record.h

@@ -0,0 +1,167 @@
+#ifndef pac_record_h
+#define pac_record_h
+
+#include "pac_common.h"
+#include "pac_field.h"
+#include "pac_id.h"
+#include "pac_let.h"
+#include "pac_type.h"
+
+class RecordType : public Type {
+public:
+    RecordType(RecordFieldList* fields);
+    ~RecordType() override;
+
+    bool DefineValueVar() const override;
+    string DataTypeStr() const override;
+
+    void Prepare(Env* env, int flags) override;
+
+    void GenPubDecls(Output* out, Env* env) override;
+    void GenPrivDecls(Output* out, Env* env) override;
+
+    void GenInitCode(Output* out, Env* env) override;
+    void GenCleanUpCode(Output* out, Env* env) override;
+
+    int StaticSize(Env* env) const override;
+
+    void SetBoundaryChecked() override;
+
+    const ID* parsing_dataptr_var() const;
+
+    bool IsPointerType() const override {
+        ASSERT(0);
+        return false;
+    }
+
+protected:
+    void DoGenParseCode(Output* out, Env* env, const DataPtr& data, int flags) override;
+    void GenDynamicSize(Output* out, Env* env, const DataPtr& data) override;
+
+    Type* DoClone() const override { return nullptr; }
+
+    void DoMarkIncrementalInput() override;
+
+    bool DoTraverse(DataDepVisitor* visitor) override;
+    bool ByteOrderSensitive() const override;
+
+private:
+    Field* parsing_dataptr_var_field_;
+    RecordFieldList* record_fields_;
+};
+
+// A data field of a record type. A RecordField corresponds to a
+// segment of input data, and therefore RecordField's are ordered---each
+// of them has a known previous and next field.
+
+class RecordField : public Field {
+public:
+    RecordField(FieldType tof, ID* id, Type* type);
+    ~RecordField() override;
+
+    RecordType* record_type() const { return record_type_; }
+    void set_record_type(RecordType* ty) { record_type_ = ty; }
+
+    virtual void GenParseCode(Output* out, Env* env) = 0;
+
+    RecordField* prev() const { return prev_; }
+    RecordField* next() const { return next_; }
+    void set_prev(RecordField* f) { prev_ = f; }
+    void set_next(RecordField* f) { next_ = f; }
+
+    int static_offset() const { return static_offset_; }
+    void set_static_offset(int offset) { static_offset_ = offset; }
+
+    int parsing_state_seq() const { return parsing_state_seq_; }
+    void set_parsing_state_seq(int x) { parsing_state_seq_ = x; }
+
+    virtual int StaticSize(Env* env, int offset) const = 0;
+    const char* FieldSize(Output* out, Env* env);
+    const char* FieldOffset(Output* out, Env* env);
+
+    virtual bool BoundaryChecked() const { return boundary_checked_; }
+    virtual void SetBoundaryChecked() { boundary_checked_ = true; }
+
+    virtual bool RequiresByteOrder() const = 0;
+
+    friend class RecordType;
+
+protected:
+    RecordType* record_type_;
+    RecordField* prev_;
+    RecordField* next_;
+    bool boundary_checked_;
+    int static_offset_;
+    int parsing_state_seq_;
+
+    DataPtr* begin_of_field_dataptr;
+    DataPtr* end_of_field_dataptr;
+    char* field_size_expr;
+    char* field_offset_expr;
+    ID* end_of_field_dataptr_var;
+
+    const DataPtr& getFieldBegin(Output* out_cc, Env* env);
+    const DataPtr& getFieldEnd(Output* out_cc, Env* env);
+    virtual void GenFieldEnd(Output* out, Env* env, const DataPtr& begin) = 0;
+
+    bool AttemptBoundaryCheck(Output* out_cc, Env* env);
+    virtual bool GenBoundaryCheck(Output* out_cc, Env* env) = 0;
+};
+
+class RecordDataField : public RecordField, public Evaluatable {
+public:
+    RecordDataField(ID* arg_id, Type* arg_type);
+    ~RecordDataField() override;
+
+    // Instantiates abstract class Field
+    void Prepare(Env* env) override;
+    void GenParseCode(Output* out, Env* env) override;
+
+    // Instantiates abstract class Evaluatable
+    void GenEval(Output* out, Env* env) override;
+
+    int StaticSize(Env* env, int) const override { return type()->StaticSize(env); }
+
+    void SetBoundaryChecked() override;
+
+    bool RequiresByteOrder() const override { return type()->RequiresByteOrder(); }
+    bool RequiresAnalyzerContext() const override;
+
+protected:
+    void GenFieldEnd(Output* out, Env* env, const DataPtr& begin) override;
+    bool GenBoundaryCheck(Output* out_cc, Env* env) override;
+    bool DoTraverse(DataDepVisitor* visitor) override;
+};
+
+enum PaddingType { PAD_BY_LENGTH, PAD_TO_OFFSET, PAD_TO_NEXT_WORD };
+
+class RecordPaddingField : public RecordField {
+public:
+    RecordPaddingField(ID* id, PaddingType ptype, Expr* expr);
+    ~RecordPaddingField() override;
+
+    void Prepare(Env* env) override;
+
+    void GenPubDecls(Output* out, Env* env) override { /* nothing */ }
+    void GenPrivDecls(Output* out, Env* env) override { /* nothing */ }
+
+    void GenInitCode(Output* out, Env* env) override { /* nothing */ }
+    void GenCleanUpCode(Output* out, Env* env) override { /* nothing */ }
+    void GenParseCode(Output* out, Env* env) override;
+
+    int StaticSize(Env* env, int offset) const override;
+
+    bool RequiresByteOrder() const override { return false; }
+
+protected:
+    void GenFieldEnd(Output* out, Env* env, const DataPtr& begin) override;
+    bool GenBoundaryCheck(Output* out_cc, Env* env) override;
+    bool DoTraverse(DataDepVisitor* visitor) override;
+
+private:
+    PaddingType ptype_;
+    Expr* expr_;
+    int wordsize_;
+};
+
+#endif // pac_record_h

tools/binpac/src/pac_redef.cc

@@ -0,0 +1,132 @@
+#include "pac_redef.h"
+
+#include "pac_analyzer.h"
+#include "pac_case.h"
+#include "pac_exception.h"
+#include "pac_expr.h"
+#include "pac_func.h"
+#include "pac_record.h"
+#include "pac_type.h"
+#include "pac_typedecl.h"
+
+namespace {
+
+Decl* find_decl(const ID* id) {
+    Decl* decl = Decl::LookUpDecl(id);
+    if ( ! decl ) {
+        throw Exception(id, strfmt("cannot find declaration for %s", id->Name()));
+    }
+
+    return decl;
+}
+
+} // namespace
+
+Decl* ProcessTypeRedef(const ID* id, FieldList* fieldlist) {
+    Decl* decl = find_decl(id);
+
+    if ( decl->decl_type() != Decl::TYPE ) {
+        throw Exception(id, strfmt("not a type declaration: %s", id->Name()));
+    }
+
+    TypeDecl* type_decl = static_cast<TypeDecl*>(decl);
+    ASSERT(type_decl);
+    Type* type = type_decl->type();
+
+    foreach (i, FieldList, fieldlist) {
+        Field* f = *i;
+
+        // One cannot change data layout in 'redef'.
+        // Only 'let' or 'action' can be added
+        if ( f->tof() == LET_FIELD || f->tof() == WITHINPUT_FIELD ) {
+            type->AddField(f);
+        }
+        else if ( f->tof() == RECORD_FIELD || f->tof() == PADDING_FIELD ) {
+            throw Exception(f, "cannot change data layout in redef");
+        }
+        else if ( f->tof() == CASE_FIELD ) {
+            throw Exception(f, "use 'redef case' adding cases");
+        }
+    }
+
+    return decl;
+}
+
+Decl* ProcessCaseTypeRedef(const ID* id, CaseFieldList* casefieldlist) {
+    Decl* decl = find_decl(id);
+
+    if ( decl->decl_type() != Decl::TYPE ) {
+        throw Exception(id, strfmt("not a type declaration: %s", id->Name()));
+    }
+
+    TypeDecl* type_decl = static_cast<TypeDecl*>(decl);
+    ASSERT(type_decl);
+
+    Type* type = type_decl->type();
+    if ( type->tot() != Type::CASE ) {
+        throw Exception(id, strfmt("not a case type: %s", id->Name()));
+    }
+
+    CaseType* casetype = static_cast<CaseType*>(type);
+    ASSERT(casetype);
+
+    foreach (i, CaseFieldList, casefieldlist) {
+        CaseField* f = *i;
+        casetype->AddCaseField(f);
+    }
+
+    return decl;
+}
+
+Decl* ProcessCaseExprRedef(const ID* id, CaseExprList* caseexprlist) {
+    Decl* decl = find_decl(id);
+
+    if ( decl->decl_type() != Decl::FUNC ) {
+        throw Exception(id, strfmt("not a function declaration: %s", id->Name()));
+    }
+
+    FuncDecl* func_decl = static_cast<FuncDecl*>(decl);
+    ASSERT(func_decl);
+
+    Expr* expr = func_decl->function()->expr();
+    if ( ! expr || expr->expr_type() != Expr::EXPR_CASE ) {
+        throw Exception(id, strfmt("function not defined by a case expression: %s", id->Name()));
+    }
+
+    foreach (i, CaseExprList, caseexprlist) {
+        CaseExpr* e = *i;
+        expr->AddCaseExpr(e);
+    }
+
+    return decl;
+}
+
+Decl* ProcessAnalyzerRedef(const ID* id, Decl::DeclType decl_type, AnalyzerElementList* elements) {
+    Decl* decl = find_decl(id);
+
+    if ( decl->decl_type() != decl_type ) {
+        throw Exception(id, strfmt("not a connection/flow declaration: %s", id->Name()));
+    }
+
+    AnalyzerDecl* analyzer_decl = static_cast<AnalyzerDecl*>(decl);
+    ASSERT(analyzer_decl);
+
+    analyzer_decl->AddElements(elements);
+
+    return decl;
+}
+
+Decl* ProcessTypeAttrRedef(const ID* id, AttrList* attrlist) {
+    Decl* decl = find_decl(id);
+
+    if ( decl->decl_type() != Decl::TYPE ) {
+        throw Exception(id, strfmt("not a type declaration: %s", id->Name()));
+    }
+
+    TypeDecl* type_decl = static_cast<TypeDecl*>(decl);
+    ASSERT(type_decl);
+
+    type_decl->AddAttrs(attrlist);
+
+    return decl;
+}

tools/binpac/src/pac_redef.h

@@ -0,0 +1,11 @@
+#ifndef pac_redef_h
+#define pac_redef_h
+
+#include "pac_decl.h"
+
+Decl* ProcessCaseTypeRedef(const ID* id, CaseFieldList* casefieldlist);
+Decl* ProcessCaseExprRedef(const ID* id, CaseExprList* caseexprlist);
+Decl* ProcessAnalyzerRedef(const ID* id, Decl::DeclType decl_type, AnalyzerElementList* elements);
+Decl* ProcessTypeAttrRedef(const ID* id, AttrList* attrlist);
+
+#endif // pac_redef_h

tools/binpac/src/pac_regex.cc

@@ -0,0 +1,63 @@
+#include "pac_regex.h"
+
+#include "pac_exttype.h"
+#include "pac_id.h"
+#include "pac_output.h"
+#include "pac_type.h"
+
+// Depends on the regular expression library we are using
+const char* RegEx::kREMatcherType = "RegExMatcher";
+const char* RegEx::kMatchPrefix = "MatchPrefix";
+
+string escape_char(const string& s) {
+    char* buf = new char[s.length() * 2 + 1];
+    int j = 0;
+    for ( int i = 0; i < (int)s.length(); ++i ) {
+        if ( s[i] == '\\' ) {
+            if ( i + 1 < (int)s.length() ) {
+                buf[j++] = '\\';
+                if ( s[i + 1] == '/' )
+                    buf[j - 1] = s[++i];
+                else if ( s[i + 1] == '/' || s[i + 1] == '\\' || s[i + 1] == '"' )
+                    buf[j++] = s[++i];
+                else
+                    buf[j++] = '\\';
+            }
+        }
+        else if ( s[i] == '"' ) {
+            buf[j++] = '\\';
+            buf[j++] = '"';
+        }
+        else {
+            buf[j++] = s[i];
+        }
+    }
+
+    buf[j++] = '\0';
+
+    string rval = buf;
+    delete[] buf;
+    return rval;
+}
+
+RegEx::RegEx(const string& s) {
+    str_ = escape_char(s);
+    string prefix = strfmt("%s_re_", current_decl_id->Name());
+    matcher_id_ = ID::NewAnonymousID(prefix);
+    decl_ = new RegExDecl(this);
+}
+
+RegEx::~RegEx() {}
+
+RegExDecl::RegExDecl(RegEx* regex) : Decl(regex->matcher_id(), REGEX) { regex_ = regex; }
+
+void RegExDecl::Prepare() { global_env()->AddID(id(), GLOBAL_VAR, extern_type_re_matcher); }
+
+void RegExDecl::GenForwardDeclaration(Output* out_h) {
+    out_h->println("extern %s %s;\n", RegEx::kREMatcherType, global_env()->LValue(regex_->matcher_id()));
+}
+
+void RegExDecl::GenCode(Output* out_h, Output* out_cc) {
+    out_cc->println("%s %s(\"%s\");\n", RegEx::kREMatcherType, global_env()->LValue(regex_->matcher_id()),
+                    regex_->str().c_str());
+}

tools/binpac/src/pac_regex.h

@@ -0,0 +1,39 @@
+#ifndef pac_regex_h
+#define pac_regex_h
+
+#include "pac_common.h"
+#include "pac_decl.h"
+
+class RegExDecl;
+
+class RegEx : public Object {
+public:
+    RegEx(const string& str);
+    ~RegEx();
+
+    const string& str() const { return str_; }
+    ID* matcher_id() const { return matcher_id_; }
+
+private:
+    string str_;
+    ID* matcher_id_;
+    RegExDecl* decl_;
+
+public:
+    static const char* kREMatcherType;
+    static const char* kMatchPrefix;
+};
+
+class RegExDecl : public Decl {
+public:
+    RegExDecl(RegEx* regex);
+
+    void Prepare() override;
+    void GenForwardDeclaration(Output* out_h) override;
+    void GenCode(Output* out_h, Output* out_cc) override;
+
+private:
+    RegEx* regex_;
+};
+
+#endif // pac_regex_h

tools/binpac/src/pac_scan.ll

@@ -0,0 +1,415 @@
+%top{
+// Include stdint.h at the start of the generated file. Typically
+// MSVC will include this header later, after the definitions of
+// the integral type macros. MSVC then complains that about the
+// redefinition of the types. Including stdint.h early avoids this.
+#include <stdint.h>
+}
+
+%{
+#include "pac_action.h"
+#include "pac_array.h"
+#include "pac_attr.h"
+#include "pac_case.h"
+#include "pac_common.h"
+#include "pac_conn.h"
+#include "pac_dataptr.h"
+#include "pac_dataunit.h"
+#include "pac_dbg.h"
+#include "pac_decl.h"
+#include "pac_exception.h"
+#include "pac_expr.h"
+#include "pac_flow.h"
+#include "pac_id.h"
+#include "pac_nullptr.h"
+#include "pac_number.h"
+#include "pac_output.h"
+#include "pac_param.h"
+#include "pac_parse.h"
+#include "pac_record.h"
+#include "pac_type.h"
+#include "pac_utils.h"
+
+#include <errno.h>
+#include <string_view>
+
+#ifdef _MSC_VER
+#include <filesystem>
+#else
+#include <libgen.h>
+#include <memory>
+#endif
+
+int line_number = 1;
+
+int begin_pac_primitive(int tok);
+int end_pac_primitive();
+
+int string_token(int tok)
+	{
+	yylval.str = copy_string(yytext);
+	return tok;
+	}
+
+int char_token(int tok)
+	{
+	yylval.val = yytext[0];
+	return tok;
+	}
+
+void include_file(const char *filename);
+
+std::string do_dirname(std::string_view s)
+	{
+#ifdef _MSC_VER
+	return std::filesystem::path(s).parent_path().string();
+#else
+	std::unique_ptr<char[]> tmp{new char[s.size()+1]};
+	strncpy(tmp.get(), s.data(), s.size());
+	tmp[s.size()] = '\0';
+
+	char* dn = dirname(tmp.get());
+	if ( !dn )
+		return "";
+
+	std::string res{dn};
+
+	return res;
+#endif
+	}
+%}
+
+/* EC -- embedded code state */
+/* PP -- PAC primitive state */
+/* INCL -- @include line */
+
+%s EC INCL PP RE
+
+WS	[ \t]+
+ID	[A-Za-z_][A-Za-z_0-9]*
+D	[0-9]+
+HEX	[0-9a-fA-F]+
+FILE	[^ \t\n]+
+ESCSEQ	(\\([^\n]|[0-7]{3}|x[[:xdigit:]]{2}))
+
+%option nounput
+
+%%
+
+<INITIAL>"%include"		{
+				BEGIN(INCL);
+				}
+
+<INCL>{WS}			/* skip whitespace */
+
+<INCL>{FILE}			{
+				BEGIN(INITIAL);
+				include_file(yytext);
+				}
+
+<INITIAL>"%extern{"		{
+				BEGIN(EC);
+				return TOK_LPB_EXTERN;
+				}
+<INITIAL>"%header{"		{
+				BEGIN(EC);
+				return TOK_LPB_HEADER;
+				}
+<INITIAL>"%code{"		{
+				BEGIN(EC);
+				return TOK_LPB_CODE;
+				}
+<INITIAL>"%init{"		{
+				BEGIN(EC);
+				return TOK_LPB_INIT;
+				}
+<INITIAL>"%cleanup{"		{
+				BEGIN(EC);
+				return TOK_LPB_CLEANUP;
+				}
+<INITIAL>"%member{"		{
+				BEGIN(EC);
+				return TOK_LPB_MEMBER;
+				}
+<INITIAL>"%eof{"		{
+				BEGIN(EC);
+				return TOK_LPB_EOF;
+				}
+<INITIAL>"%{"			{
+				BEGIN(EC);
+				return TOK_LPB;
+				}
+<EC>"%}"			{
+				BEGIN(INITIAL);
+				return TOK_RPB;
+				}
+
+<EC>"${"			return begin_pac_primitive(TOK_PAC_VAL);
+<EC>"$set{"			return begin_pac_primitive(TOK_PAC_SET);
+<EC>"$type{"			return begin_pac_primitive(TOK_PAC_TYPE);
+<EC>"$typeof{"			return begin_pac_primitive(TOK_PAC_TYPEOF);
+<EC>"$const_def{"		return begin_pac_primitive(TOK_PAC_CONST_DEF);
+
+<EC>"//".*			return string_token(TOK_EMBEDDED_STRING);
+<EC>.				return char_token(TOK_EMBEDDED_ATOM);
+<EC>\n				{ ++line_number; return char_token(TOK_EMBEDDED_ATOM); }
+
+<PP>"}"				return end_pac_primitive();
+
+<INITIAL,PP>\n			++line_number;
+<INITIAL>#.*			/* eat comments */
+<INITIAL,PP>{WS}		/* eat whitespace */
+
+<INITIAL>"RE/"			{
+				BEGIN(RE);
+				return TOK_BEGIN_RE;
+				}
+
+<RE>([^/\\\n]|{ESCSEQ})+	return string_token(TOK_REGEX);
+
+<RE>"/"				{
+				BEGIN(INITIAL);
+				return TOK_END_RE;
+				}
+
+<RE>[\\\n]			return yytext[0];
+
+<INITIAL>analyzer		return TOK_ANALYZER;
+<INITIAL>enum			return TOK_ENUM;
+<INITIAL>extern			return TOK_EXTERN;
+<INITIAL>flow			return TOK_FLOW;
+<INITIAL>function		return TOK_FUNCTION;
+<INITIAL>let			return TOK_LET;
+<INITIAL>refine			return TOK_REFINE;
+<INITIAL>type			return TOK_TYPE;
+
+<INITIAL>align			return TOK_ALIGN;
+<INITIAL>case			return TOK_CASE;
+<INITIAL>casefunc		return TOK_CASEFUNC;
+<INITIAL>casetype		return TOK_CASETYPE;
+<INITIAL>connection		return TOK_CONNECTION;
+<INITIAL>datagram		{
+				yylval.val = AnalyzerDataUnit::DATAGRAM;
+				return TOK_DATAUNIT;
+				}
+<INITIAL>default 		return TOK_DEFAULT;
+<INITIAL>downflow		{
+				yylval.val = AnalyzerFlow::DOWN;
+				return TOK_FLOWDIR;
+				}
+<INITIAL>flowunit		{
+				yylval.val = AnalyzerDataUnit::FLOWUNIT;
+				return TOK_DATAUNIT;
+				}
+<INITIAL>nullptr		{
+				yylval.nullp = new Nullptr();
+				return TOK_NULLPTR;
+				}
+<INITIAL>of			return TOK_OF;
+<INITIAL>offsetof 		return TOK_OFFSETOF;
+<INITIAL>padding		return TOK_PADDING;
+<INITIAL>record			return TOK_RECORD;
+<INITIAL>sizeof			return TOK_SIZEOF;
+<INITIAL>to			return TOK_TO;
+<INITIAL>typeattr		return TOK_TYPEATTR;
+<INITIAL>upflow			{
+				yylval.val = AnalyzerFlow::UP;
+				return TOK_FLOWDIR;
+				}
+<INITIAL>withcontext		return TOK_WITHCONTEXT;
+<INITIAL>withinput		return TOK_WITHINPUT;
+
+<INITIAL>&also			return TOK_ATTR_ALSO;
+<INITIAL>&byteorder		return TOK_ATTR_BYTEORDER;
+<INITIAL>&check			{
+	fprintf(stderr,
+	        "warning in %s:%d: &check is a deprecated no-op, use &enforce\n",
+	        input_filename.c_str(), line_number);
+	return TOK_ATTR_CHECK;
+	}
+<INITIAL>&chunked		return TOK_ATTR_CHUNKED;
+<INITIAL>&enforce			return TOK_ATTR_ENFORCE;
+<INITIAL>&exportsourcedata	return TOK_ATTR_EXPORTSOURCEDATA;
+<INITIAL>&if			return TOK_ATTR_IF;
+<INITIAL>&length		return TOK_ATTR_LENGTH;
+<INITIAL>&let			return TOK_ATTR_LET;
+<INITIAL>&linebreaker		return TOK_ATTR_LINEBREAKER;
+<INITIAL>&oneline		return TOK_ATTR_ONELINE;
+<INITIAL>&refcount		return TOK_ATTR_REFCOUNT;
+<INITIAL>&requires		return TOK_ATTR_REQUIRES;
+<INITIAL>&restofdata		return TOK_ATTR_RESTOFDATA;
+<INITIAL>&restofflow		return TOK_ATTR_RESTOFFLOW;
+<INITIAL>&transient		return TOK_ATTR_TRANSIENT;
+<INITIAL>&until			return TOK_ATTR_UNTIL;
+
+<INITIAL,PP>"0x"{HEX}		{
+				int n;
+				sscanf(yytext + 2, "%x", &n);
+				yylval.num = new Number(yytext, n);
+				return TOK_NUMBER;
+				}
+
+<INITIAL,PP>{D}			{
+				int n;
+				sscanf(yytext, "%d", &n);
+				yylval.num = new Number(yytext, n);
+				return TOK_NUMBER;
+				}
+
+<INITIAL,PP>{ID}(::{ID})*	{
+				yylval.id = new ID(yytext);
+				return TOK_ID;
+				}
+
+<INITIAL>"$"{ID}		{
+				yylval.id = new ID(yytext);
+				return TOK_ID;
+				}
+
+<INITIAL>\"([^\\\n\"]|{ESCSEQ})*\" return string_token(TOK_STRING);
+
+<INITIAL,PP>"=="		return TOK_EQUAL;
+<INITIAL,PP>"!="		return TOK_NEQ;
+<INITIAL,PP>">="		return TOK_GE;
+<INITIAL,PP>"<="		return TOK_LE;
+<INITIAL,PP>"<<"		return TOK_LSHIFT;
+<INITIAL,PP>">>"		return TOK_RSHIFT;
+<INITIAL,PP>"&&"		return TOK_AND;
+<INITIAL,PP>"||"		return TOK_OR;
+<INITIAL,PP>"+="		return TOK_PLUSEQ;
+<INITIAL>"->"			return TOK_RIGHTARROW;
+
+<INITIAL,PP>[\.!%*/+\-&|\^,:;<=>?()\[\]{}~]	return yytext[0];
+
+%%
+
+void begin_RE()
+	{
+	BEGIN(RE);
+	}
+
+void end_RE()
+	{
+	BEGIN(INITIAL);
+	}
+
+// The DECL state is deprecated
+void begin_decl()
+	{
+	// BEGIN(DECL);
+	}
+
+void end_decl()
+	{
+	// BEGIN(INITIAL);
+	}
+
+int begin_pac_primitive(int tok)
+	{
+	BEGIN(PP);
+	return tok;
+	}
+
+int end_pac_primitive()
+	{
+	BEGIN(EC);
+	return TOK_END_PAC;
+	}
+
+const int MAX_INCLUDE_DEPTH = 100;
+
+struct IncludeState {
+	YY_BUFFER_STATE yystate;
+	string input_filename;
+	int line_number;
+};
+
+IncludeState include_stack[MAX_INCLUDE_DEPTH];
+int include_stack_ptr = 0;
+
+void switch_to_file(FILE *fp)
+	{
+	yy_switch_to_buffer(yy_create_buffer(fp, YY_BUF_SIZE));
+	}
+
+void switch_to_file(const char *filename)
+	{
+	if ( include_stack_ptr >= MAX_INCLUDE_DEPTH )
+		{
+		fprintf( stderr, "Includes nested too deeply" );
+		exit( 1 );
+		}
+
+	IncludeState state =
+		{ YY_CURRENT_BUFFER, input_filename, line_number };
+	include_stack[include_stack_ptr++] = state;
+
+	FILE *fp = fopen(filename, "r");
+
+	if ( ! fp )
+		{
+		fprintf(stderr, "%s:%d: error: cannot include file \"%s\"\n",
+			input_filename.c_str(), line_number,filename);
+		exit( 1 );
+		}
+
+	yyin = fp;
+	input_filename = string(filename);
+	line_number = 1;
+	switch_to_file(yyin);
+	if ( ! FLAGS_quiet )
+		fprintf(stderr, "switching to file %s\n", input_filename.c_str());
+	}
+
+void include_file(const char *filename)
+	{
+	ASSERT(filename);
+
+	string full_filename;
+	if ( filename[0] == '/' )
+		full_filename = filename;
+	else if ( filename[0] == '.' )
+		{
+		string dir = do_dirname(input_filename);
+
+		if ( ! dir.empty() )
+			full_filename = dir + "/" + filename;
+		else
+			{
+			fprintf(stderr, "%s:%d error: cannot include file \"%s\": %s\n",
+					input_filename.c_str(), line_number, filename,
+					strerror(errno));
+			exit(1);
+			}
+		}
+	else
+		{
+		int i;
+		for ( i = 0; i < (int) FLAGS_include_directories.size(); ++i )
+			{
+			full_filename = FLAGS_include_directories[i] + filename;
+			DEBUG_MSG("Try include file: \"%s\"\n",
+				full_filename.c_str());
+			if ( access(full_filename.c_str(), R_OK) == 0 )
+				break;
+			}
+		if ( i >= (int) FLAGS_include_directories.size() )
+			full_filename = filename;
+		}
+
+	switch_to_file(full_filename.c_str());
+	}
+
+int yywrap()
+	{
+	yy_delete_buffer(YY_CURRENT_BUFFER);
+	--include_stack_ptr;
+	if ( include_stack_ptr < 0 )
+		return 1;
+
+	IncludeState state = include_stack[include_stack_ptr];
+	yy_switch_to_buffer(state.yystate);
+	input_filename = state.input_filename;
+	line_number = state.line_number;
+	return 0;
+	}

tools/binpac/src/pac_state.cc

@@ -0,0 +1,23 @@
+#include "pac_state.h"
+
+#include "pac_id.h"
+#include "pac_output.h"
+#include "pac_type.h"
+
+void StateVar::GenDecl(Output* out_h, Env* env) {
+    out_h->println("%s %s;", type_->DataTypeStr().c_str(), env->LValue(id_));
+}
+
+void StateVar::GenAccessFunction(Output* out_h, Env* env) {
+    out_h->println("%s %s const	{ return %s; }", type_->DataTypeConstRefStr().c_str(), env->RValue(id_),
+                   env->LValue(id_));
+}
+
+void StateVar::GenSetFunction(Output* out_h, Env* env) {
+    out_h->println("void %s(%s x) 	{ %s = x; }", set_function(id_).c_str(), type_->DataTypeConstRefStr().c_str(),
+                   env->LValue(id_));
+}
+
+void StateVar::GenInitCode(Output* out_cc, Env* env) {}
+
+void StateVar::GenCleanUpCode(Output* out_cc, Env* env) {}

tools/binpac/src/pac_state.h

@@ -0,0 +1,26 @@
+#ifndef pac_state_h
+#define pac_state_h
+
+// Classes representing analyzer states.
+
+#include "pac_common.h"
+
+class StateVar {
+public:
+    StateVar(ID* id, Type* type) : id_(id), type_(type) {}
+
+    const ID* id() const { return id_; }
+    Type* type() const { return type_; }
+
+    void GenDecl(Output* out_h, Env* env);
+    void GenAccessFunction(Output* out_h, Env* env);
+    void GenSetFunction(Output* out_h, Env* env);
+    void GenInitCode(Output* out_cc, Env* env);
+    void GenCleanUpCode(Output* out_cc, Env* env);
+
+private:
+    ID* id_;
+    Type* type_;
+};
+
+#endif // pac_state_h

tools/binpac/src/pac_strtype.cc

@@ -0,0 +1,305 @@
+#include "pac_strtype.h"
+
+#include "pac_attr.h"
+#include "pac_btype.h"
+#include "pac_cstr.h"
+#include "pac_dataptr.h"
+#include "pac_exception.h"
+#include "pac_expr.h"
+#include "pac_exttype.h"
+#include "pac_id.h"
+#include "pac_output.h"
+#include "pac_regex.h"
+#include "pac_varfield.h"
+
+const char* StringType::kStringTypeName = "bytestring";
+const char* StringType::kConstStringTypeName = "const_bytestring";
+
+StringType::StringType(StringTypeEnum anystr) : Type(STRING), type_(ANYSTR), str_(nullptr), regex_(nullptr) {
+    ASSERT(anystr == ANYSTR);
+    init();
+}
+
+StringType::StringType(ConstString* str) : Type(STRING), type_(CSTR), str_(str), regex_(nullptr) { init(); }
+
+StringType::StringType(RegEx* regex) : Type(STRING), type_(REGEX), str_(nullptr), regex_(regex) {
+    ASSERT(regex_);
+    init();
+}
+
+void StringType::init() {
+    string_length_var_field_ = nullptr;
+    elem_datatype_ = new BuiltInType(BuiltInType::UINT8);
+}
+
+StringType::~StringType() {
+    // TODO: Unref for Objects
+    // Question: why Unref?
+    //
+    // Unref(str_);
+    // Unref(regex_);
+
+    delete string_length_var_field_;
+    delete elem_datatype_;
+}
+
+Type* StringType::DoClone() const {
+    StringType* clone;
+
+    switch ( type_ ) {
+        case ANYSTR: clone = new StringType(ANYSTR); break;
+        case CSTR: clone = new StringType(str_); break;
+        case REGEX: clone = new StringType(regex_); break;
+        default: ASSERT(0); return nullptr;
+    }
+
+    return clone;
+}
+
+bool StringType::DefineValueVar() const { return true; }
+
+string StringType::DataTypeStr() const { return strfmt("%s", persistent() ? kStringTypeName : kConstStringTypeName); }
+
+Type* StringType::ElementDataType() const { return elem_datatype_; }
+
+void StringType::ProcessAttr(Attr* a) {
+    Type::ProcessAttr(a);
+
+    switch ( a->type() ) {
+        case ATTR_CHUNKED: {
+            if ( type_ != ANYSTR ) {
+                throw Exception(a,
+                                "&chunked can be applied"
+                                " to only type bytestring");
+            }
+            attr_chunked_ = true;
+            SetBoundaryChecked();
+        } break;
+
+        case ATTR_RESTOFDATA: {
+            if ( type_ != ANYSTR ) {
+                throw Exception(a,
+                                "&restofdata can be applied"
+                                " to only type bytestring");
+            }
+            attr_restofdata_ = true;
+            // As the string automatically extends to the end of
+            // data, we do not have to check boundary.
+            SetBoundaryChecked();
+        } break;
+
+        case ATTR_RESTOFFLOW: {
+            if ( type_ != ANYSTR ) {
+                throw Exception(a,
+                                "&restofflow can be applied"
+                                " to only type bytestring");
+            }
+            attr_restofflow_ = true;
+            // As the string automatically extends to the end of
+            // flow, we do not have to check boundary.
+            SetBoundaryChecked();
+        } break;
+
+        default: break;
+    }
+}
+
+void StringType::Prepare(Env* env, int flags) {
+    if ( (flags & TO_BE_PARSED) && StaticSize(env) < 0 ) {
+        ID* string_length_var = new ID(strfmt("%s_string_length", value_var() ? value_var()->Name() : "val"));
+        string_length_var_field_ = new TempVarField(string_length_var, extern_type_int->Clone());
+        string_length_var_field_->Prepare(env);
+    }
+    Type::Prepare(env, flags);
+}
+
+void StringType::GenPubDecls(Output* out_h, Env* env) { Type::GenPubDecls(out_h, env); }
+
+void StringType::GenPrivDecls(Output* out_h, Env* env) { Type::GenPrivDecls(out_h, env); }
+
+void StringType::GenInitCode(Output* out_cc, Env* env) { Type::GenInitCode(out_cc, env); }
+
+void StringType::GenCleanUpCode(Output* out_cc, Env* env) {
+    Type::GenCleanUpCode(out_cc, env);
+    if ( persistent() )
+        out_cc->println("%s.free();", env->LValue(value_var()));
+}
+
+void StringType::DoMarkIncrementalInput() {
+    if ( attr_restofflow_ ) {
+        // Do nothing
+        ASSERT(type_ == ANYSTR);
+    }
+    else {
+        Type::DoMarkIncrementalInput();
+    }
+}
+
+int StringType::StaticSize(Env* env) const {
+    switch ( type_ ) {
+        case CSTR:
+            // Use length of the unescaped string
+            return str_->unescaped().length();
+        case REGEX:
+            // TODO: static size for a regular expression?
+        case ANYSTR: return -1;
+
+        default: ASSERT(0); return -1;
+    }
+}
+
+const ID* StringType::string_length_var() const {
+    return string_length_var_field_ ? string_length_var_field_->id() : nullptr;
+}
+
+void StringType::GenDynamicSize(Output* out_cc, Env* env, const DataPtr& data) {
+    ASSERT(StaticSize(env) < 0);
+    DEBUG_MSG("Generating dynamic size for string `%s'\n", value_var()->Name());
+
+    if ( env->Evaluated(string_length_var()) )
+        return;
+
+    string_length_var_field_->GenTempDecls(out_cc, env);
+
+    switch ( type_ ) {
+        case ANYSTR: GenDynamicSizeAnyStr(out_cc, env, data); break;
+        case CSTR: ASSERT(0); break;
+        case REGEX:
+            // TODO: static size for a regular expression?
+            GenDynamicSizeRegEx(out_cc, env, data);
+            break;
+    }
+
+    if ( ! incremental_input() && AddSizeVar(out_cc, env) ) {
+        out_cc->println("%s = %s;", env->LValue(size_var()), env->RValue(string_length_var()));
+        env->SetEvaluated(size_var());
+    }
+}
+
+string StringType::GenStringSize(Output* out_cc, Env* env, const DataPtr& data) {
+    int static_size = StaticSize(env);
+    if ( static_size >= 0 )
+        return strfmt("%d", static_size);
+    GenDynamicSize(out_cc, env, data);
+    return env->RValue(string_length_var());
+}
+
+void StringType::DoGenParseCode(Output* out_cc, Env* env, const DataPtr& data, int flags) {
+    string str_size = GenStringSize(out_cc, env, data);
+
+    // Generate additional checking
+    switch ( type_ ) {
+        case CSTR: GenCheckingCStr(out_cc, env, data, str_size); break;
+        case REGEX:
+        case ANYSTR: break;
+    }
+
+    if ( ! anonymous_value_var() ) {
+        // Set the value variable
+
+        int len;
+
+        if ( type_ == ANYSTR && attr_length_expr_ && attr_length_expr_->ConstFold(env, &len) ) {
+            // can check for a negative length now
+            if ( len < 0 )
+                throw Exception(this, "negative &length on string");
+        }
+        else {
+            out_cc->println("// check for negative sizes");
+            out_cc->println("if ( %s < 0 )", str_size.c_str());
+            out_cc->println("throw binpac::ExceptionInvalidStringLength(\"%s\", %s);", Location(), str_size.c_str());
+        }
+
+        out_cc->println("%s.init(%s, %s);", env->LValue(value_var()), data.ptr_expr(), str_size.c_str());
+    }
+
+    if ( parsing_complete_var() ) {
+        out_cc->println("%s = true;", env->LValue(parsing_complete_var()));
+    }
+}
+
+void StringType::GenStringMismatch(Output* out_cc, Env* env, const DataPtr& data, string pattern) {
+    string tmp =
+        strfmt("string((const char *) (%s), (const char *) %s).c_str()", data.ptr_expr(), env->RValue(end_of_data));
+    out_cc->println("throw binpac::ExceptionStringMismatch(\"%s\", %s, %s);", Location(), pattern.c_str(), tmp.c_str());
+}
+
+void StringType::GenCheckingCStr(Output* out_cc, Env* env, const DataPtr& data, const string& str_size) {
+    // TODO: extend it for dynamic strings
+    ASSERT(type_ == CSTR);
+
+    GenBoundaryCheck(out_cc, env, data);
+
+    string str_val = str_->str();
+
+    // Compare the string and report error on mismatch
+    out_cc->println("if ( memcmp(%s, %s, %s) != 0 ) {", data.ptr_expr(), str_val.c_str(), str_size.c_str());
+    out_cc->inc_indent();
+    GenStringMismatch(out_cc, env, data, str_val);
+    out_cc->dec_indent();
+    out_cc->println("}");
+}
+
+void StringType::GenDynamicSizeRegEx(Output* out_cc, Env* env, const DataPtr& data) {
+    // string_length_var =
+    // 	matcher.match_prefix(
+    // 		begin,
+    //		end);
+
+    out_cc->println("%s = ", env->LValue(string_length_var()));
+    out_cc->inc_indent();
+
+    out_cc->println("%s.%s(", env->RValue(regex_->matcher_id()), RegEx::kMatchPrefix);
+
+    out_cc->inc_indent();
+    out_cc->println("%s,", data.ptr_expr());
+    out_cc->println("%s - %s);", env->RValue(end_of_data), data.ptr_expr());
+
+    out_cc->dec_indent();
+    out_cc->dec_indent();
+
+    env->SetEvaluated(string_length_var());
+
+    out_cc->println("if ( %s < 0 ) {", env->RValue(string_length_var()));
+    out_cc->inc_indent();
+    string tmp = strfmt("\"%s\"", regex_->str().c_str());
+    GenStringMismatch(out_cc, env, data, tmp);
+    out_cc->dec_indent();
+    out_cc->println("}");
+}
+
+void StringType::GenDynamicSizeAnyStr(Output* out_cc, Env* env, const DataPtr& data) {
+    ASSERT(type_ == ANYSTR);
+
+    if ( attr_restofdata_ || attr_oneline_ ) {
+        out_cc->println("%s = (%s) - (%s);", env->LValue(string_length_var()), env->RValue(end_of_data),
+                        data.ptr_expr());
+    }
+    else if ( attr_restofflow_ ) {
+        out_cc->println("%s = (%s) - (%s);", env->LValue(string_length_var()), env->RValue(end_of_data),
+                        data.ptr_expr());
+    }
+    else if ( attr_length_expr_ ) {
+        out_cc->println("%s = %s;", env->LValue(string_length_var()), attr_length_expr_->EvalExpr(out_cc, env));
+    }
+    else {
+        throw Exception(this, "cannot determine length of bytestring");
+    }
+
+    env->SetEvaluated(string_length_var());
+}
+
+bool StringType::DoTraverse(DataDepVisitor* visitor) {
+    if ( ! Type::DoTraverse(visitor) )
+        return false;
+
+    switch ( type_ ) {
+        case ANYSTR:
+        case CSTR:
+        case REGEX: break;
+    }
+
+    return true;
+}
+
+void StringType::static_init() { Type::AddPredefinedType("bytestring", new StringType(ANYSTR)); }