Merge remote-tracking branch 'origin/master' into topic/vladg/kerberos

src/3rdparty

@@ -1 +1 @@
-Subproject commit 7e15efe9d28d46bfa662fcdd1cbb15ce1db285c9
+Subproject commit 6a429e79bbaf0fcc11eff5f639bfb9d1f62be6f2

src/CMakeLists.txt

@@ -161,6 +161,14 @@ add_subdirectory(iosource)
 add_subdirectory(logging)
 add_subdirectory(probabilistic)
 
+if ( ENABLE_BROKER )
+    add_subdirectory(broker)
+else ()
+    # Just to satisfy coverage unit tests until new Broker-based
+    # communication is enabled by default.
+    add_subdirectory(broker-dummy)
+endif ()
+
 set(bro_SUBDIRS
     # Order is important here.
     ${bro_PLUGIN_LIBS}
@@ -261,6 +269,7 @@ set(bro_SRCS
     ChunkedIO.cc
     CompHash.cc
     Conn.cc
+    ConvertUTF.c
     DFA.cc
     DbgBreakpoint.cc
     DbgHelp.cc
@@ -408,6 +417,18 @@ add_dependencies(bro bif_loader_plugins)
 # Install *.bif.bro.
 install(DIRECTORY ${CMAKE_BINARY_DIR}/scripts/base/bif DESTINATION ${BRO_SCRIPT_INSTALL_PATH}/base)
 
+# Create plugin directory at install time.
+install(DIRECTORY DESTINATION ${BRO_PLUGIN_INSTALL_PATH})
+
 # Make clean removes the bif directory.
 set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES ${CMAKE_BINARY_DIR}/scripts/base/bif)
 
+# Remove some stale files and scripts that previous Bro versions put in
+# place, yet make confuse us now. This makes upgrading easier.
+install(CODE "
+   file(REMOVE_RECURSE
+       ${BRO_SCRIPT_INSTALL_PATH}/base/frameworks/logging/writers/dataseries.bro
+       ${BRO_SCRIPT_INSTALL_PATH}/base/frameworks/logging/writers/elasticsearch.bro
+       ${BRO_SCRIPT_INSTALL_PATH}/policy/tuning/logs-to-elasticsearch.bro
+   )
+")

src/ChunkedIO.cc

@@ -137,20 +137,6 @@ bool ChunkedIOFd::Write(Chunk* chunk)
 		chunk->len, fmt_bytes(chunk->data, min((uint32)20, chunk->len)));
 #endif
 
-	// Reject if our queue of pending chunks is way too large. Otherwise,
-	// memory could fill up if the other side doesn't read.
-	if ( stats.pending > MAX_BUFFERED_CHUNKS )
-		{
-		DBG_LOG(DBG_CHUNKEDIO, "write queue full");
-
-#ifdef DEBUG_COMMUNICATION
-		AddToBuffer("<false:write-queue-full>", false);
-#endif
-
-		errno = ENOSPC;
-		return false;
-		}
-
 #ifdef DEBUG_COMMUNICATION
 	AddToBuffer(chunk, false);
 #endif
@@ -627,7 +613,7 @@ bool ChunkedIOFd::IsIdle()
 
 bool ChunkedIOFd::IsFillingUp()
 	{
-	return stats.pending > MAX_BUFFERED_CHUNKS_SOFT;
+	return stats.pending > chunked_io_buffer_soft_cap;
 	}
 
 iosource::FD_Set ChunkedIOFd::ExtraReadFDs() const
@@ -838,15 +824,6 @@ bool ChunkedIOSSL::Write(Chunk* chunk)
 		chunk->len, fmt_bytes(chunk->data, 20));
 #endif
 
-	// Reject if our queue of pending chunks is way too large. Otherwise,
-	// memory could fill up if the other side doesn't read.
-	if ( stats.pending > MAX_BUFFERED_CHUNKS )
-		{
-		DBG_LOG(DBG_CHUNKEDIO, "write queue full");
-		errno = ENOSPC;
-		return false;
-		}
-
 	// Queue it.
 	++stats.pending;
 	Queue* q = new Queue;

src/ChunkedIO.h

@@ -221,13 +221,6 @@ private:
 	// than BUFFER_SIZE.
 	static const uint32 FLAG_PARTIAL = 0x80000000;
 
-	// We report that we're filling up when there are more than this number
-	// of pending chunks.
-	static const uint32 MAX_BUFFERED_CHUNKS_SOFT = 400000;
-
-	// Maximum number of chunks we store in memory before rejecting writes.
-	static const uint32 MAX_BUFFERED_CHUNKS = 500000;
-
 	char* read_buffer;
 	uint32 read_len;
 	uint32 read_pos;
@@ -275,8 +268,6 @@ public:
 	virtual void Stats(char* buffer, int length);
 
 private:
-	// Maximum number of chunks we store in memory before rejecting writes.
-	static const uint32 MAX_BUFFERED_CHUNKS = 500000;
 
 	// Only returns true if all data has been read. If not, call
 	// it again with the same parameters as long as error is not

src/Conn.h

@@ -263,6 +263,9 @@ public:
 
 	void CheckFlowLabel(bool is_orig, uint32 flow_label);
 
+	uint32 GetOrigFlowLabel() { return orig_flow_label; }
+	uint32 GetRespFlowLabel() { return resp_flow_label; }
+
 protected:
 
 	Connection()	{ persistent = 0; }

src/ConvertUTF.c

@@ -0,0 +1,755 @@
+/*===--- ConvertUTF.c - Universal Character Names conversions ---------------===
+ *
+ *                     The LLVM Compiler Infrastructure
+ *
+ * This file is distributed under the University of Illinois Open Source
+ * License:
+ *
+ *    University of Illinois/NCSA
+ *    Open Source License
+ *
+ *    Copyright (c) 2003-2014 University of Illinois at Urbana-Champaign.
+ *    All rights reserved.
+ *
+ *    Developed by:
+ *
+ *        LLVM Team
+ *
+ *        University of Illinois at Urbana-Champaign
+ *
+ *        http://llvm.org
+ *
+ *    Permission is hereby granted, free of charge, to any person
+ *    obtaining a copy of this software and associated documentation
+ *    files (the "Software"), to deal with the Software without
+ *    restriction, including without limitation the rights to use,
+ *    copy, modify, merge, publish, distribute, sublicense, and/or
+ *    sell copies of the Software, and to permit persons to whom the
+ *    Software is furnished to do so, subject to the following
+ *    conditions:
+ *
+ *        * Redistributions of source code must retain the above
+ *          copyright notice, this list of conditions and the
+ *          following disclaimers.
+ *
+ *        * Redistributions in binary form must reproduce the
+ *          above copyright notice, this list of conditions and
+ *          the following disclaimers in the documentation and/or
+ *          other materials provided with the distribution.
+ *
+ *        * Neither the names of the LLVM Team, University of
+ *          Illinois at Urbana-Champaign, nor the names of its
+ *          contributors may be used to endorse or promote
+ *          products derived from this Software without specific
+ *          prior written permission.
+ *
+ *    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *    OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *    NONINFRINGEMENT.  IN NO EVENT SHALL THE CONTRIBUTORS OR
+ *    COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ *    ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ *    USE OR OTHER DEALINGS WITH THE SOFTWARE.
+ *
+ *===------------------------------------------------------------------------=*/
+/*
+ * Copyright 2001-2004 Unicode, Inc.
+ * 
+ * Disclaimer
+ * 
+ * This source code is provided as is by Unicode, Inc. No claims are
+ * made as to fitness for any particular purpose. No warranties of any
+ * kind are expressed or implied. The recipient agrees to determine
+ * applicability of information provided. If this file has been
+ * purchased on magnetic or optical media from Unicode, Inc., the
+ * sole remedy for any claim will be exchange of defective media
+ * within 90 days of receipt.
+ * 
+ * Limitations on Rights to Redistribute This Code
+ * 
+ * Unicode, Inc. hereby grants the right to freely use the information
+ * supplied in this file in the creation of products supporting the
+ * Unicode Standard, and to make copies of this file in any form
+ * for internal or external distribution as long as this notice
+ * remains attached.
+ */
+
+/* ---------------------------------------------------------------------
+
+    Conversions between UTF32, UTF-16, and UTF-8. Source code file.
+    Author: Mark E. Davis, 1994.
+    Rev History: Rick McGowan, fixes & updates May 2001.
+    Sept 2001: fixed const & error conditions per
+        mods suggested by S. Parent & A. Lillich.
+    June 2002: Tim Dodd added detection and handling of incomplete
+        source sequences, enhanced error detection, added casts
+        to eliminate compiler warnings.
+    July 2003: slight mods to back out aggressive FFFE detection.
+    Jan 2004: updated switches in from-UTF8 conversions.
+    Oct 2004: updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions.
+
+    See the header file "ConvertUTF.h" for complete documentation.
+
+------------------------------------------------------------------------ */
+
+
+#include "ConvertUTF.h"
+#ifdef CVTUTF_DEBUG
+#include <stdio.h>
+#endif
+#include <assert.h>
+
+static const int halfShift  = 10; /* used for shifting by 10 bits */
+
+static const UTF32 halfBase = 0x0010000UL;
+static const UTF32 halfMask = 0x3FFUL;
+
+#define UNI_SUR_HIGH_START  (UTF32)0xD800
+#define UNI_SUR_HIGH_END    (UTF32)0xDBFF
+#define UNI_SUR_LOW_START   (UTF32)0xDC00
+#define UNI_SUR_LOW_END     (UTF32)0xDFFF
+#define false      0
+#define true        1
+
+/* --------------------------------------------------------------------- */
+
+/*
+ * Index into the table below with the first byte of a UTF-8 sequence to
+ * get the number of trailing bytes that are supposed to follow it.
+ * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is
+ * left as-is for anyone who may want to do such conversion, which was
+ * allowed in earlier algorithms.
+ */
+static const char trailingBytesForUTF8[256] = {
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+    2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5
+};
+
+/*
+ * Magic values subtracted from a buffer value during UTF8 conversion.
+ * This table contains as many values as there might be trailing bytes
+ * in a UTF-8 sequence.
+ */
+static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL, 
+                     0x03C82080UL, 0xFA082080UL, 0x82082080UL };
+
+/*
+ * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed
+ * into the first byte, depending on how many bytes follow.  There are
+ * as many entries in this table as there are UTF-8 sequence types.
+ * (I.e., one byte sequence, two byte... etc.). Remember that sequencs
+ * for *legal* UTF-8 will be 4 or fewer bytes total.
+ */
+static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
+
+/* --------------------------------------------------------------------- */
+
+/* The interface converts a whole buffer to avoid function-call overhead.
+ * Constants have been gathered. Loops & conditionals have been removed as
+ * much as possible for efficiency, in favor of drop-through switches.
+ * (See "Note A" at the bottom of the file for equivalent code.)
+ * If your compiler supports it, the "isLegalUTF8" call can be turned
+ * into an inline function.
+ */
+
+
+/* --------------------------------------------------------------------- */
+
+ConversionResult ConvertUTF32toUTF16 (
+        const UTF32** sourceStart, const UTF32* sourceEnd, 
+        UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) {
+    ConversionResult result = conversionOK;
+    const UTF32* source = *sourceStart;
+    UTF16* target = *targetStart;
+    while (source < sourceEnd) {
+        UTF32 ch;
+        if (target >= targetEnd) {
+            result = targetExhausted; break;
+        }
+        ch = *source++;
+        if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */
+            /* UTF-16 surrogate values are illegal in UTF-32; 0xffff or 0xfffe are both reserved values */
+            if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
+                if (flags == strictConversion) {
+                    --source; /* return to the illegal value itself */
+                    result = sourceIllegal;
+                    break;
+                } else {
+                    *target++ = UNI_REPLACEMENT_CHAR;
+                }
+            } else {
+                *target++ = (UTF16)ch; /* normal case */
+            }
+        } else if (ch > UNI_MAX_LEGAL_UTF32) {
+            if (flags == strictConversion) {
+                result = sourceIllegal;
+            } else {
+                *target++ = UNI_REPLACEMENT_CHAR;
+            }
+        } else {
+            /* target is a character in range 0xFFFF - 0x10FFFF. */
+            if (target + 1 >= targetEnd) {
+                --source; /* Back up source pointer! */
+                result = targetExhausted; break;
+            }
+            ch -= halfBase;
+            *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START);
+            *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START);
+        }
+    }
+    *sourceStart = source;
+    *targetStart = target;
+    return result;
+}
+
+/* --------------------------------------------------------------------- */
+
+ConversionResult ConvertUTF16toUTF32 (
+        const UTF16** sourceStart, const UTF16* sourceEnd, 
+        UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) {
+    ConversionResult result = conversionOK;
+    const UTF16* source = *sourceStart;
+    UTF32* target = *targetStart;
+    UTF32 ch, ch2;
+    while (source < sourceEnd) {
+        const UTF16* oldSource = source; /*  In case we have to back up because of target overflow. */
+        ch = *source++;
+        /* If we have a surrogate pair, convert to UTF32 first. */
+        if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
+            /* If the 16 bits following the high surrogate are in the source buffer... */
+            if (source < sourceEnd) {
+                ch2 = *source;
+                /* If it's a low surrogate, convert to UTF32. */
+                if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
+                    ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
+                        + (ch2 - UNI_SUR_LOW_START) + halfBase;
+                    ++source;
+                } else if (flags == strictConversion) { /* it's an unpaired high surrogate */
+                    --source; /* return to the illegal value itself */
+                    result = sourceIllegal;
+                    break;
+                }
+            } else { /* We don't have the 16 bits following the high surrogate. */
+                --source; /* return to the high surrogate */
+                result = sourceExhausted;
+                break;
+            }
+        } else if (flags == strictConversion) {
+            /* UTF-16 surrogate values are illegal in UTF-32 */
+            if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) {
+                --source; /* return to the illegal value itself */
+                result = sourceIllegal;
+                break;
+            }
+        }
+        if (target >= targetEnd) {
+            source = oldSource; /* Back up source pointer! */
+            result = targetExhausted; break;
+        }
+        *target++ = ch;
+    }
+    *sourceStart = source;
+    *targetStart = target;
+#ifdef CVTUTF_DEBUG
+if (result == sourceIllegal) {
+    fprintf(stderr, "ConvertUTF16toUTF32 illegal seq 0x%04x,%04x\n", ch, ch2);
+    fflush(stderr);
+}
+#endif
+    return result;
+}
+ConversionResult ConvertUTF16toUTF8 (
+        const UTF16** sourceStart, const UTF16* sourceEnd, 
+        UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) {
+    ConversionResult result = conversionOK;
+    const UTF16* source = *sourceStart;
+    UTF8* target = *targetStart;
+    while (source < sourceEnd) {
+        UTF32 ch;
+        unsigned short bytesToWrite = 0;
+        const UTF32 byteMask = 0xBF;
+        const UTF32 byteMark = 0x80; 
+        const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */
+        ch = *source++;
+        /* If we have a surrogate pair, convert to UTF32 first. */
+        if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
+            /* If the 16 bits following the high surrogate are in the source buffer... */
+            if (source < sourceEnd) {
+                UTF32 ch2 = *source;
+                /* If it's a low surrogate, convert to UTF32. */
+                if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
+                    ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
+                        + (ch2 - UNI_SUR_LOW_START) + halfBase;
+                    ++source;
+                } else if (flags == strictConversion) { /* it's an unpaired high surrogate */
+                    --source; /* return to the illegal value itself */
+                    result = sourceIllegal;
+                    break;
+                }
+            } else { /* We don't have the 16 bits following the high surrogate. */
+                --source; /* return to the high surrogate */
+                result = sourceExhausted;
+                break;
+            }
+        } else if (flags == strictConversion) {
+            /* UTF-16 surrogate values are illegal in UTF-32 */
+            if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) {
+                --source; /* return to the illegal value itself */
+                result = sourceIllegal;
+                break;
+            }
+        }
+        /* Figure out how many bytes the result will require */
+        if (ch < (UTF32)0x80) {      bytesToWrite = 1;
+        } else if (ch < (UTF32)0x800) {     bytesToWrite = 2;
+        } else if (ch < (UTF32)0x10000) {   bytesToWrite = 3;
+        } else if (ch < (UTF32)0x110000) {  bytesToWrite = 4;
+        } else {                            bytesToWrite = 3;
+                                            ch = UNI_REPLACEMENT_CHAR;
+        }
+
+        target += bytesToWrite;
+        if (target > targetEnd) {
+            source = oldSource; /* Back up source pointer! */
+            target -= bytesToWrite; result = targetExhausted; break;
+        }
+        switch (bytesToWrite) { /* note: everything falls through. */
+            case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
+            case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
+            case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
+            case 1: *--target =  (UTF8)(ch | firstByteMark[bytesToWrite]);
+        }
+        target += bytesToWrite;
+    }
+    *sourceStart = source;
+    *targetStart = target;
+    return result;
+}
+
+/* --------------------------------------------------------------------- */
+
+ConversionResult ConvertUTF32toUTF8 (
+        const UTF32** sourceStart, const UTF32* sourceEnd, 
+        UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) {
+    ConversionResult result = conversionOK;
+    const UTF32* source = *sourceStart;
+    UTF8* target = *targetStart;
+    while (source < sourceEnd) {
+        UTF32 ch;
+        unsigned short bytesToWrite = 0;
+        const UTF32 byteMask = 0xBF;
+        const UTF32 byteMark = 0x80; 
+        ch = *source++;
+        if (flags == strictConversion ) {
+            /* UTF-16 surrogate values are illegal in UTF-32 */
+            if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
+                --source; /* return to the illegal value itself */
+                result = sourceIllegal;
+                break;
+            }
+        }
+        /*
+         * Figure out how many bytes the result will require. Turn any
+         * illegally large UTF32 things (> Plane 17) into replacement chars.
+         */
+        if (ch < (UTF32)0x80) {      bytesToWrite = 1;
+        } else if (ch < (UTF32)0x800) {     bytesToWrite = 2;
+        } else if (ch < (UTF32)0x10000) {   bytesToWrite = 3;
+        } else if (ch <= UNI_MAX_LEGAL_UTF32) {  bytesToWrite = 4;
+        } else {                            bytesToWrite = 3;
+                                            ch = UNI_REPLACEMENT_CHAR;
+                                            result = sourceIllegal;
+        }
+        
+        target += bytesToWrite;
+        if (target > targetEnd) {
+            --source; /* Back up source pointer! */
+            target -= bytesToWrite; result = targetExhausted; break;
+        }
+        switch (bytesToWrite) { /* note: everything falls through. */
+            case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
+            case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
+            case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
+            case 1: *--target = (UTF8) (ch | firstByteMark[bytesToWrite]);
+        }
+        target += bytesToWrite;
+    }
+    *sourceStart = source;
+    *targetStart = target;
+    return result;
+}
+
+/* --------------------------------------------------------------------- */
+
+/*
+ * Utility routine to tell whether a sequence of bytes is legal UTF-8.
+ * This must be called with the length pre-determined by the first byte.
+ * If not calling this from ConvertUTF8to*, then the length can be set by:
+ *  length = trailingBytesForUTF8[*source]+1;
+ * and the sequence is illegal right away if there aren't that many bytes
+ * available.
+ * If presented with a length > 4, this returns false.  The Unicode
+ * definition of UTF-8 goes up to 4-byte sequences.
+ */
+
+static Boolean isLegalUTF8(const UTF8 *source, int length) {
+    UTF8 a;
+    const UTF8 *srcptr = source+length;
+    switch (length) {
+    default: return false;
+        /* Everything else falls through when "true"... */
+    case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
+    case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
+    case 2: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
+
+        switch (*source) {
+            /* no fall-through in this inner switch */
+            case 0xE0: if (a < 0xA0) return false; break;
+            case 0xED: if (a > 0x9F) return false; break;
+            case 0xF0: if (a < 0x90) return false; break;
+            case 0xF4: if (a > 0x8F) return false; break;
+            default:   if (a < 0x80) return false;
+        }
+
+    case 1: if (*source >= 0x80 && *source < 0xC2) return false;
+    }
+    if (*source > 0xF4) return false;
+    return true;
+}
+
+/* --------------------------------------------------------------------- */
+
+/*
+ * Exported function to return whether a UTF-8 sequence is legal or not.
+ * This is not used here; it's just exported.
+ */
+Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd) {
+    int length = trailingBytesForUTF8[*source]+1;
+    if (length > sourceEnd - source) {
+        return false;
+    }
+    return isLegalUTF8(source, length);
+}
+
+/* --------------------------------------------------------------------- */
+
+static unsigned
+findMaximalSubpartOfIllFormedUTF8Sequence(const UTF8 *source,
+                                          const UTF8 *sourceEnd) {
+  UTF8 b1, b2, b3;
+
+  assert(!isLegalUTF8Sequence(source, sourceEnd));
+
+  /*
+   * Unicode 6.3.0, D93b:
+   *
+   *   Maximal subpart of an ill-formed subsequence: The longest code unit
+   *   subsequence starting at an unconvertible offset that is either:
+   *   a. the initial subsequence of a well-formed code unit sequence, or
+   *   b. a subsequence of length one.
+   */
+
+  if (source == sourceEnd)
+    return 0;
+
+  /*
+   * Perform case analysis.  See Unicode 6.3.0, Table 3-7. Well-Formed UTF-8
+   * Byte Sequences.
+   */
+
+  b1 = *source;
+  ++source;
+  if (b1 >= 0xC2 && b1 <= 0xDF) {
+    /*
+     * First byte is valid, but we know that this code unit sequence is
+     * invalid, so the maximal subpart has to end after the first byte.
+     */
+    return 1;
+  }
+
+  if (source == sourceEnd)
+    return 1;
+
+  b2 = *source;
+  ++source;
+
+  if (b1 == 0xE0) {
+    return (b2 >= 0xA0 && b2 <= 0xBF) ? 2 : 1;
+  }
+  if (b1 >= 0xE1 && b1 <= 0xEC) {
+    return (b2 >= 0x80 && b2 <= 0xBF) ? 2 : 1;
+  }
+  if (b1 == 0xED) {
+    return (b2 >= 0x80 && b2 <= 0x9F) ? 2 : 1;
+  }
+  if (b1 >= 0xEE && b1 <= 0xEF) {
+    return (b2 >= 0x80 && b2 <= 0xBF) ? 2 : 1;
+  }
+  if (b1 == 0xF0) {
+    if (b2 >= 0x90 && b2 <= 0xBF) {
+      if (source == sourceEnd)
+        return 2;
+
+      b3 = *source;
+      return (b3 >= 0x80 && b3 <= 0xBF) ? 3 : 2;
+    }
+    return 1;
+  }
+  if (b1 >= 0xF1 && b1 <= 0xF3) {
+    if (b2 >= 0x80 && b2 <= 0xBF) {
+      if (source == sourceEnd)
+        return 2;
+
+      b3 = *source;
+      return (b3 >= 0x80 && b3 <= 0xBF) ? 3 : 2;
+    }
+    return 1;
+  }
+  if (b1 == 0xF4) {
+    if (b2 >= 0x80 && b2 <= 0x8F) {
+      if (source == sourceEnd)
+        return 2;
+
+      b3 = *source;
+      return (b3 >= 0x80 && b3 <= 0xBF) ? 3 : 2;
+    }
+    return 1;
+  }
+
+  assert((b1 >= 0x80 && b1 <= 0xC1) || b1 >= 0xF5);
+  /*
+   * There are no valid sequences that start with these bytes.  Maximal subpart
+   * is defined to have length 1 in these cases.
+   */
+  return 1;
+}
+
+/* --------------------------------------------------------------------- */
+
+/*
+ * Exported function to return the total number of bytes in a codepoint
+ * represented in UTF-8, given the value of the first byte.
+ */
+unsigned getNumBytesForUTF8(UTF8 first) {
+  return trailingBytesForUTF8[first] + 1;
+}
+
+/* --------------------------------------------------------------------- */
+
+/*
+ * Exported function to return whether a UTF-8 string is legal or not.
+ * This is not used here; it's just exported.
+ */
+Boolean isLegalUTF8String(const UTF8 **source, const UTF8 *sourceEnd) {
+    while (*source != sourceEnd) {
+        int length = trailingBytesForUTF8[**source] + 1;
+        if (length > sourceEnd - *source || !isLegalUTF8(*source, length))
+            return false;
+        *source += length;
+    }
+    return true;
+}
+
+/* --------------------------------------------------------------------- */
+
+ConversionResult ConvertUTF8toUTF16 (
+        const UTF8** sourceStart, const UTF8* sourceEnd, 
+        UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) {
+    ConversionResult result = conversionOK;
+    const UTF8* source = *sourceStart;
+    UTF16* target = *targetStart;
+    while (source < sourceEnd) {
+        UTF32 ch = 0;
+        unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
+        if (extraBytesToRead >= sourceEnd - source) {
+            result = sourceExhausted; break;
+        }
+        /* Do this check whether lenient or strict */
+        if (!isLegalUTF8(source, extraBytesToRead+1)) {
+            result = sourceIllegal;
+            break;
+        }
+        /*
+         * The cases all fall through. See "Note A" below.
+         */
+        switch (extraBytesToRead) {
+            case 5: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
+            case 4: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
+            case 3: ch += *source++; ch <<= 6;
+            case 2: ch += *source++; ch <<= 6;
+            case 1: ch += *source++; ch <<= 6;
+            case 0: ch += *source++;
+        }
+        ch -= offsetsFromUTF8[extraBytesToRead];
+
+        if (target >= targetEnd) {
+            source -= (extraBytesToRead+1); /* Back up source pointer! */
+            result = targetExhausted; break;
+        }
+        if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */
+            /* UTF-16 surrogate values are illegal in UTF-32 */
+            if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
+                if (flags == strictConversion) {
+                    source -= (extraBytesToRead+1); /* return to the illegal value itself */
+                    result = sourceIllegal;
+                    break;
+                } else {
+                    *target++ = UNI_REPLACEMENT_CHAR;
+                }
+            } else {
+                *target++ = (UTF16)ch; /* normal case */
+            }
+        } else if (ch > UNI_MAX_UTF16) {
+            if (flags == strictConversion) {
+                result = sourceIllegal;
+                source -= (extraBytesToRead+1); /* return to the start */
+                break; /* Bail out; shouldn't continue */
+            } else {
+                *target++ = UNI_REPLACEMENT_CHAR;
+            }
+        } else {
+            /* target is a character in range 0xFFFF - 0x10FFFF. */
+            if (target + 1 >= targetEnd) {
+                source -= (extraBytesToRead+1); /* Back up source pointer! */
+                result = targetExhausted; break;
+            }
+            ch -= halfBase;
+            *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START);
+            *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START);
+        }
+    }
+    *sourceStart = source;
+    *targetStart = target;
+    return result;
+}
+
+/* --------------------------------------------------------------------- */
+
+static ConversionResult ConvertUTF8toUTF32Impl(
+        const UTF8** sourceStart, const UTF8* sourceEnd, 
+        UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags,
+        Boolean InputIsPartial) {
+    ConversionResult result = conversionOK;
+    const UTF8* source = *sourceStart;
+    UTF32* target = *targetStart;
+    while (source < sourceEnd) {
+        UTF32 ch = 0;
+        unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
+        if (extraBytesToRead >= sourceEnd - source) {
+            if (flags == strictConversion || InputIsPartial) {
+                result = sourceExhausted;
+                break;
+            } else {
+                result = sourceIllegal;
+
+                /*
+                 * Replace the maximal subpart of ill-formed sequence with
+                 * replacement character.
+                 */
+                source += findMaximalSubpartOfIllFormedUTF8Sequence(source,
+                                                                    sourceEnd);
+                *target++ = UNI_REPLACEMENT_CHAR;
+                continue;
+            }
+        }
+        if (target >= targetEnd) {
+            result = targetExhausted; break;
+        }
+
+        /* Do this check whether lenient or strict */
+        if (!isLegalUTF8(source, extraBytesToRead+1)) {
+            result = sourceIllegal;
+            if (flags == strictConversion) {
+                /* Abort conversion. */
+                break;
+            } else {
+                /*
+                 * Replace the maximal subpart of ill-formed sequence with
+                 * replacement character.
+                 */
+                source += findMaximalSubpartOfIllFormedUTF8Sequence(source,
+                                                                    sourceEnd);
+                *target++ = UNI_REPLACEMENT_CHAR;
+                continue;
+            }
+        }
+        /*
+         * The cases all fall through. See "Note A" below.
+         */
+        switch (extraBytesToRead) {
+            case 5: ch += *source++; ch <<= 6;
+            case 4: ch += *source++; ch <<= 6;
+            case 3: ch += *source++; ch <<= 6;
+            case 2: ch += *source++; ch <<= 6;
+            case 1: ch += *source++; ch <<= 6;
+            case 0: ch += *source++;
+        }
+        ch -= offsetsFromUTF8[extraBytesToRead];
+
+        if (ch <= UNI_MAX_LEGAL_UTF32) {
+            /*
+             * UTF-16 surrogate values are illegal in UTF-32, and anything
+             * over Plane 17 (> 0x10FFFF) is illegal.
+             */
+            if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
+                if (flags == strictConversion) {
+                    source -= (extraBytesToRead+1); /* return to the illegal value itself */
+                    result = sourceIllegal;
+                    break;
+                } else {
+                    *target++ = UNI_REPLACEMENT_CHAR;
+                }
+            } else {
+                *target++ = ch;
+            }
+        } else { /* i.e., ch > UNI_MAX_LEGAL_UTF32 */
+            result = sourceIllegal;
+            *target++ = UNI_REPLACEMENT_CHAR;
+        }
+    }
+    *sourceStart = source;
+    *targetStart = target;
+    return result;
+}
+
+ConversionResult ConvertUTF8toUTF32Partial(const UTF8 **sourceStart,
+                                           const UTF8 *sourceEnd,
+                                           UTF32 **targetStart,
+                                           UTF32 *targetEnd,
+                                           ConversionFlags flags) {
+  return ConvertUTF8toUTF32Impl(sourceStart, sourceEnd, targetStart, targetEnd,
+                                flags, /*InputIsPartial=*/true);
+}
+
+ConversionResult ConvertUTF8toUTF32(const UTF8 **sourceStart,
+                                    const UTF8 *sourceEnd, UTF32 **targetStart,
+                                    UTF32 *targetEnd, ConversionFlags flags) {
+  return ConvertUTF8toUTF32Impl(sourceStart, sourceEnd, targetStart, targetEnd,
+                                flags, /*InputIsPartial=*/false);
+}
+
+/* ---------------------------------------------------------------------
+
+    Note A.
+    The fall-through switches in UTF-8 reading code save a
+    temp variable, some decrements & conditionals.  The switches
+    are equivalent to the following loop:
+        {
+            int tmpBytesToRead = extraBytesToRead+1;
+            do {
+                ch += *source++;
+                --tmpBytesToRead;
+                if (tmpBytesToRead) ch <<= 6;
+            } while (tmpBytesToRead > 0);
+        }
+    In UTF-8 writing code, the switches on "bytesToWrite" are
+    similarly unrolled loops.
+
+   --------------------------------------------------------------------- */

src/ConvertUTF.h

@@ -0,0 +1,230 @@
+/*===--- ConvertUTF.h - Universal Character Names conversions ---------------===
+ *
+ *                     The LLVM Compiler Infrastructure
+ *
+ * This file is distributed under the University of Illinois Open Source
+ * License:
+ *
+ *    University of Illinois/NCSA
+ *    Open Source License
+ *
+ *    Copyright (c) 2003-2014 University of Illinois at Urbana-Champaign.
+ *    All rights reserved.
+ *
+ *    Developed by:
+ *
+ *        LLVM Team
+ *
+ *        University of Illinois at Urbana-Champaign
+ *
+ *        http://llvm.org
+ *
+ *    Permission is hereby granted, free of charge, to any person
+ *    obtaining a copy of this software and associated documentation
+ *    files (the "Software"), to deal with the Software without
+ *    restriction, including without limitation the rights to use,
+ *    copy, modify, merge, publish, distribute, sublicense, and/or
+ *    sell copies of the Software, and to permit persons to whom the
+ *    Software is furnished to do so, subject to the following
+ *    conditions:
+ *
+ *        * Redistributions of source code must retain the above
+ *          copyright notice, this list of conditions and the
+ *          following disclaimers.
+ *
+ *        * Redistributions in binary form must reproduce the
+ *          above copyright notice, this list of conditions and
+ *          the following disclaimers in the documentation and/or
+ *          other materials provided with the distribution.
+ *
+ *        * Neither the names of the LLVM Team, University of
+ *          Illinois at Urbana-Champaign, nor the names of its
+ *          contributors may be used to endorse or promote
+ *          products derived from this Software without specific
+ *          prior written permission.
+ *
+ *    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *    OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *    NONINFRINGEMENT.  IN NO EVENT SHALL THE CONTRIBUTORS OR
+ *    COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ *    ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ *    USE OR OTHER DEALINGS WITH THE SOFTWARE.
+ *
+ *==------------------------------------------------------------------------==*/
+/*
+ * Copyright 2001-2004 Unicode, Inc.
+ *
+ * Disclaimer
+ *
+ * This source code is provided as is by Unicode, Inc. No claims are
+ * made as to fitness for any particular purpose. No warranties of any
+ * kind are expressed or implied. The recipient agrees to determine
+ * applicability of information provided. If this file has been
+ * purchased on magnetic or optical media from Unicode, Inc., the
+ * sole remedy for any claim will be exchange of defective media
+ * within 90 days of receipt.
+ *
+ * Limitations on Rights to Redistribute This Code
+ *
+ * Unicode, Inc. hereby grants the right to freely use the information
+ * supplied in this file in the creation of products supporting the
+ * Unicode Standard, and to make copies of this file in any form
+ * for internal or external distribution as long as this notice
+ * remains attached.
+ */
+
+/* ---------------------------------------------------------------------
+
+    Conversions between UTF32, UTF-16, and UTF-8.  Header file.
+
+    Several funtions are included here, forming a complete set of
+    conversions between the three formats.  UTF-7 is not included
+    here, but is handled in a separate source file.
+
+    Each of these routines takes pointers to input buffers and output
+    buffers.  The input buffers are const.
+
+    Each routine converts the text between *sourceStart and sourceEnd,
+    putting the result into the buffer between *targetStart and
+    targetEnd. Note: the end pointers are *after* the last item: e.g.
+    *(sourceEnd - 1) is the last item.
+
+    The return result indicates whether the conversion was successful,
+    and if not, whether the problem was in the source or target buffers.
+    (Only the first encountered problem is indicated.)
+
+    After the conversion, *sourceStart and *targetStart are both
+    updated to point to the end of last text successfully converted in
+    the respective buffers.
+
+    Input parameters:
+        sourceStart - pointer to a pointer to the source buffer.
+                The contents of this are modified on return so that
+                it points at the next thing to be converted.
+        targetStart - similarly, pointer to pointer to the target buffer.
+        sourceEnd, targetEnd - respectively pointers to the ends of the
+                two buffers, for overflow checking only.
+
+    These conversion functions take a ConversionFlags argument. When this
+    flag is set to strict, both irregular sequences and isolated surrogates
+    will cause an error.  When the flag is set to lenient, both irregular
+    sequences and isolated surrogates are converted.
+
+    Whether the flag is strict or lenient, all illegal sequences will cause
+    an error return. This includes sequences such as: <F4 90 80 80>, <C0 80>,
+    or <A0> in UTF-8, and values above 0x10FFFF in UTF-32. Conformant code
+    must check for illegal sequences.
+
+    When the flag is set to lenient, characters over 0x10FFFF are converted
+    to the replacement character; otherwise (when the flag is set to strict)
+    they constitute an error.
+
+    Output parameters:
+        The value "sourceIllegal" is returned from some routines if the input
+        sequence is malformed.  When "sourceIllegal" is returned, the source
+        value will point to the illegal value that caused the problem. E.g.,
+        in UTF-8 when a sequence is malformed, it points to the start of the
+        malformed sequence.
+
+    Author: Mark E. Davis, 1994.
+    Rev History: Rick McGowan, fixes & updates May 2001.
+         Fixes & updates, Sept 2001.
+
+------------------------------------------------------------------------ */
+
+#ifndef LLVM_SUPPORT_CONVERTUTF_H
+#define LLVM_SUPPORT_CONVERTUTF_H
+
+/* ---------------------------------------------------------------------
+    The following 4 definitions are compiler-specific.
+    The C standard does not guarantee that wchar_t has at least
+    16 bits, so wchar_t is no less portable than unsigned short!
+    All should be unsigned values to avoid sign extension during
+    bit mask & shift operations.
+------------------------------------------------------------------------ */
+
+typedef unsigned int    UTF32;  /* at least 32 bits */
+typedef unsigned short  UTF16;  /* at least 16 bits */
+typedef unsigned char   UTF8;   /* typically 8 bits */
+typedef unsigned char   Boolean; /* 0 or 1 */
+
+/* Some fundamental constants */
+#define UNI_REPLACEMENT_CHAR (UTF32)0x0000FFFD
+#define UNI_MAX_BMP (UTF32)0x0000FFFF
+#define UNI_MAX_UTF16 (UTF32)0x0010FFFF
+#define UNI_MAX_UTF32 (UTF32)0x7FFFFFFF
+#define UNI_MAX_LEGAL_UTF32 (UTF32)0x0010FFFF
+
+#define UNI_MAX_UTF8_BYTES_PER_CODE_POINT 4
+
+#define UNI_UTF16_BYTE_ORDER_MARK_NATIVE  0xFEFF
+#define UNI_UTF16_BYTE_ORDER_MARK_SWAPPED 0xFFFE
+
+typedef enum {
+  conversionOK,           /* conversion successful */
+  sourceExhausted,        /* partial character in source, but hit end */
+  targetExhausted,        /* insuff. room in target for conversion */
+  sourceIllegal           /* source sequence is illegal/malformed */
+} ConversionResult;
+
+typedef enum {
+  strictConversion = 0,
+  lenientConversion
+} ConversionFlags;
+
+/* This is for C++ and does no harm in C */
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+ConversionResult ConvertUTF8toUTF16 (
+  const UTF8** sourceStart, const UTF8* sourceEnd,
+  UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags);
+
+/**
+ * Convert a partial UTF8 sequence to UTF32.  If the sequence ends in an
+ * incomplete code unit sequence, returns \c sourceExhausted.
+ */
+ConversionResult ConvertUTF8toUTF32Partial(
+  const UTF8** sourceStart, const UTF8* sourceEnd,
+  UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags);
+
+/**
+ * Convert a partial UTF8 sequence to UTF32.  If the sequence ends in an
+ * incomplete code unit sequence, returns \c sourceIllegal.
+ */
+ConversionResult ConvertUTF8toUTF32(
+  const UTF8** sourceStart, const UTF8* sourceEnd,
+  UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags);
+
+ConversionResult ConvertUTF16toUTF8 (
+  const UTF16** sourceStart, const UTF16* sourceEnd,
+  UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags);
+
+ConversionResult ConvertUTF32toUTF8 (
+  const UTF32** sourceStart, const UTF32* sourceEnd,
+  UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags);
+
+ConversionResult ConvertUTF16toUTF32 (
+  const UTF16** sourceStart, const UTF16* sourceEnd,
+  UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags);
+
+ConversionResult ConvertUTF32toUTF16 (
+  const UTF32** sourceStart, const UTF32* sourceEnd,
+  UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags);
+
+Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd);
+
+Boolean isLegalUTF8String(const UTF8 **source, const UTF8 *sourceEnd);
+
+unsigned getNumBytesForUTF8(UTF8 firstByte);
+
+#ifdef __cplusplus
+}
+#endif
+
+/* --------------------------------------------------------------------- */
+
+#endif

src/DFA.cc

@@ -7,8 +7,6 @@
 #include "EquivClass.h"
 #include "DFA.h"
 
-int dfa_state_cache_size = 10000;
-
 unsigned int DFA_State::transition_counter = 0;
 
 DFA_State::DFA_State(int arg_state_num, const EquivClass* ec,
@@ -292,9 +290,8 @@ unsigned int DFA_State::Size()
 		+ (centry ? padded_sizeof(CacheEntry) : 0);
 	}
 
-DFA_State_Cache::DFA_State_Cache(int arg_maxsize)
+DFA_State_Cache::DFA_State_Cache()
 	{
-	maxsize = arg_maxsize;
 	hits = misses = 0;
 	}
 
@@ -402,7 +399,7 @@ DFA_Machine::DFA_Machine(NFA_Machine* n, EquivClass* arg_ec)
 
 	ec = arg_ec;
 
-	dfa_state_cache = new DFA_State_Cache(dfa_state_cache_size);
+	dfa_state_cache = new DFA_State_Cache();
 
 	NFA_state_list* ns = new NFA_state_list;
 	ns->append(n->FirstState());

src/DFA.h

@@ -15,8 +15,6 @@ class DFA_State;
 
 #include "NFA.h"
 
-extern int dfa_state_cache_size;
-
 class DFA_Machine;
 class DFA_State;
 struct CacheEntry;
@@ -78,7 +76,7 @@ struct CacheEntry {
 
 class DFA_State_Cache {
 public:
-	DFA_State_Cache(int maxsize);
+	DFA_State_Cache();
 	~DFA_State_Cache();
 
 	// If the caller stores the handle, it has to call Ref() on it.
@@ -105,8 +103,6 @@ public:
 	void GetStats(Stats* s);
 
 private:
-	int maxsize;
-
 	int hits;	// Statistics
 	int misses;
 

src/DebugLogger.cc

@@ -19,7 +19,7 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = {
 	{ "logging", 0, false }, {"input", 0, false },
 	{ "threading", 0, false }, { "file_analysis", 0, false },
 	{ "plugins", 0, false }, { "broxygen", 0, false },
-	{ "pktio", 0, false}
+	{ "pktio", 0, false }, { "broker", 0, false }
 };
 
 DebugLogger::DebugLogger(const char* filename)
@@ -55,32 +55,81 @@ DebugLogger::~DebugLogger()
 		fclose(file);
 	}
 
+void DebugLogger::ShowStreamsHelp()
+	{
+	fprintf(stderr, "\n");
+	fprintf(stderr, "Enable debug output into debug.log with -B <streams>.\n");
+	fprintf(stderr, "<streams> is a comma-separated list of streams to enable.\n");
+	fprintf(stderr, "\n");
+	fprintf(stderr, "Available streams:\n");
+
+	for ( int i = 0; i < NUM_DBGS; ++i )
+		fprintf(stderr,"  %s\n", streams[i].prefix);
+
+	fprintf(stderr, "\n");
+	fprintf(stderr, "  plugin-<plugin-name>   (replace '::' in name with '-'; e.g., '-B plugin-Bro-Netmap')\n");
+	fprintf(stderr, "\n");
+	fprintf(stderr, "Pseudo streams\n");
+	fprintf(stderr, "  verbose  Increase verbosity.\n");
+	fprintf(stderr, "  all      Enable all streams at maximum verbosity.\n");
+	fprintf(stderr, "\n");
+	}
+
 void DebugLogger::EnableStreams(const char* s)
 	{
-	char* tmp = copy_string(s);
 	char* brkt;
+	char* tmp = copy_string(s);
 	char* tok = strtok(tmp, ",");
 
 	while ( tok )
 		{
+		if ( strcasecmp("all", tok) == 0 )
+			{
+			for ( int i = 0; i < NUM_DBGS; ++i )
+				{
+				streams[i].enabled = true;
+				enabled_streams.insert(streams[i].prefix);
+				}
+
+			verbose = true;
+			goto next;
+			}
+
+		if ( strcasecmp("verbose", tok) == 0 )
+			{
+			verbose = true;
+			goto next;
+			}
+
+		if ( strcasecmp("help", tok) == 0 )
+			{
+			ShowStreamsHelp();
+			exit(0);
+			}
+
+		if ( strncmp(tok, "plugin-", strlen("plugin-")) == 0 )
+			{
+			// Cannot verify this at this time, plugins may not
+			// have been loaded.
+			enabled_streams.insert(tok);
+			goto next;
+			}
+
 		int i;
+
 		for ( i = 0; i < NUM_DBGS; ++i )
+			{
 			if ( strcasecmp(streams[i].prefix, tok) == 0 )
 				{
 				streams[i].enabled = true;
-				break;
+				enabled_streams.insert(tok);
+				goto next;
 				}
-
-		if ( i == NUM_DBGS )
-			{
-			if ( strcasecmp("verbose", tok) == 0 )
-				verbose = true;
-			else if ( strncmp(tok, "plugin-", 7) != 0 )
-				reporter->FatalError("unknown debug stream %s\n", tok);
 			}
 
-		enabled_streams.insert(tok);
+		reporter->FatalError("unknown debug stream '%s', try -B help.\n", tok);
 
+next:
 		tok = strtok(0, ",");
 		}
 

src/DebugLogger.h

@@ -32,6 +32,7 @@ enum DebugStream {
 	DBG_PLUGINS,	// Plugin system
 	DBG_BROXYGEN,	// Broxygen
 	DBG_PKTIO,	// Packet sources and dumpers.
+	DBG_BROKER,	// Broker communication
 
 	NUM_DBGS // Has to be last
 };
@@ -77,6 +78,8 @@ public:
 	void SetVerbose(bool arg_verbose)	{ verbose = arg_verbose; }
 	bool IsVerbose() const			{ return verbose; }
 
+	void ShowStreamsHelp();
+
 private:
 	FILE* file;
 	bool verbose;

src/EventHandler.cc

@@ -5,6 +5,11 @@
 #include "RemoteSerializer.h"
 #include "NetVar.h"
 
+#ifdef ENABLE_BROKER
+#include "broker/Manager.h"
+#include "broker/Data.h"
+#endif
+
 EventHandler::EventHandler(const char* arg_name)
 	{
 	name = copy_string(arg_name);
@@ -26,7 +31,12 @@ EventHandler::operator bool() const
 	{
 	return enabled && ((local && local->HasBodies())
 			   || receivers.length()
-			   || generate_always);
+			   || generate_always
+#ifdef ENABLE_BROKER
+			   || ! auto_remote_send.empty()
+			   // TODO: and require a subscriber interested in a topic or unsolicited flags?
+#endif
+	                   );
 	}
 
 FuncType* EventHandler::FType()
@@ -73,6 +83,46 @@ void EventHandler::Call(val_list* vl, bool no_remote)
 			SerialInfo info(remote_serializer);
 			remote_serializer->SendCall(&info, receivers[i], name, vl);
 			}
+
+#ifdef ENABLE_BROKER
+
+		if ( ! auto_remote_send.empty() )
+			{
+			// TODO: also short-circuit based on interested subscribers/flags?
+			broker::message msg;
+			msg.reserve(vl->length() + 1);
+			msg.emplace_back(Name());
+			bool valid_args = true;
+
+			for ( auto i = 0; i < vl->length(); ++i )
+				{
+				auto opt_data = bro_broker::val_to_data((*vl)[i]);
+
+				if ( opt_data )
+					msg.emplace_back(move(*opt_data));
+				else
+					{
+					valid_args = false;
+					auto_remote_send.clear();
+					reporter->Error("failed auto-remote event '%s', disabled",
+					                Name());
+					break;
+					}
+				}
+
+			if ( valid_args )
+				{
+				for ( auto it = auto_remote_send.begin();
+				      it != auto_remote_send.end(); ++it )
+					{
+					if ( std::next(it) == auto_remote_send.end() )
+						broker_mgr->Event(it->first, move(msg), it->second);
+					else
+						broker_mgr->Event(it->first, msg, it->second);
+					}
+				}
+			}
+#endif
 		}
 
 	if ( local )

src/EventHandler.h

@@ -4,7 +4,8 @@
 #define EVENTHANDLER
 
 #include <assert.h>
-
+#include <map>
+#include <string>
 #include "List.h"
 #include "BroList.h"
 
@@ -28,6 +29,18 @@ public:
 	void AddRemoteHandler(SourceID peer);
 	void RemoveRemoteHandler(SourceID peer);
 
+#ifdef ENABLE_BROKER
+	void AutoRemote(std::string topic, int flags)
+		{
+		auto_remote_send[std::move(topic)] = flags;
+		}
+
+	void AutoRemoteStop(const std::string& topic)
+		{
+		auto_remote_send.erase(topic);
+		}
+#endif
+
 	void Call(val_list* vl, bool no_remote = false);
 
 	// Returns true if there is at least one local or remote handler.
@@ -67,6 +80,10 @@ private:
 	declare(List, SourceID);
 	typedef List(SourceID) receiver_list;
 	receiver_list receivers;
+
+#ifdef ENABLE_BROKER
+	std::map<std::string, int> auto_remote_send;	// topic -> flags
+#endif
 };
 
 // Encapsulates a ptr to an event handler to overload the boolean operator.

src/Expr.cc

@@ -2599,6 +2599,39 @@ bool AssignExpr::TypeCheck(attr_list* attrs)
 
 	if ( ! same_type(op1->Type(), op2->Type()) )
 		{
+		if ( bt1 == TYPE_TABLE && bt2 == TYPE_TABLE )
+			{
+			if ( op2->Tag() == EXPR_SET_CONSTRUCTOR )
+				{
+				// Some elements in constructor list must not match, see if
+				// we can create a new constructor now that the expected type
+				// of LHS is known and let it do coercions where possible.
+				SetConstructorExpr* sce = dynamic_cast<SetConstructorExpr*>(op2);
+				ListExpr* ctor_list = dynamic_cast<ListExpr*>(sce->Op());
+				attr_list* attr_copy = 0;
+
+				if ( sce->Attrs() )
+					{
+					attr_list* a = sce->Attrs()->Attrs();
+					attrs = new attr_list;
+					loop_over_list(*a, i)
+						attrs->append((*a)[i]);
+					}
+
+				int errors_before = reporter->Errors();
+				op2 = new SetConstructorExpr(ctor_list, attr_copy, op1->Type());
+				int errors_after = reporter->Errors();
+
+				if ( errors_after > errors_before )
+					{
+					ExprError("type clash in assignment");
+					return false;
+					}
+
+				return true;
+				}
+			}
+
 		ExprError("type clash in assignment");
 		return false;
 		}

src/Func.cc

@@ -54,6 +54,7 @@ const Expr* calling_expr = 0;
 bool did_builtin_init = false;
 
 vector<Func*> Func::unique_ids;
+static const std::pair<bool, Val*> empty_hook_result(false, NULL);
 
 Func::Func() : scope(0), type(0)
 	{
@@ -245,20 +246,31 @@ TraversalCode Func::Traverse(TraversalCallback* cb) const
 	HANDLE_TC_STMT_POST(tc);
 	}
 
-Val* Func::HandlePluginResult(Val* plugin_result, val_list* args, function_flavor flavor) const
+std::pair<bool, Val*> Func::HandlePluginResult(std::pair<bool, Val*> plugin_result, val_list* args, function_flavor flavor) const
 	{
-	// Helper function factoring out this code from BroFunc:Call() for better
-	// readability.
+	// Helper function factoring out this code from BroFunc:Call() for
+	// better readability.
+
+	if( ! plugin_result.first )
+		{
+		if( plugin_result.second )
+			reporter->InternalError("plugin set processed flag to false but actually returned a value");
+
+		// The plugin result hasn't been processed yet (read: fall
+		// into ::Call method).
+		return plugin_result;
+		}
 
 	switch ( flavor ) {
 	case FUNC_FLAVOR_EVENT:
-		Unref(plugin_result);
-		plugin_result = 0;
+		if( plugin_result.second )
+			reporter->InternalError("plugin returned non-void result for event %s", this->Name());
+
 		break;
 
 	case FUNC_FLAVOR_HOOK:
-		if ( plugin_result->Type()->Tag() != TYPE_BOOL )
-			reporter->InternalError("plugin returned non-bool for hook");
+		if ( plugin_result.second->Type()->Tag() != TYPE_BOOL )
+			reporter->InternalError("plugin returned non-bool for hook %s", this->Name());
 
 		break;
 
@@ -268,14 +280,14 @@ Val* Func::HandlePluginResult(Val* plugin_result, val_list* args, function_flavo
 
 		if ( (! yt) || yt->Tag() == TYPE_VOID )
 			{
-			Unref(plugin_result);
-			plugin_result = 0;
+			if( plugin_result.second )
+				reporter->InternalError("plugin returned non-void result for void method %s", this->Name());
 			}
 
-		else
+		else if ( plugin_result.second && plugin_result.second->Type()->Tag() != yt->Tag() && yt->Tag() != TYPE_ANY)
 			{
-			if ( plugin_result->Type()->Tag() != yt->Tag() )
-				reporter->InternalError("plugin returned wrong type for function call");
+			reporter->InternalError("plugin returned wrong type (got %d, expecting %d) for %s",
+						plugin_result.second->Type()->Tag(), yt->Tag(), this->Name());
 			}
 
 		break;
@@ -331,10 +343,15 @@ Val* BroFunc::Call(val_list* args, Frame* parent) const
 	if ( sample_logger )
 		sample_logger->FunctionSeen(this);
 
-	Val* plugin_result = PLUGIN_HOOK_WITH_RESULT(HOOK_CALL_FUNCTION, HookCallFunction(this, args), 0);
+	std::pair<bool, Val*> plugin_result = PLUGIN_HOOK_WITH_RESULT(HOOK_CALL_FUNCTION, HookCallFunction(this, parent, args), empty_hook_result);
 
-	if ( plugin_result )
-		return HandlePluginResult(plugin_result, args, Flavor());
+	plugin_result = HandlePluginResult(plugin_result, args, Flavor());
+
+	if( plugin_result.first )
+		{
+		Val *result = plugin_result.second;
+		return result;
+		}
 
 	if ( bodies.empty() )
 		{
@@ -425,11 +442,11 @@ Val* BroFunc::Call(val_list* args, Frame* parent) const
 	// Warn if the function returns something, but we returned from
 	// the function without an explicit return, or without a value.
 	else if ( FType()->YieldType() && FType()->YieldType()->Tag() != TYPE_VOID &&
-	     (flow != FLOW_RETURN /* we fell off the end */ ||
-	      ! result /* explicit return with no result */) &&
-	     ! f->HasDelayed() )
+		 (flow != FLOW_RETURN /* we fell off the end */ ||
+		  ! result /* explicit return with no result */) &&
+		 ! f->HasDelayed() )
 		reporter->Warning("non-void function returns without a value: %s",
-		                  Name());
+				  Name());
 
 	if ( result && g_trace_state.DoTrace() )
 		{
@@ -548,10 +565,15 @@ Val* BuiltinFunc::Call(val_list* args, Frame* parent) const
 	if ( sample_logger )
 		sample_logger->FunctionSeen(this);
 
-	Val* plugin_result = PLUGIN_HOOK_WITH_RESULT(HOOK_CALL_FUNCTION, HookCallFunction(this, args), 0);
+	std::pair<bool, Val*> plugin_result = PLUGIN_HOOK_WITH_RESULT(HOOK_CALL_FUNCTION, HookCallFunction(this, parent, args), empty_hook_result);
 
-	if ( plugin_result )
-		return HandlePluginResult(plugin_result, args, FUNC_FLAVOR_FUNCTION);
+	plugin_result = HandlePluginResult(plugin_result, args, FUNC_FLAVOR_FUNCTION);
+
+	if ( plugin_result.first )
+		{
+		Val *result = plugin_result.second;
+		return result;
+		}
 
 	if ( g_trace_state.DoTrace() )
 		{

src/Func.h

@@ -3,6 +3,8 @@
 #ifndef func_h
 #define func_h
 
+#include <utility>
+
 #include "BroList.h"
 #include "Obj.h"
 #include "Debug.h"
@@ -71,7 +73,7 @@ protected:
 	Func();
 
 	// Helper function for handling result of plugin hook.
-	Val* HandlePluginResult(Val* plugin_result, val_list* args, function_flavor flavor) const;
+	std::pair<bool, Val*> HandlePluginResult(std::pair<bool, Val*> plugin_result, val_list* args, function_flavor flavor) const;
 
 	DECLARE_ABSTRACT_SERIAL(Func);
 

src/Net.cc

@@ -34,6 +34,10 @@
 #include "iosource/PktDumper.h"
 #include "plugin/Manager.h"
 
+#ifdef ENABLE_BROKER
+#include "broker/Manager.h"
+#endif
+
 extern "C" {
 #include "setsignal.h"
 };
@@ -315,6 +319,11 @@ void net_run()
 			}
 #endif
 		current_iosrc = src;
+		bool communication_enabled = using_communication;
+
+#ifdef ENABLE_BROKER
+		communication_enabled |= broker_mgr->Enabled();
+#endif
 
 		if ( src )
 			src->Process();	// which will call net_packet_dispatch()
@@ -332,7 +341,7 @@ void net_run()
 				}
 			}
 
-		else if ( (have_pending_timers || using_communication) &&
+		else if ( (have_pending_timers || communication_enabled) &&
 			  ! pseudo_realtime )
 			{
 			// Take advantage of the lull to get up to
@@ -347,7 +356,7 @@ void net_run()
 			// us a lot of idle time, but doesn't delay near-term
 			// timers too much.  (Delaying them somewhat is okay,
 			// since Bro timers are not high-precision anyway.)
-			if ( ! using_communication )
+			if ( ! communication_enabled )
 				usleep(100000);
 			else
 				usleep(1000);

src/NetVar.cc

@@ -178,6 +178,7 @@ RecordType* peer;
 int forward_remote_state_changes;
 int forward_remote_events;
 int remote_check_sync_consistency;
+bro_uint_t chunked_io_buffer_soft_cap;
 
 StringVal* ssl_ca_certificate;
 StringVal* ssl_private_key;
@@ -276,6 +277,7 @@ void init_general_global_var()
 	forward_remote_events = opt_internal_int("forward_remote_events");
 	remote_check_sync_consistency =
 		opt_internal_int("remote_check_sync_consistency");
+	chunked_io_buffer_soft_cap = opt_internal_unsigned("chunked_io_buffer_soft_cap");
 
 	ssl_ca_certificate = internal_val("ssl_ca_certificate")->AsStringVal();
 	ssl_private_key = internal_val("ssl_private_key")->AsStringVal();

src/NetVar.h

@@ -181,6 +181,7 @@ extern RecordType* peer;
 extern int forward_remote_state_changes;
 extern int forward_remote_events;
 extern int remote_check_sync_consistency;
+extern bro_uint_t chunked_io_buffer_soft_cap;
 
 extern StringVal* ssl_ca_certificate;
 extern StringVal* ssl_private_key;

src/RE.cc

@@ -20,9 +20,6 @@ int case_insensitive = 0;
 extern int RE_parse(void);
 extern void RE_set_input(const char* str);
 
-// If true, the set-wise matching always returns false - for benchmarking.
-extern int rule_bench;
-
 Specific_RE_Matcher::Specific_RE_Matcher(match_type arg_mt, int arg_multiline)
 : equiv_class(NUM_SYM)
 	{
@@ -279,9 +276,6 @@ inline void RE_Match_State::AddMatches(const AcceptingSet& as,
 bool RE_Match_State::Match(const u_char* bv, int n,
 				bool bol, bool eol, bool clear)
 	{
-	if ( rule_bench > 0 )
-		return false;
-
 	if ( current_pos == -1 )
 		{
 		// First call to Match().

src/RemoteSerializer.cc

@@ -542,6 +542,9 @@ RemoteSerializer::RemoteSerializer()
 	current_msgtype = 0;
 	current_args = 0;
 	source_peer = 0;
+
+	// Register as a "dont-count" source first, we may change that later.
+	iosource_mgr->Register(this, true);
 	}
 
 RemoteSerializer::~RemoteSerializer()
@@ -571,8 +574,6 @@ void RemoteSerializer::Enable()
 
 	Fork();
 
-	iosource_mgr->Register(this);
-
 	Log(LogInfo, fmt("communication started, parent pid is %d, child pid is %d", getpid(), child_pid));
 	initialized = 1;
 	}
@@ -612,6 +613,9 @@ void RemoteSerializer::Fork()
 	if ( child_pid )
 		return;
 
+	// Register as a "does-count" source now.
+	iosource_mgr->Register(this, false);
+
 	// If we are re-forking, remove old entries
 	loop_over_list(peers, i)
 		RemovePeer(peers[i]);
@@ -3460,7 +3464,8 @@ void SocketComm::Run()
 		int a = select(max_fd + 1, &fd_read, &fd_write, &fd_except, 0);
 
 		if ( selects % 100000 == 0 )
-			Log(fmt("selects=%ld canwrites=%ld", selects, canwrites));
+			Log(fmt("selects=%ld canwrites=%ld pending=%lu",
+			        selects, canwrites, io->Stats()->pending));
 
 		if ( a < 0 )
 			// Ignore errors for now.

src/Reporter.cc

@@ -123,6 +123,19 @@ void Reporter::ExprRuntimeError(const Expr* expr, const char* fmt, ...)
 	throw InterpreterException();
 	}
 
+void Reporter::RuntimeError(const Location* location, const char* fmt, ...)
+	{
+	++errors;
+	PushLocation(location);
+	va_list ap;
+	va_start(ap, fmt);
+	FILE* out = errors_to_stderr ? stderr : 0;
+	DoLog("runtime error", reporter_error, out, 0, 0, true, true, "", fmt, ap);
+	va_end(ap);
+	PopLocation();
+	throw InterpreterException();
+	}
+
 void Reporter::InternalError(const char* fmt, ...)
 	{
 	va_list ap;

src/Reporter.h

@@ -73,6 +73,10 @@ public:
 	// function will not return but raise an InterpreterException.
 	void ExprRuntimeError(const Expr* expr, const char* fmt, ...);
 
+	// Report a runtime error in evaluating a Bro script expression. This
+	// function will not return but raise an InterpreterException.
+	void RuntimeError(const Location* location, const char* fmt, ...);
+
 	// Report a traffic weirdness, i.e., an unexpected protocol situation
 	// that may lead to incorrectly processing a connnection.
 	void Weird(const char* name);	// Raises net_weird().

src/RuleMatcher.cc

@@ -577,9 +577,6 @@ RuleFileMagicState* RuleMatcher::InitFileMagic() const
 	{
 	RuleFileMagicState* state = new RuleFileMagicState();
 
-	if ( rule_bench == 3 )
-		return state;
-
 	loop_over_list(root->psets[Rule::FILE_MAGIC], i)
 		{
 		RuleHdrTest::PatternSet* set = root->psets[Rule::FILE_MAGIC][i];
@@ -630,9 +627,6 @@ RuleMatcher::MIME_Matches* RuleMatcher::Match(RuleFileMagicState* state,
 		return rval;
 		}
 
-	if ( rule_bench >= 2 )
-		return rval;
-
 #ifdef DEBUG
 	if ( debug_logger.IsEnabled(DBG_RULES) )
 		{
@@ -712,9 +706,6 @@ RuleEndpointState* RuleMatcher::InitEndpoint(analyzer::Analyzer* analyzer,
 	RuleEndpointState* state =
 		new RuleEndpointState(analyzer, from_orig, opposite, pia);
 
-	if ( rule_bench == 3 )
-		return state;
-
 	rule_hdr_test_list tests;
 	tests.append(root);
 
@@ -837,9 +828,6 @@ void RuleMatcher::Match(RuleEndpointState* state, Rule::PatternType type,
 	// for 'accepted' (that depends on the average number of matching
 	// patterns).
 
-	if ( rule_bench >= 2 )
-		return;
-
 	bool newmatch = false;
 
 #ifdef DEBUG
@@ -956,9 +944,6 @@ void RuleMatcher::Match(RuleEndpointState* state, Rule::PatternType type,
 
 void RuleMatcher::FinishEndpoint(RuleEndpointState* state)
 	{
-	if ( rule_bench == 3 )
-		return;
-
 	// Send EOL to payload matchers.
 	Match(state, Rule::PAYLOAD, (const u_char *) "", 0, false, true, false);
 
@@ -1110,15 +1095,9 @@ void RuleMatcher::ExecRule(Rule* rule, RuleEndpointState* state, bool eos)
 
 void RuleMatcher::ClearEndpointState(RuleEndpointState* state)
 	{
-	if ( rule_bench == 3 )
-		return;
+	state->payload_size = -1;
 
 	ExecPureRules(state, 1);
-	state->payload_size = -1;
-	state->matched_by_patterns.clear();
-	loop_over_list(state->matched_text, i)
-		delete state->matched_text[i];
-	state->matched_text.clear();
 
 	loop_over_list(state->matchers, j)
 		state->matchers[j]->state->Clear();
@@ -1126,9 +1105,6 @@ void RuleMatcher::ClearEndpointState(RuleEndpointState* state)
 
 void RuleMatcher::ClearFileMagicState(RuleFileMagicState* state) const
 	{
-	if ( rule_bench == 3 )
-		return;
-
 	loop_over_list(state->matchers, j)
 		state->matchers[j]->state->Clear();
 	}
@@ -1496,8 +1472,12 @@ void RuleMatcherState::ClearMatchState(bool orig)
 	if ( ! rule_matcher )
 		return;
 
-	if ( orig_match_state )
-		rule_matcher->ClearEndpointState(orig_match_state);
-	if ( resp_match_state )
+	if ( orig )
+		{
+		if ( orig_match_state )
+			rule_matcher->ClearEndpointState(orig_match_state);
+		}
+
+	else if ( resp_match_state )
 		rule_matcher->ClearEndpointState(resp_match_state);
 	}

src/RuleMatcher.h

@@ -22,8 +22,6 @@
 
 //#define MATCHER_PRINT_STATS
 
-extern int rule_bench;
-
 // Parser interface:
 
 extern void rules_error(const char* msg);

src/SerialTypes.h

@@ -113,6 +113,8 @@ SERIAL_VAL(TOPK_VAL, 20)
 SERIAL_VAL(BLOOMFILTER_VAL, 21)
 SERIAL_VAL(CARDINALITY_VAL, 22)
 SERIAL_VAL(X509_VAL, 23)
+SERIAL_VAL(COMM_STORE_HANDLE_VAL, 24)
+SERIAL_VAL(COMM_DATA_VAL, 25)
 
 #define SERIAL_EXPR(name, val) SERIAL_CONST(name, val, EXPR)
 SERIAL_EXPR(EXPR, 1)
@@ -181,6 +183,7 @@ SERIAL_STMT(INIT_STMT, 17)
 SERIAL_STMT(NULL_STMT, 18)
 SERIAL_STMT(WHEN_STMT, 19)
 SERIAL_STMT(FALLTHROUGH_STMT, 20)
+SERIAL_STMT(WHILE_STMT, 21)
 
 #define SERIAL_TYPE(name, val) SERIAL_CONST(name, val, BRO_TYPE)
 SERIAL_TYPE(BRO_TYPE, 1)

src/Sessions.cc

@@ -466,6 +466,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 	id.src_addr = ip_hdr->SrcAddr();
 	id.dst_addr = ip_hdr->DstAddr();
 	Dictionary* d = 0;
+	BifEnum::Tunnel::Type tunnel_type = BifEnum::Tunnel::IP;
 
 	switch ( proto ) {
 	case IPPROTO_TCP:
@@ -606,6 +607,8 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 
 		// Treat GRE tunnel like IP tunnels, fallthrough to logic below now
 		// that GRE header is stripped and only payload packet remains.
+		// The only thing different is the tunnel type enum value to use.
+		tunnel_type = BifEnum::Tunnel::GRE;
 		}
 
 	case IPPROTO_IPV4:
@@ -653,7 +656,8 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 
 		if ( it == ip_tunnels.end() )
 			{
-			EncapsulatingConn ec(ip_hdr->SrcAddr(), ip_hdr->DstAddr());
+			EncapsulatingConn ec(ip_hdr->SrcAddr(), ip_hdr->DstAddr(),
+			                     tunnel_type);
 			ip_tunnels[tunnel_idx] = TunnelActivity(ec, network_time);
 			timer_mgr->Add(new IPTunnelTimer(network_time, tunnel_idx));
 			}

src/Stats.cc

@@ -10,6 +10,10 @@
 #include "Trigger.h"
 #include "threading/Manager.h"
 
+#ifdef ENABLE_BROKER
+#include "broker/Manager.h"
+#endif
+
 int killed_by_inactivity = 0;
 
 uint64 tot_ack_events = 0;
@@ -222,6 +226,26 @@ void ProfileLogger::Log()
 			    ));
 		}
 
+#ifdef ENABLE_BROKER
+	auto cs = broker_mgr->ConsumeStatistics();
+
+	file->Write(fmt("%0.6f Comm: peers=%zu stores=%zu "
+	                "store_queries=%zu store_responses=%zu "
+	                "outgoing_conn_status=%zu incoming_conn_status=%zu "
+	                "reports=%zu\n",
+	                network_time, cs.outgoing_peer_count, cs.data_store_count,
+	                cs.pending_query_count, cs.response_count,
+	                cs.outgoing_conn_status_count, cs.incoming_conn_status_count,
+	                cs.report_count));
+
+	for ( const auto& s : cs.print_count )
+		file->Write(fmt("    %-25s prints dequeued=%zu\n", s.first.data(), s.second));
+	for ( const auto& s : cs.event_count )
+		file->Write(fmt("    %-25s events dequeued=%zu\n", s.first.data(), s.second));
+	for ( const auto& s : cs.log_count )
+		file->Write(fmt("    %-25s logs dequeued=%zu\n", s.first.data(), s.second));
+#endif
+
 	// Script-level state.
 	unsigned int size, mem = 0;
 	PDict(ID)* globals = global_scope()->Vars();

src/Stmt.cc

@@ -23,7 +23,7 @@ const char* stmt_name(BroStmtTag t)
 		"print", "event", "expr", "if", "when", "switch",
 		"for", "next", "break", "return", "add", "delete",
 		"list", "bodylist",
-		"<init>", "fallthrough",
+		"<init>", "fallthrough", "while",
 		"null",
 	};
 
@@ -1127,6 +1127,126 @@ bool EventStmt::DoUnserialize(UnserialInfo* info)
 	return event_expr != 0;
 	}
 
+WhileStmt::WhileStmt(Expr* arg_loop_condition, Stmt* arg_body)
+	: loop_condition(arg_loop_condition), body(arg_body)
+	{
+	if ( ! loop_condition->IsError() &&
+	     ! IsBool(loop_condition->Type()->Tag()) )
+		loop_condition->Error("while conditional must be boolean");
+	}
+
+WhileStmt::~WhileStmt()
+	{
+	Unref(loop_condition);
+	Unref(body);
+	}
+
+int WhileStmt::IsPure() const
+	{
+	return loop_condition->IsPure() && body->IsPure();
+	}
+
+void WhileStmt::Describe(ODesc* d) const
+	{
+	Stmt::Describe(d);
+
+	if ( d->IsReadable() )
+		d->Add("(");
+
+	loop_condition->Describe(d);
+
+	if ( d->IsReadable() )
+		d->Add(")");
+
+	d->SP();
+	d->PushIndent();
+	body->AccessStats(d);
+	body->Describe(d);
+	d->PopIndent();
+	}
+
+TraversalCode WhileStmt::Traverse(TraversalCallback* cb) const
+	{
+	TraversalCode tc = cb->PreStmt(this);
+	HANDLE_TC_STMT_PRE(tc);
+
+	tc = loop_condition->Traverse(cb);
+	HANDLE_TC_STMT_PRE(tc);
+
+	tc = body->Traverse(cb);
+	HANDLE_TC_STMT_PRE(tc);
+
+	tc = cb->PostStmt(this);
+	HANDLE_TC_STMT_POST(tc);
+	}
+
+Val* WhileStmt::Exec(Frame* f, stmt_flow_type& flow) const
+	{
+	RegisterAccess();
+	flow = FLOW_NEXT;
+	Val* rval = 0;
+
+	for ( ; ; )
+		{
+		Val* cond = loop_condition->Eval(f);
+
+		if ( ! cond )
+			break;
+
+		bool cont = cond->AsBool();
+		Unref(cond);
+
+		if ( ! cont )
+			break;
+
+		flow = FLOW_NEXT;
+		rval = body->Exec(f, flow);
+
+		if ( flow == FLOW_BREAK || flow == FLOW_RETURN )
+			break;
+		}
+
+	if ( flow == FLOW_LOOP || flow == FLOW_BREAK )
+		flow = FLOW_NEXT;
+
+	return rval;
+	}
+
+Stmt* WhileStmt::Simplify()
+	{
+	loop_condition = simplify_expr(loop_condition, SIMPLIFY_GENERAL);
+
+	if ( loop_condition->IsConst() && loop_condition->IsZero() )
+		return new NullStmt();
+
+	body = simplify_stmt(body);
+	return this;
+	}
+
+IMPLEMENT_SERIAL(WhileStmt, SER_WHILE_STMT);
+
+bool WhileStmt::DoSerialize(SerialInfo* info) const
+	{
+	DO_SERIALIZE(SER_WHILE_STMT, Stmt);
+
+	if ( ! loop_condition->Serialize(info) )
+		return false;
+
+	return body->Serialize(info);
+	}
+
+bool WhileStmt::DoUnserialize(UnserialInfo* info)
+	{
+	DO_UNSERIALIZE(Stmt);
+	loop_condition = Expr::Unserialize(info);
+
+	if ( ! loop_condition )
+		return false;
+
+	body = Stmt::Unserialize(info);
+	return body != 0;
+	}
+
 ForStmt::ForStmt(id_list* arg_loop_vars, Expr* loop_expr)
 : ExprStmt(STMT_FOR, loop_expr)
 	{

src/Stmt.h

@@ -310,6 +310,33 @@ protected:
 	EventExpr* event_expr;
 };
 
+class WhileStmt : public Stmt {
+public:
+
+	WhileStmt(Expr* loop_condition, Stmt* body);
+	~WhileStmt();
+
+	int IsPure() const;
+
+	void Describe(ODesc* d) const;
+
+	TraversalCode Traverse(TraversalCallback* cb) const;
+
+protected:
+	friend class Stmt;
+
+	WhileStmt()
+		{ loop_condition = 0; body = 0; }
+
+	Val* Exec(Frame* f, stmt_flow_type& flow) const;
+	Stmt* Simplify();
+
+	DECLARE_SERIAL(WhileStmt);
+
+	Expr* loop_condition;
+	Stmt* body;
+};
+
 class ForStmt : public ExprStmt {
 public:
 	ForStmt(id_list* loop_vars, Expr* loop_expr);

src/StmtEnums.h

@@ -17,6 +17,7 @@ typedef enum {
 	STMT_LIST, STMT_EVENT_BODY_LIST,
 	STMT_INIT,
 	STMT_FALLTHROUGH,
+	STMT_WHILE,
 	STMT_NULL
 #define NUM_STMTS (int(STMT_NULL) + 1)
 } BroStmtTag;

src/Trigger.cc

@@ -112,6 +112,7 @@ Trigger::Trigger(Expr* arg_cond, Stmt* arg_body, Stmt* arg_timeout_stmts,
 	attached = 0;
 	is_return = arg_is_return;
 	location = arg_location;
+	timeout_value = -1;
 
 	++total_triggers;
 
@@ -133,17 +134,22 @@ Trigger::Trigger(Expr* arg_cond, Stmt* arg_body, Stmt* arg_timeout_stmts,
 
 	Val* timeout_val = arg_timeout ? arg_timeout->Eval(arg_frame) : 0;
 
+	if ( timeout_val )
+		{
+		Unref(timeout_val);
+		timeout_value = timeout_val->AsInterval();
+		}
+
 	// Make sure we don't get deleted if somebody calls a method like
 	// Timeout() while evaluating the trigger. 
 	Ref(this);
 
-	if ( ! Eval() && timeout_val )
+	if ( ! Eval() && timeout_value >= 0 )
 		{
-		timer = new TriggerTimer(timeout_val->AsInterval(), this);
+		timer = new TriggerTimer(timeout_value, this);
 		timer_mgr->Add(timer);
 		}
 
-	Unref(timeout_val);
 	Unref(this);
 	}
 

src/Trigger.h

@@ -32,6 +32,10 @@ public:
 	// Executes timeout code and deletes the object.
 	void Timeout();
 
+	// Return the timeout interval (negative if none was specified).
+	double TimeoutValue() const
+		{ return timeout_value; }
+
 	// Called if another entity needs to complete its operations first
 	// in any case before this trigger can proceed.
 	void Hold()	{ delayed = true; }
@@ -51,6 +55,8 @@ public:
 	// may not immediately delete it as other references may still exist.
 	void Disable();
 
+	bool Disabled() const { return disabled; }
+
 	virtual void Describe(ODesc* d) const { d->Add("<trigger>"); }
 
 	// Overidden from Notifier.  We queue the trigger and evaluate it
@@ -87,6 +93,7 @@ private:
 	Stmt* body;
 	Stmt* timeout_stmts;
 	Expr* timeout;
+	double timeout_value;
 	Frame* frame;
 	bool is_return;
 	const Location* location;

src/TunnelEncapsulation.h

@@ -37,10 +37,12 @@ public:
 	 *
 	 * @param s The tunnel source address, likely taken from an IP header.
 	 * @param d The tunnel destination address, likely taken from an IP header.
+	 * @param t The type of IP tunnel.
 	 */
-	EncapsulatingConn(const IPAddr& s, const IPAddr& d)
+	EncapsulatingConn(const IPAddr& s, const IPAddr& d,
+	                  BifEnum::Tunnel::Type t = BifEnum::Tunnel::IP)
 		: src_addr(s), dst_addr(d), src_port(0), dst_port(0),
-		  proto(TRANSPORT_UNKNOWN), type(BifEnum::Tunnel::IP),
+		  proto(TRANSPORT_UNKNOWN), type(t),
 		  uid(Bro::UID(bits_per_uid))
 		{
 		}
@@ -85,7 +87,8 @@ public:
 		if ( ec1.type != ec2.type )
 			return false;
 
-		if ( ec1.type == BifEnum::Tunnel::IP )
+		if ( ec1.type == BifEnum::Tunnel::IP ||
+		     ec1.type == BifEnum::Tunnel::GRE )
 			// Reversing endpoints is still same tunnel.
 			return ec1.uid == ec2.uid && ec1.proto == ec2.proto &&
 			  ((ec1.src_addr == ec2.src_addr && ec1.dst_addr == ec2.dst_addr) ||

src/analyzer/Analyzer.h

@@ -400,7 +400,7 @@ public:
 	 *
 	 * @return The analyzer, or null if not found.
 	 */
-	Analyzer* FindChild(ID id);
+	virtual Analyzer* FindChild(ID id);
 
 	/**
 	 * Recursively searches all (direct or indirect) childs of the
@@ -411,7 +411,7 @@ public:
 	 * @return The first analyzer of the given type found, or null if
 	 * none.
 	 */
-	Analyzer* FindChild(Tag tag);
+	virtual Analyzer* FindChild(Tag tag);
 
 	/**
 	 * Recursively searches all (direct or indirect) childs of the

src/analyzer/protocol/CMakeLists.txt

@@ -30,6 +30,7 @@ add_subdirectory(ntp)
 add_subdirectory(pia)
 add_subdirectory(pop3)
 add_subdirectory(radius)
+add_subdirectory(rdp)
 add_subdirectory(rpc)
 add_subdirectory(snmp)
 add_subdirectory(smb)

src/analyzer/protocol/conn-size/CMakeLists.txt

@@ -6,4 +6,5 @@ include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DI
 bro_plugin_begin(Bro ConnSize)
 bro_plugin_cc(ConnSize.cc Plugin.cc)
 bro_plugin_bif(events.bif)
+bro_plugin_bif(functions.bif)
 bro_plugin_end()

src/analyzer/protocol/conn-size/ConnSize.cc

@@ -29,6 +29,11 @@ void ConnSize_Analyzer::Init()
 	orig_pkts = 0;
 	resp_bytes = 0;
 	resp_pkts = 0;
+
+	orig_bytes_thresh = 0;
+	orig_pkts_thresh = 0;
+	resp_bytes_thresh = 0;
+	resp_pkts_thresh = 0;
 	}
 
 void ConnSize_Analyzer::Done()
@@ -36,6 +41,50 @@ void ConnSize_Analyzer::Done()
 	Analyzer::Done();
 	}
 
+void ConnSize_Analyzer::ThresholdEvent(EventHandlerPtr f, uint64 threshold, bool is_orig)
+	{
+	if ( ! f )
+		return;
+
+	val_list* vl = new val_list;
+	vl->append(BuildConnVal());
+	vl->append(new Val(threshold, TYPE_COUNT));
+	vl->append(new Val(is_orig, TYPE_BOOL));
+	ConnectionEvent(f, vl);
+	}
+
+void ConnSize_Analyzer::CheckSizes(bool is_orig)
+	{
+	if ( is_orig )
+		{
+		if ( orig_bytes_thresh && orig_bytes >= orig_bytes_thresh )
+			{
+			ThresholdEvent(conn_bytes_threshold_crossed, orig_bytes_thresh, is_orig);
+			orig_bytes_thresh = 0;
+			}
+
+		if ( orig_pkts_thresh && orig_pkts >= orig_pkts_thresh )
+			{
+			ThresholdEvent(conn_packets_threshold_crossed, orig_pkts_thresh, is_orig);
+			orig_pkts_thresh = 0;
+			}
+		}
+	else
+		{
+		if ( resp_bytes_thresh && resp_bytes >= resp_bytes_thresh )
+			{
+			ThresholdEvent(conn_bytes_threshold_crossed, resp_bytes_thresh, is_orig);
+			resp_bytes_thresh = 0;
+			}
+
+		if ( resp_pkts_thresh && resp_pkts >= resp_pkts_thresh )
+			{
+			ThresholdEvent(conn_packets_threshold_crossed, resp_pkts_thresh, is_orig);
+			resp_pkts_thresh = 0;
+			}
+		}
+	}
+
 void ConnSize_Analyzer::DeliverPacket(int len, const u_char* data, bool is_orig, uint64 seq, const IP_Hdr* ip, int caplen)
 	{
 	Analyzer::DeliverPacket(len, data, is_orig, seq, ip, caplen);
@@ -50,6 +99,47 @@ void ConnSize_Analyzer::DeliverPacket(int len, const u_char* data, bool is_orig,
 		resp_bytes += ip->TotalLen();
 		resp_pkts ++;
 		}
+
+	CheckSizes(is_orig);
+	}
+
+void ConnSize_Analyzer::SetThreshold(uint64 threshold, bool bytes, bool orig)
+	{
+	if ( bytes )
+		{
+		if ( orig )
+			orig_bytes_thresh = threshold;
+		else
+			resp_bytes_thresh = threshold;
+		}
+	else
+		{
+		if ( orig )
+			orig_pkts_thresh = threshold;
+		else
+			resp_pkts_thresh = threshold;
+		}
+
+	// Check if threshold is already crossed.
+	CheckSizes(orig);
+	}
+
+uint64_t ConnSize_Analyzer::GetThreshold(bool bytes, bool orig)
+	{
+	if ( bytes )
+		{
+		if ( orig )
+			return orig_bytes_thresh;
+		else
+			return resp_bytes_thresh;
+		}
+	else
+		{
+		if ( orig )
+			return orig_pkts_thresh;
+		else
+			return resp_pkts_thresh;
+		}
 	}
 
 void ConnSize_Analyzer::UpdateConnVal(RecordVal *conn_val)

src/analyzer/protocol/conn-size/ConnSize.h

@@ -21,18 +21,28 @@ public:
 	virtual void UpdateConnVal(RecordVal *conn_val);
 	virtual void FlipRoles();
 
+	void SetThreshold(uint64_t threshold, bool bytes, bool orig);
+	uint64 GetThreshold(bool bytes, bool orig);
+
 	static analyzer::Analyzer* Instantiate(Connection* conn)
 		{ return new ConnSize_Analyzer(conn); }
 
 protected:
 	virtual void DeliverPacket(int len, const u_char* data, bool is_orig,
 					uint64 seq, const IP_Hdr* ip, int caplen);
+	void CheckSizes(bool is_orig);
 
+	void ThresholdEvent(EventHandlerPtr f, uint64 threshold, bool is_orig);
 
 	uint64_t orig_bytes;
 	uint64_t resp_bytes;
 	uint64_t orig_pkts;
 	uint64_t resp_pkts;
+
+	uint64_t orig_bytes_thresh;
+	uint64_t resp_bytes_thresh;
+	uint64_t orig_pkts_thresh;
+	uint64_t resp_pkts_thresh;
 };
 
 } } // namespace analyzer::* 

src/analyzer/protocol/conn-size/events.bif

@@ -0,0 +1,27 @@
+## Generated for a connection that crossed a set byte threshold. Note that this
+## is a low level event that should usually be avoided for user code. Use
+## ConnThreshold::bytes_threshold_crossed instead.
+##
+## c: the connection
+##
+## threshold: the threshold that was set
+##
+## is_orig: true if the threshold was crossed by the originator of the connection
+##
+## .. bro:see:: set_current_conn_packets_threshold set_current_conn_bytes_threshold conn_packets_threshold_crossed
+##              get_current_conn_bytes_threshold get_current_conn_packets_threshold
+event conn_bytes_threshold_crossed%(c: connection, threshold: count, is_orig: bool%);
+
+## Generated for a connection that crossed a set packet threshold. Note that this
+## is a low level event that should usually be avoided for user code. Use
+## ConnThreshold::bytes_threshold_crossed instead.
+##
+## c: the connection
+##
+## threshold: the threshold that was set
+##
+## is_orig: true if the threshold was crossed by the originator of the connection
+##
+## .. bro:see:: set_current_conn_packets_threshold set_current_conn_bytes_threshold conn_bytes_threshold_crossed
+##              get_current_conn_bytes_threshold get_current_conn_packets_threshold
+event conn_packets_threshold_crossed%(c: connection, threshold: count, is_orig: bool%);

src/analyzer/protocol/conn-size/functions.bif

@@ -0,0 +1,109 @@
+%%{
+#include "analyzer/protocol/conn-size/ConnSize.h"
+
+static analyzer::Analyzer* GetConnsizeAnalyzer(Val* cid)
+	{
+	Connection* c = sessions->FindConnection(cid);
+	if ( ! c )
+		{
+		reporter->Error("cannot find connection");
+		return 0;
+		}
+
+	analyzer::Analyzer* a = c->FindAnalyzer("CONNSIZE");
+	if ( ! a )
+		reporter->Error("connection does not have ConnSize analyzer");
+
+	return a;
+	}
+
+%%}
+
+## Sets the current byte threshold for connection sizes, overwriting any potential old
+## threshold. Be aware that in nearly any case you will want to use the high level API
+## instead (ConnThreshold::set_bytes_threshold).
+##
+## cid: The connection id.
+##
+## threshold: Threshold in bytes.
+##
+## is_orig: If true, threshold is set for bytes from originator, otherwhise for bytes from responder.
+##
+## .. bro:see:: set_current_conn_packets_threshold conn_bytes_threshold_crossed conn_packets_threshold_crossed
+##              get_current_conn_bytes_threshold get_current_conn_packets_threshold
+function set_current_conn_bytes_threshold%(cid: conn_id, threshold: count, is_orig: bool%): bool
+	%{
+	analyzer::Analyzer* a = GetConnsizeAnalyzer(cid);
+	if ( ! a )
+		return new Val(0, TYPE_BOOL);
+
+	static_cast<analyzer::conn_size::ConnSize_Analyzer*>(a)->SetThreshold(threshold, 1, is_orig);
+
+	return new Val(1, TYPE_BOOL);
+	%}
+
+## Sets a threshold for connection packets, overwtiting any potential old thresholds.
+## Be aware that in nearly any case you will want to use the high level API
+## instead (ConnThreshold::set_packets_threshold).
+##
+## cid: The connection id.
+##
+## threshold: Threshold in packets.
+##
+## is_orig: If true, threshold is set for packets from originator, otherwhise for packets from responder.
+##
+## .. bro:see:: set_current_conn_bytes_threshold conn_bytes_threshold_crossed conn_packets_threshold_crossed
+##              get_current_conn_bytes_threshold get_current_conn_packets_threshold
+function set_current_conn_packets_threshold%(cid: conn_id, threshold: count, is_orig: bool%): bool
+	%{
+	analyzer::Analyzer* a = GetConnsizeAnalyzer(cid);
+	if ( ! a )
+		return new Val(0, TYPE_BOOL);
+
+	static_cast<analyzer::conn_size::ConnSize_Analyzer*>(a)->SetThreshold(threshold, 0, is_orig);
+
+	return new Val(1, TYPE_BOOL);
+	%}
+
+## Gets the current byte threshold size for a connection.
+##
+## cid: The connection id.
+##
+## is_orig: If true, threshold of originator, otherwhise threshold of responder.
+##
+## Returns: 0 if no threshold is set or the threshold in bytes
+##
+## .. bro:see:: set_current_conn_packets_threshold conn_bytes_threshold_crossed conn_packets_threshold_crossed
+##              get_current_conn_packets_threshold
+function get_current_conn_bytes_threshold%(cid: conn_id, is_orig: bool%): count
+	%{
+	analyzer::Analyzer* a = GetConnsizeAnalyzer(cid);
+	if ( ! a )
+		return new Val(0, TYPE_COUNT);
+
+	return new Val(
+		static_cast<analyzer::conn_size::ConnSize_Analyzer*>(a)->GetThreshold(1, is_orig),
+		TYPE_COUNT);
+	%}
+
+## Gets the current packet threshold size for a connection.
+##
+## cid: The connection id.
+##
+## is_orig: If true, threshold of originator, otherwhise threshold of responder.
+##
+## Returns: 0 if no threshold is set or the threshold in packets
+##
+## .. bro:see:: set_current_conn_packets_threshold conn_bytes_threshold_crossed conn_packets_threshold_crossed
+##              get_current_conn_bytes_threshold
+function get_current_conn_packets_threshold%(cid: conn_id, is_orig: bool%): count
+	%{
+	analyzer::Analyzer* a = GetConnsizeAnalyzer(cid);
+	if ( ! a )
+		return new Val(0, TYPE_COUNT);
+
+	return new Val(
+		static_cast<analyzer::conn_size::ConnSize_Analyzer*>(a)->GetThreshold(0, is_orig),
+		TYPE_COUNT);
+	%}
+

src/analyzer/protocol/dns/DNS.cc

@@ -19,6 +19,7 @@ using namespace analyzer::dns;
 DNS_Interpreter::DNS_Interpreter(analyzer::Analyzer* arg_analyzer)
 	{
 	analyzer = arg_analyzer;
+	first_message = true;
 	}
 
 int DNS_Interpreter::ParseMessage(const u_char* data, int len, int is_query)
@@ -33,6 +34,16 @@ int DNS_Interpreter::ParseMessage(const u_char* data, int len, int is_query)
 
 	DNS_MsgInfo msg((DNS_RawMsgHdr*) data, is_query);
 
+	if ( first_message && msg.QR && is_query == 1 )
+		{
+		is_query = msg.is_query = 0;
+
+		if ( ! analyzer->Conn()->RespAddr().IsMulticast() )
+			analyzer->Conn()->FlipRoles();
+		}
+
+	first_message = false;
+
 	if ( dns_message )
 		{
 		val_list* vl = new val_list();
@@ -308,7 +319,7 @@ int DNS_Interpreter::ParseAnswer(DNS_MsgInfo* msg,
 				analyzer->ConnectionEvent(dns_unknown_reply, vl);
 				}
 
-			analyzer->Weird("DNS_RR_unknown_type");
+			analyzer->Weird("DNS_RR_unknown_type", fmt("%d", msg->atype));
 			data += rdlength;
 			len -= rdlength;
 			status = 1;
@@ -1064,7 +1075,8 @@ void Contents_DNS::Flush()
 	{
 	if ( buf_n > 0 )
 		{ // Deliver partial message.
-		interp->ParseMessage(msg_buf, buf_n, true);
+		// '2' here means whether it's a query is unknown.
+		interp->ParseMessage(msg_buf, buf_n, 2);
 		msg_size = 0;
 		}
 	}

src/analyzer/protocol/dns/DNS.h

@@ -220,6 +220,7 @@ protected:
 					BroString* question_name);
 
 	analyzer::Analyzer* analyzer;
+	bool first_message;
 };
 
 

src/analyzer/protocol/http/HTTP.cc

@@ -985,9 +985,7 @@ void HTTP_Analyzer::DeliverStream(int len, const u_char* data, bool is_orig)
 				{
 				++num_replies;
 
-				if ( unanswered_requests.empty() )
-					Weird("unmatched_HTTP_reply");
-				else
+				if ( ! unanswered_requests.empty() )
 					ProtocolConfirmation();
 
 				reply_state = EXPECT_REPLY_MESSAGE;

src/analyzer/protocol/icmp/ICMP.cc

@@ -130,7 +130,7 @@ void ICMP_Analyzer::NextICMP4(double t, const struct icmp* icmpp, int len, int c
 			break;
 
 		default:
-			ICMPEvent(icmp_sent, icmpp, len, 0, ip_hdr);
+			ICMP_Sent(icmpp, len, caplen, 0, data, ip_hdr);
 			break;
 		}
 	}
@@ -172,7 +172,7 @@ void ICMP_Analyzer::NextICMP6(double t, const struct icmp* icmpp, int len, int c
 			RouterSolicit(t, icmpp, len, caplen, data, ip_hdr);
 			break;
 		case ICMP6_ROUTER_RENUMBERING:
-			ICMPEvent(icmp_sent, icmpp, len, 1, ip_hdr);
+			ICMP_Sent(icmpp, len, caplen, 1, data, ip_hdr);
 			break;
 
 #if 0
@@ -188,21 +188,32 @@ void ICMP_Analyzer::NextICMP6(double t, const struct icmp* icmpp, int len, int c
 			if ( icmpp->icmp_type < 128 )
 				Context6(t, icmpp, len, caplen, data, ip_hdr);
 			else
-				ICMPEvent(icmp_sent, icmpp, len, 1, ip_hdr);
+				ICMP_Sent(icmpp, len, caplen, 1, data, ip_hdr);
 			break;
 		}
 	}
 
-void ICMP_Analyzer::ICMPEvent(EventHandlerPtr f, const struct icmp* icmpp,
-                              int len, int icmpv6, const IP_Hdr* ip_hdr)
+void ICMP_Analyzer::ICMP_Sent(const struct icmp* icmpp, int len, int caplen,
+                              int icmpv6, const u_char* data,
+                              const IP_Hdr* ip_hdr)
     {
-	if ( ! f )
-		return;
+	if ( icmp_sent )
+		{
+		val_list* vl = new val_list;
+		vl->append(BuildConnVal());
+		vl->append(BuildICMPVal(icmpp, len, icmpv6, ip_hdr));
+		ConnectionEvent(icmp_sent, vl);
+		}
 
-	val_list* vl = new val_list;
-	vl->append(BuildConnVal());
-	vl->append(BuildICMPVal(icmpp, len, icmpv6, ip_hdr));
-	ConnectionEvent(f, vl);
+	if ( icmp_sent_payload )
+		{
+		val_list* vl = new val_list;
+		vl->append(BuildConnVal());
+		vl->append(BuildICMPVal(icmpp, len, icmpv6, ip_hdr));
+		BroString* payload = new BroString(data, min(len, caplen), 0);
+		vl->append(new StringVal(payload));
+		ConnectionEvent(icmp_sent_payload, vl);
+		}
 	}
 
 RecordVal* ICMP_Analyzer::BuildICMPVal(const struct icmp* icmpp, int len,

src/analyzer/protocol/icmp/ICMP.h

@@ -33,8 +33,8 @@ protected:
 	virtual bool IsReuse(double t, const u_char* pkt);
 	virtual unsigned int MemoryAllocation() const;
 
-	void ICMPEvent(EventHandlerPtr f, const struct icmp* icmpp, int len,
-	               int icmpv6, const IP_Hdr* ip_hdr);
+	void ICMP_Sent(const struct icmp* icmpp, int len, int caplen, int icmpv6,
+	               const u_char* data, const IP_Hdr* ip_hdr);
 
 	void Echo(double t, const struct icmp* icmpp, int len,
 			 int caplen, const u_char*& data, const IP_Hdr* ip_hdr);

src/analyzer/protocol/icmp/events.bif

@@ -12,9 +12,21 @@
 ## icmp: Additional ICMP-specific information augmenting the standard
 ##       connection record *c*.
 ##
-## .. bro:see:: icmp_error_message
+## .. bro:see:: icmp_error_message icmp_sent_payload
 event icmp_sent%(c: connection, icmp: icmp_conn%);
 
+## The same as :bro:see:`icmp_sent` except containing the ICMP payload.
+##
+## c: The connection record for the corresponding ICMP flow.
+##
+## icmp: Additional ICMP-specific information augmenting the standard
+##       connection record *c*.
+##
+## payload: The payload of the ICMP message.
+##
+## .. bro:see:: icmp_error_message icmp_sent_payload
+event icmp_sent_payload%(c: connection, icmp: icmp_conn, payload: string%);
+
 ## Generated for ICMP *echo request* messages.
 ##
 ## See `Wikipedia

src/analyzer/protocol/pia/PIA.cc

@@ -81,7 +81,7 @@ void PIA::PIA_Done()
 	}
 
 void PIA::PIA_DeliverPacket(int len, const u_char* data, bool is_orig, uint64 seq,
-				const IP_Hdr* ip, int caplen)
+				const IP_Hdr* ip, int caplen, bool clear_state)
 	{
 	if ( pkt_buffer.state == SKIPPING )
 		return;
@@ -108,6 +108,9 @@ void PIA::PIA_DeliverPacket(int len, const u_char* data, bool is_orig, uint64 se
 	// FIXME: I'm not sure why it does not work with eol=true...
 	DoMatch(data, len, is_orig, true, false, false, ip);
 
+	if ( clear_state )
+		RuleMatcherState::ClearMatchState(is_orig);
+
 	pkt_buffer.state = new_state;
 
 	current_packet.data = 0;

src/analyzer/protocol/pia/PIA.h

@@ -42,7 +42,7 @@ public:
 protected:
 	void PIA_Done();
 	void PIA_DeliverPacket(int len, const u_char* data, bool is_orig,
-				uint64 seq, const IP_Hdr* ip, int caplen);
+				uint64 seq, const IP_Hdr* ip, int caplen, bool clear_state);
 
 	enum State { INIT, BUFFERING, MATCHING_ONLY, SKIPPING } state;
 
@@ -109,7 +109,7 @@ protected:
 					uint64 seq, const IP_Hdr* ip, int caplen)
 		{
 		Analyzer::DeliverPacket(len, data, is_orig, seq, ip, caplen);
-		PIA_DeliverPacket(len, data, is_orig, seq, ip, caplen);
+		PIA_DeliverPacket(len, data, is_orig, seq, ip, caplen, true);
 		}
 
 	virtual void ActivateAnalyzer(analyzer::Tag tag, const Rule* rule);
@@ -154,7 +154,7 @@ protected:
 					uint64 seq, const IP_Hdr* ip, int caplen)
 		{
 		Analyzer::DeliverPacket(len, data, is_orig, seq, ip, caplen);
-		PIA_DeliverPacket(len, data, is_orig, seq, ip, caplen);
+		PIA_DeliverPacket(len, data, is_orig, seq, ip, caplen, false);
 		}
 
 	virtual void DeliverStream(int len, const u_char* data, bool is_orig);

src/analyzer/protocol/rdp/CMakeLists.txt

@@ -0,0 +1,10 @@
+include(BroPlugin)
+
+include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
+
+bro_plugin_begin(Bro RDP)
+	bro_plugin_cc(RDP.cc Plugin.cc)
+	bro_plugin_bif(events.bif)
+	bro_plugin_bif(types.bif)
+	bro_plugin_pac(rdp.pac rdp-analyzer.pac rdp-protocol.pac)
+bro_plugin_end()

src/analyzer/protocol/rdp/Plugin.cc

@@ -0,0 +1,22 @@
+#include "plugin/Plugin.h"
+
+#include "RDP.h"
+
+namespace plugin {
+namespace Bro_RDP {
+
+class Plugin : public plugin::Plugin {
+public:
+        plugin::Configuration Configure()
+                {
+                AddComponent(new ::analyzer::Component("RDP", ::analyzer::rdp::RDP_Analyzer::InstantiateAnalyzer));
+
+                plugin::Configuration config;
+                config.name = "Bro::RDP";
+                config.description = "RDP analyzer";
+                return config;
+                }
+} plugin;
+
+}
+}

src/analyzer/protocol/rdp/RDP.cc

@@ -0,0 +1,94 @@
+#include "RDP.h"
+#include "analyzer/protocol/tcp/TCP_Reassembler.h"
+#include "Reporter.h"
+#include "events.bif.h"
+#include "types.bif.h"
+
+using namespace analyzer::rdp;
+
+RDP_Analyzer::RDP_Analyzer(Connection* c)
+	: tcp::TCP_ApplicationAnalyzer("RDP", c)
+	{
+	interp = new binpac::RDP::RDP_Conn(this);
+	
+	had_gap = false;
+	pia = 0;
+	}
+
+RDP_Analyzer::~RDP_Analyzer()
+	{
+	delete interp;
+	}
+
+void RDP_Analyzer::Done()
+	{
+	tcp::TCP_ApplicationAnalyzer::Done();
+
+	interp->FlowEOF(true);
+	interp->FlowEOF(false);
+	}
+
+void RDP_Analyzer::EndpointEOF(bool is_orig)
+	{
+	tcp::TCP_ApplicationAnalyzer::EndpointEOF(is_orig);
+	interp->FlowEOF(is_orig);
+	}
+
+void RDP_Analyzer::DeliverStream(int len, const u_char* data, bool orig)
+	{
+	tcp::TCP_ApplicationAnalyzer::DeliverStream(len, data, orig);
+
+	assert(TCP());
+	if ( TCP()->IsPartial() )
+		return;
+
+	if ( had_gap )
+		// If only one side had a content gap, we could still try to
+		// deliver data to the other side if the script layer can handle this.
+		return;
+
+	if ( interp->is_encrypted() )
+		{
+		// 0x00 is RDP native encryption which we don't do anything with now.
+		// 0x01 is SSL/TLS
+		// 0x03-0x04 is CredSSP which is effectively SSL/TLS
+		if ( interp->encryption_method() > 0x00 )
+			{
+			if ( ! pia )
+				{
+				pia = new pia::PIA_TCP(Conn());
+
+				if ( ! AddChildAnalyzer(pia) )
+					{
+					reporter->AnalyzerError(this,
+					                        "failed to add TCP child analyzer "
+					                        "to RDP analyzer: already exists");
+					return;
+					}
+
+				pia->FirstPacket(true, 0);
+				pia->FirstPacket(false, 0);
+				}
+
+			ForwardStream(len, data, orig);
+			}
+		}
+	else // if not encrypted
+		{
+		try
+			{
+			interp->NewData(orig, data, data + len);
+			}
+		catch ( const binpac::Exception& e )
+			{
+			ProtocolViolation(fmt("Binpac exception: %s", e.c_msg()));
+			}
+		}
+	}
+
+void RDP_Analyzer::Undelivered(uint64 seq, int len, bool orig)
+	{
+	tcp::TCP_ApplicationAnalyzer::Undelivered(seq, len, orig);
+	had_gap = true;
+	interp->NewGap(orig, len);
+	}

src/analyzer/protocol/rdp/RDP.h

@@ -0,0 +1,39 @@
+#ifndef ANALYZER_PROTOCOL_RDP_RDP_H
+#define ANALYZER_PROTOCOL_RDP_RDP_H
+
+#include "events.bif.h"
+
+
+#include "analyzer/protocol/tcp/TCP.h"
+#include "analyzer/protocol/pia/PIA.h"
+
+#include "rdp_pac.h"
+
+namespace analyzer { namespace rdp {
+
+class RDP_Analyzer : public tcp::TCP_ApplicationAnalyzer {
+
+public:
+	RDP_Analyzer(Connection* conn);
+	virtual ~RDP_Analyzer();
+
+	// Overriden from Analyzer.
+	virtual void Done();
+	
+	virtual void DeliverStream(int len, const u_char* data, bool orig);
+	virtual void Undelivered(uint64 seq, int len, bool orig);
+	virtual void EndpointEOF(bool is_orig);
+
+	static analyzer::Analyzer* InstantiateAnalyzer(Connection* conn)
+		{ return new RDP_Analyzer(conn); }
+
+protected:
+	binpac::RDP::RDP_Conn* interp;
+	
+	bool had_gap;
+	pia::PIA_TCP *pia;
+};
+
+} } // namespace analyzer::* 
+
+#endif

src/analyzer/protocol/rdp/events.bif

@@ -0,0 +1,61 @@
+## Generated for X.224 client requests.
+##
+## c: The connection record for the underlying transport-layer session/flow.
+##
+## cookie: The cookie included in the request.
+event rdp_connect_request%(c: connection, cookie: string%);
+
+## Generated for RDP Negotiation Response messages.
+##
+## c: The connection record for the underlying transport-layer session/flow.
+##
+## security_protocol: The security protocol selected by the server.
+event rdp_negotiation_response%(c: connection, security_protocol: count%);
+
+## Generated for RDP Negotiation Failure messages.
+##
+## c: The connection record for the underlying transport-layer session/flow.
+##
+## failure_code: The failure code sent by the server.
+event rdp_negotiation_failure%(c: connection, failure_code: count%);
+
+## Generated for MCS client requests.
+##
+## c: The connection record for the underlying transport-layer session/flow.
+##
+## data: The data contained in the client core data structure.
+event rdp_client_core_data%(c: connection, data: RDP::ClientCoreData%);
+
+## Generated for MCS server responses.
+##
+## c: The connection record for the underlying transport-layer session/flow.
+##
+## result: The 8-bit integer representing the GCC Conference Create Response result.
+event rdp_gcc_server_create_response%(c: connection, result: count%);
+
+## Generated for MCS server responses.
+##
+## c: The connection record for the underlying transport-layer session/flow.
+##
+## encryption_method: The 32-bit integer representing the encryption method used in the connection.
+##
+## encryption_level: The 32-bit integer representing the encryption level used in the connection.
+event rdp_server_security%(c: connection, encryption_method: count, encryption_level: count%);
+
+## Generated for a server certificate section.  If multiple X.509 
+## certificates are included in chain, this event will still
+## only be generated a single time.
+##
+## c: The connection record for the underlying transport-layer session/flow.
+##
+## cert_type: Indicates the type of certificate.
+##
+## permanently_issued: Value will be true is the certificate(s) is permanent on the server.
+event rdp_server_certificate%(c: connection, cert_type: count, permanently_issued: bool%);
+
+## Generated when an RDP session becomes encrypted.
+##
+## c: The connection record for the underlying transport-layer session/flow.
+##
+## security_protocol: The security protocol being used for the session.
+event rdp_begin_encryption%(c: connection, security_protocol: count%);

src/analyzer/protocol/rdp/rdp-analyzer.pac

@@ -0,0 +1,212 @@
+%extern{
+#include "ConvertUTF.h"
+#include "file_analysis/Manager.h"
+#include "types.bif.h"
+%}
+
+refine flow RDP_Flow += {
+
+	function utf16_to_utf8_val(utf16: bytestring): StringVal
+		%{
+		std::string resultstring;
+		size_t widesize = utf16.length();
+
+		size_t utf8size = 3 * widesize + 1;
+		resultstring.resize(utf8size, '\0');
+		const UTF16* sourcestart = reinterpret_cast<const UTF16*>(utf16.begin());
+		const UTF16* sourceend = sourcestart + widesize;
+		UTF8* targetstart = reinterpret_cast<UTF8*>(&resultstring[0]);
+		UTF8* targetend = targetstart + utf8size;
+
+		ConversionResult res = ConvertUTF16toUTF8(&sourcestart, 
+		                                          sourceend,
+		                                          &targetstart, 
+		                                          targetend, 
+		                                          lenientConversion);
+		if ( res != conversionOK )
+			{
+			connection()->bro_analyzer()->Weird("Failed UTF-16 to UTF-8 conversion");
+			return new StringVal(utf16.length(), (const char *) utf16.begin());
+			}
+
+		*targetstart = 0;
+		// We're relying on no nulls being in the string.
+		return new StringVal(resultstring.c_str());
+		%}
+
+	function proc_rdp_connect_request(cr: Connect_Request): bool
+		%{
+		if ( rdp_connect_request )
+			{
+			BifEvent::generate_rdp_connect_request(connection()->bro_analyzer(),
+			                                       connection()->bro_analyzer()->Conn(),
+			                                       bytestring_to_val(${cr.cookie_value}));
+			}
+		
+		return true;
+		%}
+
+	function proc_rdp_negotiation_response(nr: RDP_Negotiation_Response): bool
+		%{
+		if ( rdp_negotiation_response )
+			{
+			BifEvent::generate_rdp_negotiation_response(connection()->bro_analyzer(),
+			                                            connection()->bro_analyzer()->Conn(),
+			                                            ${nr.selected_protocol});
+			}
+		
+		return true;
+		%}
+
+	function proc_rdp_negotiation_failure(nf: RDP_Negotiation_Failure): bool
+		%{
+		if ( rdp_negotiation_failure )
+			{
+			BifEvent::generate_rdp_negotiation_failure(connection()->bro_analyzer(),
+			                                           connection()->bro_analyzer()->Conn(),
+			                                           ${nf.failure_code});
+			}
+
+		return true;
+		%}
+
+
+	function proc_rdp_gcc_server_create_response(gcc_response: GCC_Server_Create_Response): bool
+		%{
+		connection()->bro_analyzer()->ProtocolConfirmation();
+
+		if ( rdp_gcc_server_create_response )
+			BifEvent::generate_rdp_gcc_server_create_response(connection()->bro_analyzer(),
+			                                                  connection()->bro_analyzer()->Conn(),
+			                                                  ${gcc_response.result});
+
+		return true;
+		%}
+
+
+	function proc_rdp_client_core_data(ccore: Client_Core_Data): bool
+		%{
+		connection()->bro_analyzer()->ProtocolConfirmation();
+
+		if ( rdp_client_core_data )
+			{
+			RecordVal* ec_flags = new RecordVal(BifType::Record::RDP::EarlyCapabilityFlags);
+			ec_flags->Assign(0, new Val(${ccore.SUPPORT_ERRINFO_PDU}, TYPE_BOOL));
+			ec_flags->Assign(1, new Val(${ccore.WANT_32BPP_SESSION}, TYPE_BOOL));
+			ec_flags->Assign(2, new Val(${ccore.SUPPORT_STATUSINFO_PDU}, TYPE_BOOL));
+			ec_flags->Assign(3, new Val(${ccore.STRONG_ASYMMETRIC_KEYS}, TYPE_BOOL));
+			ec_flags->Assign(4, new Val(${ccore.SUPPORT_MONITOR_LAYOUT_PDU}, TYPE_BOOL));
+			ec_flags->Assign(5, new Val(${ccore.SUPPORT_NETCHAR_AUTODETECT}, TYPE_BOOL));
+			ec_flags->Assign(6, new Val(${ccore.SUPPORT_DYNVC_GFX_PROTOCOL}, TYPE_BOOL));
+			ec_flags->Assign(7, new Val(${ccore.SUPPORT_DYNAMIC_TIME_ZONE}, TYPE_BOOL));
+			ec_flags->Assign(8, new Val(${ccore.SUPPORT_HEARTBEAT_PDU}, TYPE_BOOL));
+
+			RecordVal* ccd = new RecordVal(BifType::Record::RDP::ClientCoreData);
+			ccd->Assign(0, new Val(${ccore.version_major}, TYPE_COUNT));
+			ccd->Assign(1, new Val(${ccore.version_minor}, TYPE_COUNT));
+			ccd->Assign(2, new Val(${ccore.desktop_width}, TYPE_COUNT));
+			ccd->Assign(3, new Val(${ccore.desktop_height}, TYPE_COUNT));
+			ccd->Assign(4, new Val(${ccore.color_depth}, TYPE_COUNT));
+			ccd->Assign(5, new Val(${ccore.sas_sequence}, TYPE_COUNT));
+			ccd->Assign(6, new Val(${ccore.keyboard_layout}, TYPE_COUNT));
+			ccd->Assign(7, new Val(${ccore.client_build}, TYPE_COUNT));
+			ccd->Assign(8, utf16_to_utf8_val(${ccore.client_name}));
+			ccd->Assign(9, new Val(${ccore.keyboard_type}, TYPE_COUNT));
+			ccd->Assign(10, new Val(${ccore.keyboard_sub}, TYPE_COUNT));
+			ccd->Assign(11, new Val(${ccore.keyboard_function_key}, TYPE_COUNT));
+			ccd->Assign(12, utf16_to_utf8_val(${ccore.ime_file_name}));
+			ccd->Assign(13, new Val(${ccore.post_beta2_color_depth}, TYPE_COUNT));
+			ccd->Assign(14, new Val(${ccore.client_product_id}, TYPE_COUNT));
+			ccd->Assign(15, new Val(${ccore.serial_number}, TYPE_COUNT));
+			ccd->Assign(16, new Val(${ccore.high_color_depth}, TYPE_COUNT));
+			ccd->Assign(17, new Val(${ccore.supported_color_depths}, TYPE_COUNT));
+			ccd->Assign(18, ec_flags);
+			ccd->Assign(19, utf16_to_utf8_val(${ccore.dig_product_id}));
+
+			BifEvent::generate_rdp_client_core_data(connection()->bro_analyzer(),
+			                                        connection()->bro_analyzer()->Conn(),
+			                                        ccd);
+			}
+
+		return true;
+		%}
+
+	function proc_rdp_server_security(ssd: Server_Security_Data): bool
+		%{
+		connection()->bro_analyzer()->ProtocolConfirmation();
+
+		if ( rdp_server_security )
+			BifEvent::generate_rdp_server_security(connection()->bro_analyzer(),
+			                                       connection()->bro_analyzer()->Conn(),
+			                                       ${ssd.encryption_method},
+			                                       ${ssd.encryption_level});
+
+		return true;
+		%}
+
+	function proc_rdp_server_certificate(cert: Server_Certificate): bool
+		%{
+		if ( rdp_server_certificate )
+			{
+			BifEvent::generate_rdp_server_certificate(connection()->bro_analyzer(),
+			                                          connection()->bro_analyzer()->Conn(),
+			                                          ${cert.cert_type},
+			                                          ${cert.permanently_issued});
+			}
+
+		return true;
+		%}
+
+	function proc_x509_cert_data(x509: X509_Cert_Data): bool
+		%{
+		const bytestring& cert = ${x509.cert};
+
+		ODesc file_handle;
+		file_handle.AddRaw("Analyzer::ANALYZER_RDP");
+		file_handle.Add(connection()->bro_analyzer()->Conn()->StartTime());
+		connection()->bro_analyzer()->Conn()->IDString(&file_handle);
+		string file_id = file_mgr->HashHandle(file_handle.Description());
+
+		file_mgr->DataIn(reinterpret_cast<const u_char*>(cert.data()),
+		                 cert.length(), 
+		                 connection()->bro_analyzer()->GetAnalyzerTag(),
+		                 connection()->bro_analyzer()->Conn(), 
+		                 false, // It seems there are only server certs?
+		                 file_id);
+		file_mgr->EndOfFile(file_id);
+
+		return true;
+		%}
+};
+
+refine typeattr Connect_Request += &let {
+	proc: bool = $context.flow.proc_rdp_connect_request(this);
+};
+
+refine typeattr RDP_Negotiation_Response += &let {
+	proc: bool = $context.flow.proc_rdp_negotiation_response(this);
+};
+
+refine typeattr RDP_Negotiation_Failure += &let {
+	proc: bool = $context.flow.proc_rdp_negotiation_failure(this);
+};
+
+refine typeattr Client_Core_Data += &let {
+	proc: bool = $context.flow.proc_rdp_client_core_data(this);
+};
+
+refine typeattr GCC_Server_Create_Response += &let {
+	proc: bool = $context.flow.proc_rdp_gcc_server_create_response(this);
+};
+
+refine typeattr Server_Security_Data += &let {
+	proc: bool = $context.flow.proc_rdp_server_security(this);
+};
+
+refine typeattr Server_Certificate += &let {
+	proc: bool = $context.flow.proc_rdp_server_certificate(this);
+};
+
+refine typeattr X509_Cert_Data += &let {
+	proc: bool = $context.flow.proc_x509_cert_data(this);
+};

src/analyzer/protocol/rdp/rdp-protocol.pac

@@ -0,0 +1,423 @@
+
+type TPKT(is_orig: bool) = record {
+	version:  uint8;
+	reserved: uint8;
+	tpkt_len: uint16;
+
+# These data structures are merged together into TPKT
+# because there are packets that report incorrect 
+# lengths in the tpkt length field.  No clue why.
+
+	cotp:     COTP(this);
+} &byteorder=bigendian &length=tpkt_len;
+
+type COTP(tpkt: TPKT) = record {
+	cotp_len:  uint8;
+	pdu:       uint8;
+	switch:    case pdu of {
+		0xd0    -> connect_confirm: Connect_Confirm(this);
+		0xe0    -> client_request:  Connect_Request(this);
+		0xf0    -> data:            DT_Data;
+
+		# In case we don't support the PDU we just
+		# consume the rest of it and throw it away.
+		default -> not_done:  bytestring &restofdata &transient;
+	};
+} &byteorder=littleendian;
+
+type DT_Data = record {
+	tpdu_number:              uint8;
+	# multiple octet variant of the ASN.1 type field, should handle this better.
+	application_defined_type: uint8;
+	application_type:         uint8;
+
+	data: case application_type of {
+		0x65    -> client: Client_Header; # 0x65 is a client
+		0x66    -> server: Server_Header; # 0x66 is a server
+		default -> none:   empty;
+	};
+} &byteorder=littleendian;
+
+######################################################################
+# Data Blocks
+######################################################################
+
+type Data_Header = record {
+	type:   uint16;
+	length: uint16;
+} &byteorder=littleendian;
+
+type Data_Block = record {
+	header: Data_Header;
+	block: case header.type of {
+		0xc001  -> client_core:       Client_Core_Data;
+		#0xc002  -> client_security:   Client_Security_Data;
+		#0xc003  -> client_network:    Client_Network_Data;
+		#0xc004  -> client_cluster:    Client_Cluster_Data;
+		#0xc005  -> client_monitor:    Client_Monitor_Data;
+		#0xc006  -> client_msgchannel: Client_MsgChannel_Data;
+		#0xc008  -> client_monitor_ex: Client_MonitorExtended_Data;
+		#0xc00A  -> client_multitrans: Client_MultiTransport_Data;
+
+		0x0c01  -> server_core:       Server_Core_Data(header);
+		0x0c02  -> server_security:   Server_Security_Data;
+		0x0c03  -> server_network:    Server_Network_Data;
+		#0x0c04  -> server_msgchannel: Server_MsgChannel_Data;
+		#0x0c08  -> server_multitrans: Server_MultiTransport_Data;
+
+		default -> unhandled:  bytestring &restofdata &transient;
+	} &length=header.length-4;
+} &byteorder=littleendian;
+
+######################################################################
+# Client X.224
+######################################################################
+
+type Connect_Request(cotp: COTP) = record {
+	destination_reference: uint16;
+	source_reference:      uint16;
+	flow_control:          uint8;
+	cookie_mstshash:       RE/Cookie: mstshash\=/;
+	cookie_value:          RE/[^\x0d]*/;
+	cookie_terminator:     RE/\x0d\x0a/;
+	# Terrifying little case statement to figure out if there
+	# is any data left in the COTP structure.
+	switch1:   case (offsetof(switch1) + 2 - cotp.cotp_len - 1) of {
+		0       -> none:        empty;
+		default -> rdp_neg_req: RDP_Negotiation_Request;
+	};
+} &byteorder=littleendian;
+
+type RDP_Negotiation_Request = record {
+	type:                uint8;
+	flags:               uint8;
+	length:              uint16; # must be set to 8
+	requested_protocols: uint32;
+} &let {
+	PROTOCOL_RDP:       bool = requested_protocols & 0x00;
+	PROTOCOL_SSL:       bool = requested_protocols & 0x01;
+	PROTOCOL_HYBRID:    bool = requested_protocols & 0x02;
+	PROTOCOL_HYBRID_EX: bool = requested_protocols & 0x08;
+} &byteorder=littleendian;
+
+######################################################################
+# Server X.224
+######################################################################
+
+type Connect_Confirm(cotp: COTP) = record {
+	destination_reference: uint16;
+	source_reference:      uint16;
+	flags:                 uint8;
+	# Terrifying little case statement to figure out if there
+	# is any data left in the COTP structure.
+	switch1: case (offsetof(switch1) + 2 - cotp.cotp_len - 1) of {
+		0       -> none1:    empty;
+		default -> response: Connect_Confirm_Record;
+	};
+};
+
+type Connect_Confirm_Record = record {
+	response_type:         uint8;
+	switch1: case response_type of {
+		0x02 -> neg_resp: RDP_Negotiation_Response;
+		0x03 -> neg_fail: RDP_Negotiation_Failure;
+	};
+};
+
+type RDP_Negotiation_Response = record {
+	flags:               uint8;
+	length:              uint16; # must be set to 8
+	selected_protocol:   uint32;
+} &let {
+	# Seems to be SSL encrypted (maybe CredSSP also?) 
+	# after this message if the selected_protocol is > 0.
+	enc_ssl: bool = $context.connection.go_encrypted(selected_protocol) &if(selected_protocol > 0);
+} &byteorder=littleendian;
+
+type RDP_Negotiation_Failure = record {
+	flags: uint8;
+	length: uint16;
+	failure_code: uint32;
+} &byteorder=littleendian;
+
+######################################################################
+# Client MCS
+######################################################################
+
+type Client_Header = record {
+	type_length:               ASN1Integer;
+	calling_domain_selector:   ASN1OctetString;
+	called_domain_selector:    ASN1OctetString;
+	upward_flag:               ASN1Boolean;
+	target_parameters:         ASN1SequenceMeta;
+	targ_parameters_pad:       bytestring &length=target_parameters.encoding.length &transient;
+	minimum_parameters:        ASN1SequenceMeta;
+	min_parameters_pad:        bytestring &length=minimum_parameters.encoding.length &transient;
+	maximum_parameters:        ASN1SequenceMeta;
+	max_parameters_pad:        bytestring &length=maximum_parameters.encoding.length &transient;
+	# BER encoded OctetString and long variant, can be safely skipped for now
+	user_data_length:          uint32;
+	gcc_connection_data:       GCC_Client_Connection_Data;
+	gcc_client_create_request: GCC_Client_Create_Request;
+	data_blocks:               Data_Block[] &until($input.length() == 0);
+};
+
+type GCC_Client_Connection_Data = record {
+	key_object_length:        uint16;
+	key_object:               uint8[key_object_length];
+	connect_data_connect_pdu: uint16;
+} &byteorder=bigendian;
+
+type GCC_Client_Create_Request = record {
+	extension_bit:           uint8;
+	privileges:              uint8;
+	numeric_length:          uint8;
+	numeric:                 uint8;
+	termination_method:      uint8;
+	number_user_data_sets:   uint8;
+	user_data_value_present: uint8;
+	h221_nonstandard_length: uint8;
+	h221_nonstandard_key:    RE/Duca/;
+	user_data_value_length:  uint16;
+} &byteorder=bigendian;
+
+type Client_Core_Data = record {
+	version_major:            uint16;
+	version_minor:            uint16;
+	desktop_width:            uint16;
+	desktop_height:           uint16;
+	color_depth:              uint16;
+	sas_sequence:             uint16;
+	keyboard_layout:          uint32;
+	client_build:             uint32;
+	client_name:              bytestring &length=32;
+	keyboard_type:            uint32;
+	keyboard_sub:             uint32;
+	keyboard_function_key:    uint32;
+	ime_file_name:            bytestring &length=64;
+	# Everything below here is optional and should be handled better.
+	# If some of these fields aren't included it could lead to parse failure.
+	post_beta2_color_depth:   uint16;
+	client_product_id:        uint16;
+	serial_number:            uint32;
+	high_color_depth:         uint16;
+	supported_color_depths:   uint16;
+	early_capability_flags:   uint16;
+	dig_product_id:           bytestring &length=64;
+	# There are more optional fields here but they are  
+	# annoying to optionally parse in binpac.
+	# Documented here: https://msdn.microsoft.com/en-us/library/cc240510.aspx
+} &let {
+	SUPPORT_ERRINFO_PDU:        bool = early_capability_flags & 0x01;
+	WANT_32BPP_SESSION:         bool = early_capability_flags & 0x02;
+	SUPPORT_STATUSINFO_PDU:     bool = early_capability_flags & 0x04;
+	STRONG_ASYMMETRIC_KEYS:     bool = early_capability_flags & 0x08;
+	SUPPORT_MONITOR_LAYOUT_PDU: bool = early_capability_flags & 0x40;
+	SUPPORT_NETCHAR_AUTODETECT: bool = early_capability_flags & 0x80;
+	SUPPORT_DYNVC_GFX_PROTOCOL: bool = early_capability_flags & 0x0100;
+	SUPPORT_DYNAMIC_TIME_ZONE:  bool = early_capability_flags & 0x0200;
+	SUPPORT_HEARTBEAT_PDU:      bool = early_capability_flags & 0x0400;
+} &byteorder=littleendian;
+
+######################################################################
+# Server MCS
+######################################################################
+
+type Server_Header = record {
+	# We don't need this value, but it's ASN.1 integer in definite length
+	# so I think we can skip over it.
+	type_length:                        uint8[3];
+	connect_response_result:            ASN1Enumerated;
+	connect_response_called_id:         ASN1Integer;
+	connect_response_domain_parameters: ASN1SequenceMeta;
+	# Skipping over domain parameters for now.
+	domain_parameters:                  bytestring &length=connect_response_domain_parameters.encoding.length &transient;
+	# I think this is another definite length encoded value.
+	user_data_length:                   uint32;
+	gcc_connection_data:                GCC_Server_Connection_Data;
+	gcc_create_response:                GCC_Server_Create_Response;
+	data_blocks:                        Data_Block[] &until($input.length() == 0);
+} &byteorder=littleendian;
+
+type GCC_Server_Connection_Data = record {
+	key_object_length:        uint16;
+	key_object:               uint8[key_object_length];
+	connect_data_connect_pdu: uint8;
+} &byteorder=bigendian;
+
+type GCC_Server_Create_Response = record {
+	extension_bit:           uint8;
+	node_id:                 uint16;
+	tag_length:              uint8;
+	tag:                     uint8;
+	result:                  uint8;
+	number_user_data_sets:   uint8;
+	user_data_value_present: uint8;
+	h221_nonstandard_length: uint8;
+	h221_nonstandard_key:    RE/McDn/;
+	user_data_value_length:  uint16;
+} &byteorder=bigendian;
+
+type Server_Core_Data(h: Data_Header) = record {
+	version_major: uint16;
+	version_minor: uint16;
+	switch1:       case h.length of {
+		8       -> none:                       empty;
+		default -> client_requested_protocols: uint32;
+	};
+} &byteorder=littleendian;
+
+type Server_Network_Data = record {
+	mcs_channel_id: uint16;
+	channel_count:  uint16;
+} &byteorder=littleendian;
+
+type Server_Security_Data = record {
+	encryption_method:      uint32;
+	encryption_level:       uint32;
+	server_random_length:   uint32;
+	server_cert_length:     uint32;
+	server_random:          bytestring &length=server_random_length;
+	server_certificate:     Server_Certificate &length=server_cert_length;
+} &let {
+	# Seems to be encrypted after this message if 
+	# encryption level is >0
+	# 0 means RDP encryption.
+	enc: bool = $context.connection.go_encrypted(0) &if(encryption_method > 0 && encryption_level > 0);
+} &byteorder=littleendian;
+
+type Server_Certificate = record {
+	version: uint32;
+	switch:  case cert_type of {
+		0x01 -> proprietary: Server_Proprietary_Cert(this);
+		0x02 -> x509:        X509;
+	};
+} &let {
+	cert_type:          uint32 = version & 0x7FFFFFFF;
+	permanently_issued: bool   = (version & 0x80000000) == 0;
+} &byteorder=littleendian;
+
+type Server_Proprietary_Cert(cert: Server_Certificate) = record {
+	signature_algorithm:    uint32;
+	key_algorithm:          uint32;
+	public_key_blob_type:   uint16;
+	public_key_blob_length: uint16;
+	public_key_blob:        Public_Key_Blob &length=public_key_blob_length;
+	signature_blob_type:    uint16;
+	signature_blob_length:  uint16;
+	signature_blob:         bytestring &length=signature_blob_length;
+} &byteorder=littleendian;
+
+type Public_Key_Blob = record {
+	magic:           bytestring &length=4;
+	key_length:      uint32;
+	bit_length:      uint32;
+	public_exponent: uint32;
+	modulus:         bytestring &length=key_length;
+} &byteorder=littleendian;
+
+type X509 = record {
+	num_of_certs: uint32;
+	certs: X509_Cert_Data[num_of_certs];
+} &byteorder=littleendian;
+
+type X509_Cert_Data = record {
+	cert_len: uint32;
+	cert: bytestring &length=cert_len;
+} &byteorder=littleendian;
+
+######################################################################
+# ASN.1 Encodings
+######################################################################
+
+type ASN1Encoding = record {
+	meta:    ASN1EncodingMeta;
+	content: bytestring &length = meta.length;
+};
+
+type ASN1EncodingMeta = record {
+	tag:      uint8;
+	len:      uint8;
+	more_len: bytestring &length = long_len ? len & 0x7f : 0;
+} &let {
+	long_len: bool = (len & 0x80) > 0;
+	length:   uint64 = long_len ? binary_to_int64(more_len) : len & 0x7f;
+};
+
+type ASN1SequenceMeta = record {
+	encoding: ASN1EncodingMeta;
+};
+
+type ASN1Integer = record {
+	encoding: ASN1Encoding;
+};
+
+type ASN1OctetString = record {
+	encoding: ASN1Encoding;
+};
+
+type ASN1ObjectIdentifier = record {
+	encoding: ASN1Encoding;
+};
+
+type ASN1Boolean = record {
+	encoding: ASN1Encoding;
+};
+
+type ASN1Enumerated = record {
+	encoding: ASN1Encoding;
+};
+
+######################################################################
+# ASN.1 Conversion Functions
+######################################################################
+
+function binary_to_int64(bs: bytestring): int64
+	%{
+	int64 rval = 0;
+	for ( int i = 0; i < bs.length(); ++i )
+		{
+		uint64 byte = bs[i];
+		rval |= byte << (8 * (bs.length() - (i + 1)));
+		}
+
+	return rval;
+	%}
+
+refine connection RDP_Conn += {
+
+	%member{
+		bool is_encrypted_;
+		uint32 encryption_method_;
+	%}
+
+	%init{
+		is_encrypted_ = false;
+		encryption_method_ = 0;
+	%}
+
+	function go_encrypted(method: uint32): bool
+		%{
+		is_encrypted_ = true;
+		encryption_method_ = method;
+
+		if ( rdp_begin_encryption )
+			{
+			BifEvent::generate_rdp_begin_encryption(bro_analyzer(),
+			                                        bro_analyzer()->Conn(),
+			                                        ${method});
+			}
+
+		return is_encrypted_;
+		%}
+
+	function is_encrypted(): bool
+		%{
+		return is_encrypted_;
+		%}
+
+	function encryption_method(): uint32
+		%{
+		return encryption_method_;
+		%}
+};

src/analyzer/protocol/rdp/rdp.pac

@@ -0,0 +1,25 @@
+%include binpac.pac
+%include bro.pac
+
+%extern{
+	#include "events.bif.h"
+%}
+
+analyzer RDP withcontext {
+	connection: RDP_Conn;
+	flow:       RDP_Flow;
+};
+
+# Our connection consists of two flows, one in each direction.
+connection RDP_Conn(bro_analyzer: BroAnalyzer) {
+	upflow   = RDP_Flow(true);
+	downflow = RDP_Flow(false);
+};
+
+%include rdp-protocol.pac
+
+flow RDP_Flow(is_orig: bool) {
+	flowunit = TPKT(is_orig) withcontext(connection, this);
+};
+
+%include rdp-analyzer.pac

src/analyzer/protocol/rdp/types.bif

@@ -0,0 +1,5 @@
+
+module RDP;
+
+type EarlyCapabilityFlags: record;
+type ClientCoreData: record;

src/analyzer/protocol/smtp/SMTP.cc

@@ -896,7 +896,7 @@ void SMTP_Analyzer::BeginData()
 	skip_data = 0; // reset the flag at the beginning of the mail
 	if ( mail != 0 )
 		{
-		reporter->Warning("nested mail transaction");
+		Weird("smtp_nested_mail_transaction");
 		mail->Done();
 		delete mail;
 		}
@@ -907,7 +907,7 @@ void SMTP_Analyzer::BeginData()
 void SMTP_Analyzer::EndData()
 	{
 	if ( ! mail )
-		reporter->Warning("Unmatched end of data");
+		Weird("smtp_unmatched_end_of_data");
 	else
 		{
 		mail->Done();

src/analyzer/protocol/socks/SOCKS.cc

@@ -57,8 +57,7 @@ void SOCKS_Analyzer::DeliverStream(int len, const u_char* data, bool orig)
 		// with the rest of the conneciton.
 		//
 		// Note that we assume that no payload data arrives before both endpoints
-		// are done with there part of the SOCKS protocol.
-
+		// are done with their part of the SOCKS protocol.
 		if ( ! pia )
 			{
 			pia = new pia::PIA_TCP(Conn());

src/analyzer/protocol/socks/events.bif

@@ -27,3 +27,19 @@ event socks_request%(c: connection, version: count, request_type: count, sa: SOC
 ## p: The destination port for the proxied traffic.
 event socks_reply%(c: connection, version: count, reply: count, sa: SOCKS::Address, p: port%);
 
+## Generated when a SOCKS client performs username and password based login.
+##
+## c: The parent connection of the proxy.
+##
+## user: The given username.
+##
+## password: The given password.
+event socks_login_userpass_request%(c: connection, user: string, password: string%);
+
+## Generated when a SOCKS server replies to a username/password login attempt.
+##
+## c: The parent connection of the proxy.
+## 
+## code: The response code for the attempted login.
+event socks_login_userpass_reply%(c: connection, code: count%);
+

src/analyzer/protocol/socks/socks-analyzer.pac

@@ -148,6 +148,37 @@ refine connection SOCKS_Conn += {
 		return true;
 		%}
 
+	function socks5_auth_request_userpass(request: SOCKS5_Auth_Request_UserPass_v1): bool
+		%{
+		StringVal* user = new StringVal(${request.username}.length(), (const char*) ${request.username}.begin());
+		StringVal* pass = new StringVal(${request.password}.length(), (const char*) ${request.password}.begin());
+		
+		BifEvent::generate_socks_login_userpass_request(bro_analyzer(),
+		                                                bro_analyzer()->Conn(),
+		                                                user, pass);
+		return true;
+		%}
+
+	function socks5_unsupported_authentication_method(auth_method: uint8): bool
+		%{
+		reporter->Weird(bro_analyzer()->Conn(), fmt("socks5_unsupported_authentication_method_%d", auth_method));
+		return true;
+		%}
+
+	function socks5_unsupported_authentication_version(auth_method: uint8, version: uint8): bool
+		%{
+		reporter->Weird(bro_analyzer()->Conn(), fmt("socks5_unsupported_authentication_%d_%d", auth_method, version));
+		return true;
+		%}
+	
+	function socks5_auth_reply_userpass(reply: SOCKS5_Auth_Reply_UserPass_v1): bool
+		%{
+		BifEvent::generate_socks_login_userpass_reply(bro_analyzer(),
+		                                              bro_analyzer()->Conn(),
+		                                              ${reply.code});
+		return true;
+		%}
+
 	function version_error(version: uint8): bool
 		%{
 		bro_analyzer()->ProtocolViolation(fmt("unsupported/unknown SOCKS version %d", version));
@@ -176,3 +207,22 @@ refine typeattr SOCKS5_Request += &let {
 refine typeattr SOCKS5_Reply += &let {
 	proc: bool = $context.connection.socks5_reply(this);
 };
+
+refine typeattr SOCKS5_Auth_Negotiation_Reply += &let {
+};
+
+refine typeattr SOCKS5_Auth_Request_UserPass_v1 += &let {
+	proc: bool = $context.connection.socks5_auth_request_userpass(this);
+};
+
+refine typeattr SOCKS5_Auth_Reply_UserPass_v1 += &let {
+	proc: bool = $context.connection.socks5_auth_reply_userpass(this);
+};
+
+refine typeattr SOCKS5_Unsupported_Authentication_Method += &let {
+	proc: bool = $context.connection.socks5_unsupported_authentication_method($context.connection.v5_auth_method());
+};
+
+refine typeattr SOCKS5_Unsupported_Authentication_Version += &let {
+	proc: bool = $context.connection.socks5_unsupported_authentication_version($context.connection.v5_auth_method(), version);
+};

src/analyzer/protocol/socks/socks-protocol.pac

@@ -1,10 +1,15 @@
 
+type SOCKS_Message(is_orig: bool) = case $context.connection.v5_in_auth_sub_negotiation() of {
+	true  -> auth: SOCKS5_Auth_Message(is_orig);
+	false -> msg:  SOCKS_Version(is_orig);
+};
+
 type SOCKS_Version(is_orig: bool) = record {
 	version: uint8;
 	msg:     case version of {
-		4       -> socks4_msg:     SOCKS4_Message(is_orig);
-		5       -> socks5_msg:     SOCKS5_Message(is_orig);
-		default -> socks_msg_fail: SOCKS_Version_Error(version);
+		4       -> socks4_msg:      SOCKS4_Message(is_orig);
+		5       -> socks5_msg:      SOCKS5_Message(is_orig);
+		default -> socks_msg_fail:  SOCKS_Version_Error(version);
 	};
 };
 
@@ -14,10 +19,11 @@ type SOCKS_Version_Error(version: uint8) = record {
 
 # SOCKS5 Implementation
 type SOCKS5_Message(is_orig: bool) = case $context.connection.v5_past_authentication() of {
-	true  -> msg:  SOCKS5_Real_Message(is_orig);
 	false -> auth: SOCKS5_Auth_Negotiation(is_orig);
+	true  -> msg:  SOCKS5_Real_Message(is_orig);
 };
 
+
 type SOCKS5_Auth_Negotiation(is_orig: bool) = case is_orig of {
 	true  -> req:  SOCKS5_Auth_Negotiation_Request;
 	false -> rep:  SOCKS5_Auth_Negotiation_Reply;
@@ -31,7 +37,61 @@ type SOCKS5_Auth_Negotiation_Request = record {
 type SOCKS5_Auth_Negotiation_Reply = record {
 	selected_auth_method: uint8;
 } &let {
+	in_auth_sub_neg = $context.connection.set_v5_in_auth_sub_negotiation(selected_auth_method == 0 || selected_auth_method == 0xff ? false : true);
 	past_auth = $context.connection.set_v5_past_authentication();
+	set_auth = $context.connection.set_v5_auth_method(selected_auth_method);
+};
+
+type SOCKS5_Auth_Message(is_orig: bool) = case is_orig of {
+	true  -> req: SOCKS5_Auth_Request;
+	false -> rep: SOCKS5_Auth_Reply;
+};
+
+type SOCKS5_Auth_Request = case $context.connection.v5_auth_method() of {
+	0x02    -> userpass    : SOCKS5_Auth_Request_UserPass;
+	default -> unsupported : SOCKS5_Unsupported_Authentication_Method;
+};
+
+type SOCKS5_Unsupported_Authentication_Method = record {
+	crap: bytestring &restofdata;
+};
+
+type SOCKS5_Unsupported_Authentication_Version(version: uint8) = record {
+	crap: bytestring &restofdata;
+};
+
+type SOCKS5_Auth_Request_UserPass = record {
+	version: uint8;
+	msg:     case version of {
+		1       -> v1:           SOCKS5_Auth_Request_UserPass_v1;
+		default -> unsupported:  SOCKS5_Unsupported_Authentication_Version(version);
+	};
+};
+
+type SOCKS5_Auth_Request_UserPass_v1 = record {
+	ulen     : uint8;
+	username : bytestring &length=ulen;
+	plen     : uint8;
+	password : bytestring &length=plen;
+};
+
+type SOCKS5_Auth_Reply = case $context.connection.v5_auth_method() of {
+	0x02    -> userpass    : SOCKS5_Auth_Reply_UserPass;
+	default -> unsupported : SOCKS5_Unsupported_Authentication_Method;
+} &let {
+	in_auth_sub_neg = $context.connection.set_v5_in_auth_sub_negotiation(false);
+};
+
+type SOCKS5_Auth_Reply_UserPass = record {
+	version: uint8;
+	msg:     case version of {
+		1       -> v1:           SOCKS5_Auth_Reply_UserPass_v1;
+		default -> unsupported:  SOCKS5_Unsupported_Authentication_Version(version);
+	};
+};
+
+type SOCKS5_Auth_Reply_UserPass_v1 = record {
+	code : uint8;
 };
 
 type SOCKS5_Real_Message(is_orig: bool) = case is_orig of {
@@ -55,10 +115,10 @@ type SOCKS5_Address = record {
 } &byteorder = bigendian;
 
 type SOCKS5_Request = record {
-	command: uint8;
-	reserved: uint8;
-	remote_name: SOCKS5_Address;
-	port: uint16;
+	command     : uint8;
+	reserved    : uint8;
+	remote_name : SOCKS5_Address;
+	port        : uint16;
 } &byteorder = bigendian;
 
 type SOCKS5_Reply = record {
@@ -98,13 +158,28 @@ type SOCKS4_Reply = record {
 
 refine connection SOCKS_Conn += {
 	%member{
+		bool v5_in_auth_sub_negotiation_;
 		bool v5_authenticated_;
+		uint8 selected_auth_method_;
 	%}
 
 	%init{
+		v5_in_auth_sub_negotiation_ = false;
 		v5_authenticated_ = false;
+		selected_auth_method_ = 255;
 	%}
 
+	function v5_in_auth_sub_negotiation(): bool
+		%{
+		return v5_in_auth_sub_negotiation_;
+		%}
+
+	function set_v5_in_auth_sub_negotiation(b: bool): bool
+		%{
+		v5_in_auth_sub_negotiation_ = b;
+		return true;
+		%}
+
 	function v5_past_authentication(): bool
 		%{
 		return v5_authenticated_;
@@ -115,5 +190,16 @@ refine connection SOCKS_Conn += {
 		v5_authenticated_ = true;
 		return true;
 		%}
+
+	function set_v5_auth_method(method: uint8): bool
+		%{
+		selected_auth_method_ = method;
+		return true;
+		%}
+
+	function v5_auth_method(): uint8
+		%{
+		return selected_auth_method_;
+		%}
 };
 

src/analyzer/protocol/socks/socks.pac

@@ -20,7 +20,7 @@ connection SOCKS_Conn(bro_analyzer: BroAnalyzer) {
 %include socks-protocol.pac
 
 flow SOCKS_Flow(is_orig: bool) {
-	datagram = SOCKS_Version(is_orig) withcontext(connection, this);
+	datagram = SOCKS_Message(is_orig) withcontext(connection, this);
 };
 
 %include socks-analyzer.pac

src/analyzer/protocol/ssh/CMakeLists.txt

@@ -4,6 +4,8 @@ include(BroPlugin)
 include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
 
 bro_plugin_begin(Bro SSH)
-bro_plugin_cc(SSH.cc Plugin.cc)
-bro_plugin_bif(events.bif)
+	bro_plugin_cc(SSH.cc Plugin.cc)
+	bro_plugin_bif(types.bif)
+	bro_plugin_bif(events.bif)
+	bro_plugin_pac(ssh.pac ssh-analyzer.pac ssh-protocol.pac)
 bro_plugin_end()

src/analyzer/protocol/ssh/Plugin.cc

@@ -1,25 +1,24 @@
 // See the file  in the main distribution directory for copyright.
 
-
 #include "plugin/Plugin.h"
-
 #include "SSH.h"
 
 namespace plugin {
-namespace Bro_SSH {
+	namespace Bro_SSH {
 
-class Plugin : public plugin::Plugin {
-public:
-	plugin::Configuration Configure()
-		{
-		AddComponent(new ::analyzer::Component("SSH", ::analyzer::ssh::SSH_Analyzer::Instantiate));
+		class Plugin : public plugin::Plugin {
+		public:
+			plugin::Configuration Configure()
+				{
+				AddComponent(new ::analyzer::Component("SSH", ::analyzer::SSH::SSH_Analyzer::Instantiate));
+
+				plugin::Configuration config;
+				config.name = "Bro::SSH";
+				config.description = "Secure Shell analyzer";
+				return config;
+				}
+			} plugin;
 
-		plugin::Configuration config;
-		config.name = "Bro::SSH";
-		config.description = "SSH analyzer";
-		return config;
 		}
-} plugin;
+	}
 
-}
-}

src/analyzer/protocol/ssh/SSH.cc

@@ -1,105 +1,148 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 
-#include "config.h"
-
-#include <ctype.h>
-
-#include "NetVar.h"
 #include "SSH.h"
-#include "Event.h"
-#include "analyzer/protocol/tcp/ContentLine.h"
 
+#include "analyzer/protocol/tcp/TCP_Reassembler.h"
+
+#include "Reporter.h"
+
+#include "types.bif.h"
 #include "events.bif.h"
 
-using namespace analyzer::ssh;
+using namespace analyzer::SSH;
 
 SSH_Analyzer::SSH_Analyzer(Connection* c)
-: tcp::TCP_ApplicationAnalyzer("SSH", c)
+	: tcp::TCP_ApplicationAnalyzer("SSH", c)
 	{
-	orig = new tcp::ContentLine_Analyzer(c, true);
-	orig->SetSkipPartial(true);
-	orig->SetCRLFAsEOL(LF_as_EOL);
-	AddSupportAnalyzer(orig);
-
-	resp = new tcp::ContentLine_Analyzer(c, false);
-	resp->SetSkipPartial(true);
-	resp->SetCRLFAsEOL(LF_as_EOL);
-	AddSupportAnalyzer(resp);
+	interp = new binpac::SSH::SSH_Conn(this);
+	had_gap = false;
+	auth_decision_made = false;
+	skipped_banner = false;
+	service_accept_size = 0;
+	userauth_failure_size = 0;
 	}
 
-void SSH_Analyzer::DeliverStream(int length, const u_char* data, bool is_orig)
+SSH_Analyzer::~SSH_Analyzer()
 	{
-	tcp::TCP_ApplicationAnalyzer::DeliverStream(length, data, is_orig);
+	delete interp;
+	}
 
-	// We're all done processing this endpoint - flag it as such,
-	// before we even determine whether we have any event generation
-	// work to do, to make sure we don't do any further work on it.
-	if ( is_orig )
-		orig->SetSkipDeliveries(true);
-	else
-		resp->SetSkipDeliveries(true);
+void SSH_Analyzer::Done()
+	{
+	tcp::TCP_ApplicationAnalyzer::Done();
 
-	if ( TCP() )
+	interp->FlowEOF(true);
+	interp->FlowEOF(false);
+	}
+
+void SSH_Analyzer::EndpointEOF(bool is_orig)
+	{
+	tcp::TCP_ApplicationAnalyzer::EndpointEOF(is_orig);
+	interp->FlowEOF(is_orig);
+	}
+
+void SSH_Analyzer::DeliverStream(int len, const u_char* data, bool orig)
+	{
+	tcp::TCP_ApplicationAnalyzer::DeliverStream(len, data, orig);
+
+	assert(TCP());
+	if ( TCP()->IsPartial() )
+		return;
+
+	if ( had_gap )
+		// If only one side had a content gap, we could still try to
+		// deliver data to the other side if the script layer can handle this.
+		return;
+
+	if ( interp->get_state(orig) == binpac::SSH::ENCRYPTED )
 		{
-		// Don't try to parse version if there has already been a gap.
-		tcp::TCP_Endpoint* endp = is_orig ? TCP()->Orig() : TCP()->Resp();
-		if ( endp->HadGap() )
+		if ( ssh_encrypted_packet )
+			BifEvent::generate_ssh_encrypted_packet(interp->bro_analyzer(), interp->bro_analyzer()->Conn(),
+				orig, len);
+
+		if ( ! auth_decision_made )
+			ProcessEncrypted(len, orig);
+
+		return;
+		}
+
+	try
+		{
+		interp->NewData(orig, data, data + len);
+		}
+	catch ( const binpac::Exception& e )
+		{
+		ProtocolViolation(fmt("Binpac exception: %s", e.c_msg()));
+		}
+	}
+
+void SSH_Analyzer::Undelivered(uint64 seq, int len, bool orig)
+	{
+	tcp::TCP_ApplicationAnalyzer::Undelivered(seq, len, orig);
+	had_gap = true;
+	interp->NewGap(orig, len);
+	}
+
+void SSH_Analyzer::ProcessEncrypted(int len, bool orig)
+	{
+	// We're interested in messages from the server for SSH2
+	if ( ! orig && (interp->get_version() == binpac::SSH::SSH2) )
+		{
+		// The first thing we see and want to know is the length of
+		// SSH_MSG_SERVICE_REQUEST, which has a fixed (decrypted) size
+		// of 24 bytes (17 for content pad-aligned to 8-byte
+		// boundaries)
+		if ( ! service_accept_size )
+			{
+			service_accept_size = len;
 			return;
-		}
-
-	const char* line = (const char*) data;
-
-	// The SSH identification looks like this:
-	//
-	//     SSH-<protocolmajor>.<protocolminor>-<version>\n
-	//
-	// We're interested in the "version" part here.
-
-	if ( length < 4 || memcmp(line, "SSH-", 4) != 0 )
-		{
-		Weird("malformed_ssh_identification");
-		ProtocolViolation("malformed ssh identification", line, length);
-		return;
-		}
-
-	int i;
-	for ( i = 4; i < length && line[i] != '-'; ++i )
-		;
-
-	if ( TCP() )
-		{
-		if ( length >= i )
-			{
-			IPAddr dst;
-
-			if ( is_orig )
-				dst = TCP()->Orig()->dst_addr;
-			else
-				dst = TCP()->Resp()->dst_addr;
-
-			if ( Conn()->VersionFoundEvent(dst, line + i,
-							length - i) )
-				ProtocolConfirmation();
-			else
-				ProtocolViolation("malformed ssh version",
-							line, length);
 			}
-		else
+
+		// If our user can authenticate via the "none" method, this
+		// packet will be a SSH_MSG_USERAUTH_SUCCESS, which has a
+		// fixed (decrypted) size of 8 bytes (1 for content
+		// pad-aligned to 8-byte boundaries). relative_len would be
+		// -16.
+		if ( ! userauth_failure_size && (len + 16 == service_accept_size) )
 			{
-			Weird("malformed_ssh_version");
-			ProtocolViolation("malformed ssh version", line, length);
+			auth_decision_made = true;
+			if ( ssh_auth_successful )
+				BifEvent::generate_ssh_auth_successful(interp->bro_analyzer(), interp->bro_analyzer()->Conn(), true);
+			return;
+			}
+
+		// Normally, this packet would be a SSH_MSG_USERAUTH_FAILURE
+		// message, with a variable length, depending on the
+		// authentication methods the server supports. If it's too
+		// big, it might contain a pre-auth MOTD/banner, so we'll just
+		// skip it.
+		if ( ! userauth_failure_size )
+			{
+			if ( ! skipped_banner && (len - service_accept_size) > 256 )
+				{
+				skipped_banner = true;
+				return;
+				}
+			userauth_failure_size = len;
+			return;
+			}
+
+		// If we've already seen a failure, let's see if this is
+		// another packet of the same size.
+		if ( len == userauth_failure_size )
+			{
+			if ( ssh_auth_failed )
+				BifEvent::generate_ssh_auth_failed(interp->bro_analyzer(), interp->bro_analyzer()->Conn());
+			return;
+			}
+
+		// ...or a success packet.
+		if ( len - service_accept_size == -16 )
+			{
+			auth_decision_made = true;
+			if ( ssh_auth_successful )
+				BifEvent::generate_ssh_auth_successful(interp->bro_analyzer(), interp->bro_analyzer()->Conn(), false);
+			return;
 			}
 		}
-
-	// Generate SSH events.
-	EventHandlerPtr event = is_orig ?
-				ssh_client_version : ssh_server_version;
-	if ( ! event )
-		return;
-
-	val_list* vl = new val_list;
-	vl->append(BuildConnVal());
-	vl->append(new StringVal(length, line));
-
-	ConnectionEvent(event, vl);
 	}

src/analyzer/protocol/ssh/SSH.h

@@ -3,25 +3,46 @@
 #ifndef ANALYZER_PROTOCOL_SSH_SSH_H
 #define ANALYZER_PROTOCOL_SSH_SSH_H
 
+#include "events.bif.h"
+
 #include "analyzer/protocol/tcp/TCP.h"
-#include "analyzer/protocol/tcp/ContentLine.h"
+#include "ssh_pac.h"
 
-namespace analyzer { namespace ssh {
+namespace analyzer {
+	namespace SSH {
+		class SSH_Analyzer : public tcp::TCP_ApplicationAnalyzer {
 
-class SSH_Analyzer : public tcp::TCP_ApplicationAnalyzer {
-public:
-	SSH_Analyzer(Connection* conn);
+		public:
+			SSH_Analyzer(Connection* conn);
+			virtual ~SSH_Analyzer();
 
-	virtual void DeliverStream(int len, const u_char* data, bool orig);
+			// Overriden from Analyzer.
+			virtual void Done();
+			virtual void DeliverStream(int len, const u_char* data, bool orig);
+			virtual void Undelivered(uint64 seq, int len, bool orig);
 
-	static analyzer::Analyzer* Instantiate(Connection* conn)
-		{ return new SSH_Analyzer(conn); }
+			// Overriden from tcp::TCP_ApplicationAnalyzer.
+			virtual void EndpointEOF(bool is_orig);
 
-private:
-	tcp::ContentLine_Analyzer* orig;
-	tcp::ContentLine_Analyzer* resp;
-};
+			static analyzer::Analyzer* Instantiate(Connection* conn)
+				{ return new SSH_Analyzer(conn); }
 
-} } // namespace analyzer::* 
+		protected:
+			binpac::SSH::SSH_Conn* interp;
 
+			void ProcessEncrypted(int len, bool orig);
+
+			bool had_gap;
+
+			// Packet analysis stuff
+			bool auth_decision_made;
+			bool skipped_banner;
+
+			int service_accept_size;
+			int userauth_failure_size;
+
+			};
+
+		}
+	}
 #endif

src/analyzer/protocol/ssh/consts.pac

@@ -0,0 +1,126 @@
+enum version {
+	SSH1 = 1,
+	SSH2 = 2,
+	UNK  = 3,
+};
+
+enum state {
+	VERSION_EXCHANGE = 0,
+	KEX_INIT         = 1,
+	KEX_DH_GEX       = 2,
+	KEX_DH           = 3,
+	KEX_ECC          = 4,
+	KEX_GSS          = 5,
+	KEX_RSA          = 6,
+	ENCRYPTED        = 7,
+};
+
+# diffie-hellman-group1-sha1	[RFC4253]	Section 8.1
+# diffie-hellman-group14-sha1	[RFC4253]	Section 8.2
+enum KEX_DH_message_id {
+	SSH_MSG_KEXDH_INIT  = 30,
+	SSH_MSG_KEXDH_REPLY = 31,
+};
+
+# diffie-hellman-group-exchange-sha1	[RFC4419]	Section 4.1
+# diffie-hellman-group-exchange-sha256	[RFC4419]	Section 4.2
+enum KEX_DH_GEX_message_id {
+	SSH_MSG_KEX_DH_GEX_REQUEST_OLD = 30,
+	SSH_MSG_KEX_DH_GEX_GROUP       = 31,
+	SSH_MSG_KEX_DH_GEX_INIT        = 32,
+	SSH_MSG_KEX_DH_GEX_REPLY       = 33,
+	SSH_MSG_KEX_DH_GEX_REQUEST     = 34,
+};
+
+# rsa1024-sha1		[RFC4432]
+# rsa2048-sha256	[RFC4432]
+enum KEX_RSA_message_id {
+	SSH_MSG_KEXRSA_PUBKEY = 30,
+	SSH_MSG_KEXRSA_SECRET = 31,
+	SSH_MSG_KEXRSA_DONE   = 32,
+};
+
+# gss-group1-sha1-*	[RFC4462]	Section 2.3
+# gss-group14-sha1-*	[RFC4462]	Section 2.4
+# gss-gex-sha1-*	[RFC4462]	Section 2.5
+# gss-*			[RFC4462]	Section 2.6
+enum KEX_GSS_message_id {
+	SSH_MSG_KEXGSS_INIT     = 30,
+	SSH_MSG_KEXGSS_CONTINUE = 31,
+	SSH_MSG_KEXGSS_COMPLETE = 32,
+	SSH_MSG_KEXGSS_HOSTKEY  = 33,
+	SSH_MSG_KEXGSS_ERROR    = 34,
+	SSH_MSG_KEXGSS_GROUPREQ = 40,
+	SSH_MSG_KEXGSS_GROUP    = 41,
+};
+
+# ecdh-sha2-*	[RFC5656]
+enum KEX_ECDH_message_id {
+	SSH_MSG_KEX_ECDH_INIT  = 30,
+	SSH_MSG_KEX_ECDH_REPLY = 31,
+};
+
+# ecmqv-sha2	[RFC5656]
+enum KEX_ECMQV_message_id {
+	SSH_MSG_ECMQV_INIT  = 30,
+	SSH_MSG_ECMQV_REPLY = 31,
+};
+
+enum ssh1_message_id {
+	SSH_MSG_NONE                        = 0,
+	SSH_MSG_DISCONNECT                  = 1,
+	SSH_SMSG_PUBLIC_KEY                 = 2,
+	SSH_CMSG_SESSION_KEY                = 3,
+	SSH_CMSG_USER                       = 4,
+	SSH_CMSG_AUTH_RHOSTS                = 5,
+	SSH_CMSG_AUTH_RSA                   = 6,
+	SSH_SMSG_AUTH_RSA_CHALLENGE         = 7,
+	SSH_CMSG_AUTH_RSA_RESPONSE          = 8,
+	SSH_CMSG_AUTH_PASSWORD              = 9,
+	SSH_CMSG_REQUEST_PTY                = 10,
+	SSH_CMSG_WINDOW_SIZE                = 11,
+	SSH_CMSG_EXEC_SHELL                 = 12,
+	SSH_CMSG_EXEC_CMD                   = 13,
+	SSH_SMSG_SUCCESS                    = 14,
+	SSH_SMSG_FAILURE                    = 15,
+	SSH_CMSG_STDIN_DATA                 = 16,
+	SSH_SMSG_STDOUT_DATA                = 17,
+	SSH_SMSG_STDERR_DATA                = 18,
+	SSH_CMSG_EOF                        = 19,
+	SSH_SMSG_EXITSTATUS                 = 20,
+	SSH_MSG_CHANNEL_OPEN_CONFIRMATION   = 21,
+	SSH_MSG_CHANNEL_OPEN_FAILURE        = 22,
+	SSH_MSG_CHANNEL_DATA                = 23,
+	SSH_MSG_CHANNEL_CLOSE               = 24,
+	SSH_MSG_CHANNEL_CLOSE_CONFIRMATION  = 25,
+	SSH_CMSG_X11_REQUEST_FORWARDING_OLD = 26,
+	SSH_SMSG_X11_OPEN                   = 27,
+	SSH_CMSG_PORT_FORWARD_REQUEST       = 28,
+	SSH_MSG_PORT_OPEN                   = 29,
+	SSH_CMSG_AGENT_REQUEST_FORWARDING   = 30,
+	SSH_SMSG_AGENT_OPEN                 = 31,
+	SSH_MSG_IGNORE                      = 32,
+	SSH_CMSG_EXIT_CONFIRMATION          = 33,
+	SSH_CMSG_X11_REQUEST_FORWARDING     = 34,
+	SSH_CMSG_AUTH_RHOSTS_RSA            = 35,
+	SSH_MSG_DEBUG                       = 36,
+	SSH_CMSG_REQUEST_COMPRESSION        = 37,
+	SSH_CMSG_MAX_PACKET_SIZE            = 38,
+	SSH_CMSG_AUTH_TIS                   = 39,
+	SSH_SMSG_AUTH_TIS_CHALLENGE         = 40,
+	SSH_CMSG_AUTH_TIS_RESPONSE          = 41,
+	SSH_CMSG_AUTH_KERBEROS              = 42,
+	SSH_SMSG_AUTH_KERBEROS_RESPONSE     = 43,
+	SSH_CMSG_HAVE_KERBEROS_TGT          = 44,
+};
+
+enum ssh2_message_id {
+	MSG_DISCONNECT      = 1,
+	MSG_IGNORE          = 2,
+	MSG_UNIMPLEMENTED   = 3,
+	MSG_DEBUG           = 4,
+	MSG_SERVICE_REQUEST = 5,
+	MSG_SERVICE_ACCEPT  = 6,
+	MSG_KEXINIT         = 20,
+	MSG_NEWKEYS         = 21,
+};

src/analyzer/protocol/ssh/events.bif

@@ -1,38 +1,194 @@
-## Generated when seeing an SSH client's version identification. The SSH
-## protocol starts with a clear-text handshake message that reports client and
-## server protocol/software versions. This event provides access to what the
-## client sent.
+## An :abbr:`SSH (Secure Shell)` Protocol Version Exchange message
+## from the server. This contains an identification string that's used
+## for version identification. See :rfc:`4253#section-4.2` for
+## details.
 ##
+## c: The connection over which the message was sent.
 ##
-## See `Wikipedia <http://en.wikipedia.org/wiki/Secure_Shell>`__ for more
-## information about the SSH protocol.
+## version: The identification string
 ##
-## c: The connection.
-##
-## version: The version string the client sent (e.g., `SSH-2.0-libssh-0.11`).
-##
-## .. bro:see:: ssh_server_version
-##
-## .. note:: As everything after the initial version handshake proceeds
-##    encrypted, Bro cannot further analyze SSH sessions.
-event ssh_client_version%(c: connection, version: string%);
-
-## Generated when seeing an SSH server's version identification. The SSH
-## protocol starts with a clear-text handshake message that reports client and
-## server protocol/software versions. This event provides access to what the
-## server sent.
-##
-## See `Wikipedia <http://en.wikipedia.org/wiki/Secure_Shell>`__ for more
-## information about the SSH protocol.
-##
-## c: The connection.
-##
-## version: The version string the server sent (e.g.,
-##          ``SSH-1.99-OpenSSH_3.9p1``).
-##
-## .. bro:see:: ssh_client_version
-##
-## .. note:: As everything coming after the initial version handshake proceeds
-##    encrypted, Bro cannot further analyze SSH sessions.
+## .. bro:see:: ssh_client_version ssh_auth_successful ssh_auth_failed
+##    ssh_capabilities ssh2_server_host_key ssh1_server_host_key
+##    ssh_encrypted_packet ssh2_dh_server_params
+##    ssh2_gss_error ssh2_ecc_key
 event ssh_server_version%(c: connection, version: string%);
 
+## An :abbr:`SSH (Secure Shell)` Protocol Version Exchange message
+## from the client. This contains an identification string that's used
+## for version identification. See :rfc:`4253#section-4.2` for
+## details.
+##
+## c: The connection over which the message was sent.
+##
+## version: The identification string
+##
+## .. bro:see:: ssh_server_version ssh_auth_successful ssh_auth_failed
+##    ssh_capabilities ssh2_server_host_key ssh1_server_host_key
+##    ssh_encrypted_packet ssh2_dh_server_params
+##    ssh2_gss_error ssh2_ecc_key
+event ssh_client_version%(c: connection, version: string%);
+
+## This event is generated when an :abbr:`SSH (Secure Shell)`
+## connection was determined to have had a successful
+## authentication. This determination is based on packet size
+## analysis, and errs on the side of caution - that is, if there's any
+## doubt about the authentication success, this event is *not* raised.
+##
+## c: The connection over which the :abbr:`SSH (Secure Shell)`
+##    connection took place.
+##
+## auth_method_none: This is true if the analyzer detected a
+##    successful connection before any authentication challenge. The
+##    :abbr:`SSH (Secure Shell)` protocol provides a mechanism for
+##    unauthenticated access, which some servers support.
+##
+## .. bro:see:: ssh_server_version ssh_client_version ssh_auth_failed
+##    ssh_capabilities ssh2_server_host_key ssh1_server_host_key
+##    ssh_encrypted_packet ssh2_dh_server_params
+##    ssh2_gss_error ssh2_ecc_key
+event ssh_auth_successful%(c: connection, auth_method_none: bool%);
+
+## This event is generated when an :abbr:`SSH (Secure Shell)`
+## connection was determined to have had a failed authentication. This
+## determination is based on packet size analysis, and errs on the
+## side of caution - that is, if there's any doubt about the
+## authentication failure, this event is *not* raised.
+##
+## c: The connection over which the :abbr:`SSH (Secure Shell)`
+##    connection took place.
+##
+## .. bro:see:: ssh_server_version ssh_client_version
+##    ssh_auth_successful ssh_capabilities ssh2_server_host_key
+##    ssh1_server_host_key ssh_encrypted_packet ssh2_dh_server_params
+##    ssh2_gss_error ssh2_ecc_key
+event ssh_auth_failed%(c: connection%);
+
+## During the initial :abbr:`SSH (Secure Shell)` key exchange, each
+## endpoint lists the algorithms that it supports, in order of
+## preference. This event is generated for each endpoint, when the
+## SSH_MSG_KEXINIT message is seen. See :rfc:`4253#section-7.1` for
+## details.
+##
+## c: The connection over which the :abbr:`SSH (Secure Shell)`
+##    connection took place.
+##
+## cookie: The SSH_MSG_KEXINIT cookie - a random value generated by
+##    the sender.
+##
+## capabilities: The list of algorithms and languages that the sender
+##    advertises support for, in order of preference.
+##
+## .. bro:see:: ssh_server_version ssh_client_version
+##    ssh_auth_successful ssh_auth_failed ssh2_server_host_key
+##    ssh1_server_host_key ssh_encrypted_packet ssh2_dh_server_params
+##    ssh2_gss_error ssh2_ecc_key
+event ssh_capabilities%(c: connection, cookie: string, capabilities: SSH::Capabilities%);
+
+## During the :abbr:`SSH (Secure Shell)` key exchange, the server
+## supplies its public host key. This event is generated when the
+## appropriate key exchange message is seen for SSH2.
+##
+## c: The connection over which the :abbr:`SSH (Secure Shell)`
+##    connection took place.
+##
+## key: The server's public host key. Note that this is the public key
+##    itself, and not just the fingerprint or hash.
+##
+## .. bro:see:: ssh_server_version ssh_client_version
+##    ssh_auth_successful ssh_auth_failed ssh_capabilities
+##    ssh1_server_host_key ssh_encrypted_packet ssh2_dh_server_params
+##    ssh2_gss_error ssh2_ecc_key
+event ssh2_server_host_key%(c: connection, key: string%);
+
+## During the :abbr:`SSH (Secure Shell)` key exchange, the server
+## supplies its public host key. This event is generated when the
+## appropriate key exchange message is seen for SSH1.
+##
+## c: The connection over which the :abbr:`SSH (Secure Shell)`
+##    connection took place.
+##
+## p: The prime for the server's public host key.
+##
+## e: The exponent for the serer's public host key.
+##
+## .. bro:see:: ssh_server_version ssh_client_version
+##    ssh_auth_successful ssh_auth_failed ssh_capabilities
+##    ssh2_server_host_key ssh_encrypted_packet ssh2_dh_server_params
+##    ssh2_gss_error ssh2_ecc_key
+event ssh1_server_host_key%(c: connection, p: string, e: string%);
+
+## This event is generated when an :abbr:`SSH (Secure Shell)`
+## encrypted packet is seen. This event is not handled by default, but
+## is provided for heuristic analysis scripts. Note that you have to set
+## :bro:id:`SSH::skip_processing_after_detection` to false to use this
+## event. This carries a performance penalty.
+##
+## c: The connection over which the :abbr:`SSH (Secure Shell)`
+##    connection took place.
+##
+## orig: Whether the packet was sent by the originator of the TCP
+##    connection.
+##
+## len: The length of the :abbr:`SSH (Secure Shell)` payload, in
+##    bytes. Note that this ignores reassembly, as this is unknown.
+##
+## .. bro:see:: ssh_server_version ssh_client_version
+##    ssh_auth_successful ssh_auth_failed ssh_capabilities
+##    ssh2_server_host_key ssh1_server_host_key ssh2_dh_server_params
+##    ssh2_gss_error ssh2_ecc_key
+event ssh_encrypted_packet%(c: connection, orig: bool, len: count%);
+
+## Generated if the connection uses a Diffie-Hellman Group Exchange
+## key exchange method. This event contains the server DH parameters,
+## which are sent in the SSH_MSG_KEY_DH_GEX_GROUP message as defined in
+## :rfc:`4419#section-3`.
+##
+## c: The connection.
+##
+## p: The DH prime modulus.
+##
+## q: The DH generator.
+##
+## .. bro:see:: ssl_dh_server_params ssh_server_version
+##    ssh_client_version ssh_auth_successful ssh_auth_failed
+##    ssh_capabilities ssh2_server_host_key ssh1_server_host_key
+##    ssh_encrypted_packet ssh2_gss_error ssh2_ecc_key
+event ssh2_dh_server_params%(c: connection, p: string, q: string%);
+
+## In the event of a GSS-API error on the server, the server MAY send
+## send an error message with some additional details. This event is
+## generated when such an error message is seen. For more information,
+## see :rfc:`4462#section-2.1`.
+##
+## c: The connection.
+##
+## major_status: GSS-API major status code.
+##
+## minor_status: GSS-API minor status code.
+##
+## err_msg: Detailed human-readable error message
+##
+## .. bro:see:: ssh_server_version ssh_client_version
+##    ssh_auth_successful ssh_auth_failed ssh_capabilities
+##    ssh2_server_host_key ssh1_server_host_key ssh_encrypted_packet
+##    ssh2_dh_server_params ssh2_ecc_key
+event ssh2_gss_error%(c: connection, major_status: count, minor_status: count, err_msg: string%);
+
+## The :abbr:`ECDH (Elliptic Curve Diffie-Hellman)` and
+## :abbr:`ECMQV (Elliptic Curve Menezes-Qu-Vanstone)` key exchange
+## algorithms use two ephemeral key pairs to generate a shared
+## secret. This event is generated when either the client's or
+## server's ephemeral public key is seen. For more information, see:
+## :rfc:`5656#section-4`.
+##
+## c: The connection
+##
+## is_orig: Did this message come from the originator?
+##
+## q: The ephemeral public key
+##
+## .. bro:see:: ssh_server_version ssh_client_version
+##    ssh_auth_successful ssh_auth_failed ssh_capabilities
+##    ssh2_server_host_key ssh1_server_host_key ssh_encrypted_packet
+##    ssh2_dh_server_params ssh2_gss_error
+event ssh2_ecc_key%(c: connection, is_orig: bool, q: string%);

src/analyzer/protocol/ssh/ssh-analyzer.pac

@@ -0,0 +1,221 @@
+%extern{
+#include <cstdlib>
+#include <vector>
+#include <string>
+%}
+
+%header{
+VectorVal* name_list_to_vector(const bytestring nl);
+%}
+
+%code{
+// Copied from IRC_Analyzer::SplitWords
+VectorVal* name_list_to_vector(const bytestring nl)
+	{
+	VectorVal* vv = new VectorVal(internal_type("string_vec")->AsVectorType());
+
+	string name_list = std_str(nl);
+	if ( name_list.size() < 1 )
+		return vv;
+
+	unsigned int start = 0;
+	unsigned int split_pos = 0;
+
+	while ( name_list[start] == ',' )
+		{
+		++start;
+		++split_pos;
+		}
+
+	string word;
+	while ( (split_pos = name_list.find(',', start)) < name_list.size() )
+		{
+		word = name_list.substr(start, split_pos - start);
+		if ( word.size() > 0 && word[0] != ',' )
+			vv->Assign(vv->Size(), new StringVal(word));
+
+		start = split_pos + 1;
+		}
+
+	// Add line end if needed.
+	if ( start < name_list.size() )
+		{
+		word = name_list.substr(start, name_list.size() - start);
+		vv->Assign(vv->Size(), new StringVal(word));
+		}
+	return vv;
+	}
+%}
+
+refine flow SSH_Flow += {
+	function proc_ssh_version(msg: SSH_Version): bool
+		%{
+		if ( ssh_client_version && ${msg.is_orig } )
+			{
+			BifEvent::generate_ssh_client_version(connection()->bro_analyzer(), 
+				connection()->bro_analyzer()->Conn(), 
+				bytestring_to_val(${msg.version}));
+			}
+		else if ( ssh_server_version )
+			{
+			BifEvent::generate_ssh_server_version(connection()->bro_analyzer(), 
+				connection()->bro_analyzer()->Conn(), 
+				bytestring_to_val(${msg.version}));
+			}
+		return true;
+		%}
+
+	function proc_ssh2_kexinit(msg: SSH2_KEXINIT): bool
+		%{
+		if ( ! ssh_capabilities )
+			return false;
+
+		RecordVal* result = new RecordVal(BifType::Record::SSH::Capabilities);
+		result->Assign(0, name_list_to_vector(${msg.kex_algorithms.val}));
+		result->Assign(1, name_list_to_vector(${msg.server_host_key_algorithms.val}));
+
+		RecordVal* encryption_algs = new RecordVal(BifType::Record::SSH::Algorithm_Prefs);
+		encryption_algs->Assign(0, name_list_to_vector(${msg.encryption_algorithms_client_to_server.val}));
+		encryption_algs->Assign(1, name_list_to_vector(${msg.encryption_algorithms_server_to_client.val}));
+		result->Assign(2, encryption_algs);
+
+		RecordVal* mac_algs = new RecordVal(BifType::Record::SSH::Algorithm_Prefs);
+		mac_algs->Assign(0, name_list_to_vector(${msg.mac_algorithms_client_to_server.val}));
+		mac_algs->Assign(1, name_list_to_vector(${msg.mac_algorithms_server_to_client.val}));
+		result->Assign(3, mac_algs);
+
+		RecordVal* compression_algs = new RecordVal(BifType::Record::SSH::Algorithm_Prefs);
+		compression_algs->Assign(0, name_list_to_vector(${msg.compression_algorithms_client_to_server.val}));
+		compression_algs->Assign(1, name_list_to_vector(${msg.compression_algorithms_server_to_client.val}));
+		result->Assign(4, compression_algs);
+
+		if ( ${msg.languages_client_to_server.len} || ${msg.languages_server_to_client.len} )
+			{
+			RecordVal* languages = new RecordVal(BifType::Record::SSH::Algorithm_Prefs);
+			if ( ${msg.languages_client_to_server.len} )
+				languages->Assign(0, name_list_to_vector(${msg.languages_client_to_server.val}));
+			if ( ${msg.languages_server_to_client.len} )
+				languages->Assign(1, name_list_to_vector(${msg.languages_server_to_client.val}));
+
+			result->Assign(5, languages);
+			}
+
+
+		result->Assign(6, new Val(${msg.is_orig}, TYPE_BOOL));
+
+		BifEvent::generate_ssh_capabilities(connection()->bro_analyzer(),
+			connection()->bro_analyzer()->Conn(), bytestring_to_val(${msg.cookie}),
+			result);
+
+		return true;
+		%}
+
+
+	function proc_ssh2_dh_gex_group(msg: SSH2_DH_GEX_GROUP): bool
+		%{
+		if ( ssh2_dh_server_params )
+			{
+			BifEvent::generate_ssh2_dh_server_params(connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				bytestring_to_val(${msg.p.val}), bytestring_to_val(${msg.g.val}));
+			}
+		return true;
+		%}
+
+	function proc_ssh2_ecc_key(q: bytestring, is_orig: bool): bool
+		%{
+		if ( ssh2_ecc_key )
+			{
+			BifEvent::generate_ssh2_ecc_key(connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig, bytestring_to_val(q));
+			}
+		return true;
+		%}
+
+	function proc_ssh2_gss_error(msg: SSH2_GSS_ERROR): bool
+		%{
+		if ( ssh2_gss_error )
+			{
+			BifEvent::generate_ssh2_gss_error(connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				${msg.major_status}, ${msg.minor_status},
+				bytestring_to_val(${msg.message.val}));
+			}
+		return true;
+		%}
+
+	function proc_ssh2_server_host_key(key: bytestring): bool
+		%{
+		if ( ssh2_server_host_key )
+			{
+			BifEvent::generate_ssh2_server_host_key(connection()->bro_analyzer(), 
+				connection()->bro_analyzer()->Conn(), 
+				bytestring_to_val(${key}));
+			}
+		return true;
+		%}
+
+	function proc_ssh1_server_host_key(p: bytestring, e: bytestring): bool
+		%{
+		if ( ssh1_server_host_key )
+			{
+			BifEvent::generate_ssh1_server_host_key(connection()->bro_analyzer(), 
+				connection()->bro_analyzer()->Conn(), 
+				bytestring_to_val(${p}),
+				bytestring_to_val(${e}));
+			}
+		return true;
+		%}
+
+	function proc_newkeys(): bool
+		%{
+		connection()->bro_analyzer()->ProtocolConfirmation();
+		return true;
+		%}
+};
+
+refine typeattr SSH_Version += &let {
+	proc: bool = $context.flow.proc_ssh_version(this);
+};
+
+refine typeattr SSH2_KEXINIT += &let {
+	proc: bool = $context.flow.proc_ssh2_kexinit(this);
+};
+
+refine typeattr SSH1_Message += &let {
+	proc_newkeys: bool = $context.flow.proc_newkeys() &if(msg_type == SSH_CMSG_SESSION_KEY);
+};
+
+refine typeattr SSH2_Message += &let {
+	proc_newkeys: bool = $context.flow.proc_newkeys() &if(msg_type == MSG_NEWKEYS);
+};
+
+refine typeattr SSH2_DH_GEX_REPLY += &let {
+	proc: bool = $context.flow.proc_ssh2_server_host_key(k_s.val);
+};
+
+refine typeattr SSH2_GSS_HOSTKEY += &let {
+	proc: bool = $context.flow.proc_ssh2_server_host_key(k_s.val);
+};
+
+refine typeattr SSH2_GSS_ERROR += &let {
+	proc: bool = $context.flow.proc_ssh2_gss_error(this);
+};
+
+refine typeattr SSH2_DH_GEX_GROUP += &let {
+	proc: bool = $context.flow.proc_ssh2_dh_gex_group(this);
+};
+
+refine typeattr SSH2_ECC_REPLY += &let {
+	proc_k: bool = $context.flow.proc_ssh2_server_host_key(k_s.val);
+	proc_q: bool = $context.flow.proc_ssh2_ecc_key(q_s.val, false);
+};
+
+refine typeattr SSH2_ECC_INIT += &let {
+	proc: bool = $context.flow.proc_ssh2_ecc_key(q_c.val, true);
+};
+
+refine typeattr SSH1_PUBLIC_KEY += &let {
+	proc:  bool = $context.flow.proc_ssh1_server_host_key(host_key_p.val, host_key_e.val);
+};

src/analyzer/protocol/ssh/ssh-protocol.pac

@@ -0,0 +1,430 @@
+%include consts.pac
+
+# Common constructs across SSH1 and SSH2
+########################################
+
+# We have 3 basic types of messages:
+#
+#  - SSH_Version messages just have a string with the banner string of the client or server
+#  - Encrypted messages have no usable data, so we'll just ignore them as best we can.
+#  - Finally, key exchange messages have a common format.
+
+type SSH_PDU(is_orig: bool) = case $context.connection.get_state(is_orig) of {
+	VERSION_EXCHANGE -> version   : SSH_Version(is_orig);
+	ENCRYPTED        -> encrypted : bytestring &length=1 &transient;
+	default          -> kex       : SSH_Key_Exchange(is_orig);
+} &byteorder=bigendian;
+
+type SSH_Version(is_orig: bool) = record {
+	version : bytestring &oneline;
+} &let {
+	update_state   : bool = $context.connection.update_state(KEX_INIT, is_orig);
+	update_version : bool = $context.connection.update_version(version, is_orig);
+};
+
+type SSH_Key_Exchange(is_orig: bool) = case $context.connection.get_version() of {
+	SSH1 -> ssh1_msg : SSH1_Key_Exchange(is_orig);
+	SSH2 -> ssh2_msg : SSH2_Key_Exchange(is_orig);
+};
+
+# SSH1 constructs
+#################
+
+type SSH1_Key_Exchange(is_orig: bool) = record {
+	packet_length : uint32;
+	pad_fill      : bytestring &length = 8 - (packet_length % 8);
+	msg_type      : uint8;
+	message       : SSH1_Message(is_orig, msg_type, packet_length - 5);
+	crc           : uint32;
+} &length = packet_length + 4 + 8 - (packet_length % 8);
+
+type SSH1_Message(is_orig: bool, msg_type: uint8, length: uint32) = case msg_type of {
+	SSH_SMSG_PUBLIC_KEY  -> public_key  : SSH1_PUBLIC_KEY(length);
+	SSH_CMSG_SESSION_KEY -> session_key : SSH1_SESSION_KEY(length);
+} &let {
+	detach : bool=$context.connection.update_state(ENCRYPTED, is_orig);
+};
+
+type SSH1_PUBLIC_KEY(length: uint32) = record {
+	cookie            : bytestring &length=8;
+	server_key        : uint32;
+	server_key_p      : ssh1_mp_int;
+	server_key_e      : ssh1_mp_int;
+	host_key          : uint32;
+	host_key_p        : ssh1_mp_int;
+	host_key_e        : ssh1_mp_int;
+	flags             : uint32;
+	supported_ciphers : uint32;
+	supported_auths   : uint32;
+} &length=length;
+
+type SSH1_SESSION_KEY(length: uint32) = record {
+	cipher      : uint8;
+	cookie      : bytestring &length=8;
+	session_key : ssh1_mp_int;
+	flags       : uint32;
+} &length=length;
+
+type ssh1_mp_int = record {
+	len : uint16;
+	val : bytestring &length=(len+7)/8;
+};
+
+
+## SSH2
+
+type SSH2_Header(is_orig: bool) = record {
+	packet_length  : uint32;
+	padding_length : uint8;
+	msg_type       : uint8;
+} &let {
+	payload_length : uint32 = packet_length - padding_length - 2;
+	detach         : bool = $context.connection.update_state(ENCRYPTED, is_orig) &if(msg_type == MSG_NEWKEYS);
+};
+
+type SSH2_Key_Exchange(is_orig: bool) = record {
+	header   : SSH2_Header(is_orig);
+	payload  : SSH2_Message(is_orig, header.msg_type, header.payload_length);
+	pad      : bytestring &length=header.padding_length;
+} &length=header.packet_length + 4;
+
+type SSH2_Message(is_orig: bool, msg_type: uint8, length: uint32) = case $context.connection.get_state(is_orig) of {
+	KEX_INIT   -> kex        : SSH2_KEXINIT(length, is_orig);
+	KEX_DH_GEX -> kex_dh_gex : SSH2_Key_Exchange_DH_GEX_Message(is_orig, msg_type, length);
+	KEX_DH     -> kex_dh     : SSH2_Key_Exchange_DH_Message(is_orig, msg_type, length);
+	KEX_ECC    -> kex_ecc    : SSH2_Key_Exchange_ECC_Message(is_orig, msg_type, length);
+	KEX_GSS    -> kex_gss    : SSH2_Key_Exchange_GSS_Message(is_orig, msg_type, length);
+	KEX_RSA    -> kex_rsa    : SSH2_Key_Exchange_RSA_Message(is_orig, msg_type, length);
+	default    -> unknown    : bytestring &length=length;
+};
+
+type SSH2_KEXINIT(length: uint32, is_orig: bool) = record {
+	cookie                                  : bytestring &length=16;
+	kex_algorithms                          : ssh_string;
+	server_host_key_algorithms              : ssh_string;
+	encryption_algorithms_client_to_server  : ssh_string;
+	encryption_algorithms_server_to_client  : ssh_string;
+	mac_algorithms_client_to_server         : ssh_string;
+	mac_algorithms_server_to_client         : ssh_string;
+	compression_algorithms_client_to_server : ssh_string;
+	compression_algorithms_server_to_client : ssh_string;
+	languages_client_to_server              : ssh_string;
+	languages_server_to_client              : ssh_string;
+	first_kex_packet_follows                : uint8;
+	reserved                                : uint32;
+} &let {
+	proc_kex : bool= $context.connection.update_kex(kex_algorithms.val, is_orig);
+} &length=length;
+
+# KEX_DH exchanges
+
+type SSH2_Key_Exchange_DH_Message(is_orig: bool, msg_type: uint8, length: uint32) = case msg_type of {
+	SSH_MSG_KEXDH_INIT  -> init   : SSH2_DH_GEX_INIT(length);
+	SSH_MSG_KEXDH_REPLY -> reply  : SSH2_DH_GEX_REPLY(length);
+	default             -> unknown: bytestring &length=length &transient;
+};
+
+# KEX_DH_GEX exchanges
+
+type SSH2_Key_Exchange_DH_GEX_Message(is_orig: bool, msg_type: uint8, length: uint32) = case msg_type of {
+	SSH_MSG_KEX_DH_GEX_REQUEST_OLD -> request_old : SSH2_DH_GEX_REQUEST_OLD;
+	SSH_MSG_KEX_DH_GEX_REQUEST     -> request     : SSH2_DH_GEX_REQUEST;
+	SSH_MSG_KEX_DH_GEX_GROUP       -> group       : SSH2_DH_GEX_GROUP(length);
+	SSH_MSG_KEX_DH_GEX_INIT        -> init        : SSH2_DH_GEX_INIT(length);
+	SSH_MSG_KEX_DH_GEX_REPLY       -> reply       : SSH2_DH_GEX_REPLY(length);
+	default                        -> unknown     : bytestring &length=length &transient;
+};
+
+type SSH2_DH_GEX_REQUEST = record {
+	min : uint32;
+	n   : uint32;
+	max : uint32;
+} &length=12;
+
+type SSH2_DH_GEX_REQUEST_OLD = record {
+	n : uint32;
+} &length=4;
+
+type SSH2_DH_GEX_GROUP(length: uint32) = record {
+	p : ssh_string;
+	g : ssh_string;
+} &length=length;
+
+type SSH2_DH_GEX_INIT(length: uint32) = record {
+	e : ssh_string;
+} &length=length;
+
+type SSH2_DH_GEX_REPLY(length: uint32) = record {
+	k_s       : ssh_string;
+	f         : ssh_string;
+	signature : ssh_string;
+} &length=length;
+
+# KEX_RSA exchanges
+
+type SSH2_Key_Exchange_RSA_Message(is_orig: bool, msg_type: uint8, length: uint32) = case msg_type of {
+	SSH_MSG_KEXRSA_PUBKEY -> pubkey : SSH2_RSA_PUBKEY(length);
+	SSH_MSG_KEXRSA_SECRET -> secret : SSH2_RSA_SECRET(length);
+	SSH_MSG_KEXRSA_DONE   -> done   : SSH2_RSA_DONE(length);
+};
+
+type SSH2_RSA_PUBKEY(length: uint32) = record {
+	k_s : ssh_string;
+	k_t : ssh_string;
+} &length=length;
+
+type SSH2_RSA_SECRET(length: uint32) = record {
+	encrypted_payload : ssh_string;
+} &length=length;
+
+type SSH2_RSA_DONE(length: uint32) = record {
+	signature : ssh_string;
+} &length=length;
+
+# KEX_GSS exchanges
+
+type SSH2_Key_Exchange_GSS_Message(is_orig: bool, msg_type: uint8, length: uint32) = case msg_type of {
+	SSH_MSG_KEXGSS_INIT     -> init     : SSH2_GSS_INIT(length);
+	SSH_MSG_KEXGSS_CONTINUE -> cont     : SSH2_GSS_CONTINUE(length);
+	SSH_MSG_KEXGSS_COMPLETE -> complete : SSH2_GSS_COMPLETE(length);
+	SSH_MSG_KEXGSS_HOSTKEY  -> hostkey  : SSH2_GSS_HOSTKEY(length);
+	SSH_MSG_KEXGSS_ERROR    -> error    : SSH2_GSS_ERROR(length);
+	SSH_MSG_KEXGSS_GROUPREQ -> groupreq : SSH2_DH_GEX_REQUEST;
+	SSH_MSG_KEXGSS_GROUP    -> group    : SSH2_DH_GEX_GROUP(length);
+};
+
+type SSH2_GSS_INIT(length: uint32) = record {
+	output_token : ssh_string;
+	e            : ssh_string;
+} &length=length;
+
+type SSH2_GSS_CONTINUE(length: uint32) = record {
+	output_token : ssh_string;
+} &length=length;
+
+type SSH2_GSS_COMPLETE(length: uint32) = record {
+	f                           : ssh_string;
+	per_msg_token               : ssh_string;
+	have_token                  : uint8;
+	parse_token                 : case have_token of {
+		0       -> no_token : empty;
+		default -> token    : ssh_string;
+	};
+} &length=length;
+
+type SSH2_GSS_HOSTKEY(length: uint32) = record {
+	k_s : ssh_string;
+} &length=length;
+
+type SSH2_GSS_ERROR(length: uint32) = record {
+	major_status : uint32;
+	minor_status : uint32;
+	message      : ssh_string;
+	language     : ssh_string;
+} &length=length;
+
+# KEX_ECDH and KEX_ECMQV exchanges
+
+type SSH2_Key_Exchange_ECC_Message(is_orig: bool, msg_type: uint8, length: uint32) = case msg_type of {
+	SSH_MSG_KEX_ECDH_INIT  -> init  : SSH2_ECC_INIT(length);
+	SSH_MSG_KEX_ECDH_REPLY -> reply : SSH2_ECC_REPLY(length);
+};
+
+# This deviates from the RFC. SSH_MSG_KEX_ECDH_INIT and
+# SSH_MSG_KEX_ECMQV_INIT can be parsed the same way.
+type SSH2_ECC_INIT(length: uint32) = record {
+	q_c : ssh_string;
+};
+
+# This deviates from the RFC. SSH_MSG_KEX_ECDH_REPLY and
+# SSH_MSG_KEX_ECMQV_REPLY can be parsed the same way.
+type SSH2_ECC_REPLY(length: uint32) = record {
+	k_s       : ssh_string;
+	q_s       : ssh_string;
+	signature : ssh_string;
+};
+
+# Helper types
+
+type ssh_string = record {
+	len : uint32;
+	val : bytestring &length=len;
+};
+
+type ssh_host_key = record {
+	len      : uint32;
+	key_type : ssh_string;
+	key      : ssh_string;
+} &length=(len + 4);
+
+## Done with types
+
+refine connection SSH_Conn += {
+	%member{
+		int state_up_;
+		int state_down_;
+		int version_;
+
+		bool kex_orig_;
+		bool kex_seen_;
+		bytestring kex_algs_cache_;
+		bytestring kex_algorithm_;
+	%}
+
+	%init{
+		state_up_   = VERSION_EXCHANGE;
+		state_down_ = VERSION_EXCHANGE;
+		version_    = UNK;
+
+		kex_seen_ = false;
+		kex_orig_ = false;
+		kex_algs_cache_ = bytestring();
+		kex_algorithm_ = bytestring();
+	%}
+
+	%cleanup{
+		kex_algorithm_.free();
+		kex_algs_cache_.free();
+	%}
+
+	function get_state(is_orig: bool) : int
+		%{
+		if ( is_orig )
+			{
+			return state_up_;
+			}
+		else
+			{
+			return state_down_;
+			}
+		%}
+
+	function update_state(s: state, is_orig: bool) : bool
+		%{
+		if ( is_orig )
+			state_up_ = s;
+		else
+			state_down_ = s;
+		return true;
+		%}
+
+	function get_version() : int
+		%{
+		return version_;
+		%}
+
+	function update_version(v: bytestring, is_orig: bool) : bool
+		%{
+		if ( is_orig && ( v.length() >= 4 ) )
+			{
+			if ( v[4] == '2' )
+				version_ = SSH2;
+			if ( v[4] == '1' )
+				version_ = SSH1;
+			}
+		return true;
+		%}
+
+	function update_kex_state_if_equal(s: string, to_state: state) : bool
+		%{
+		if ( std_str(kex_algorithm_).compare(s) == 0 )
+			{
+			update_state(to_state, true);
+			update_state(to_state, false);
+			return true;
+			}
+		return false;
+		%}
+
+	function update_kex_state_if_startswith(s: string, to_state: state) : bool
+		%{
+		if ( (uint) kex_algorithm_.length() < s.length() )
+			return false;
+
+		if ( std_str(kex_algorithm_).substr(0, s.length()).compare(s) == 0 )
+			{
+			update_state(to_state, true);
+			update_state(to_state, false);
+			return true;
+			}
+		return false;
+		%}
+
+	function update_kex(algs: bytestring, orig: bool) : bool
+		%{
+		if ( !kex_seen_ )
+			{
+			kex_seen_ = true;
+			kex_orig_ = orig;
+			kex_algs_cache_.init(${algs}.data(), ${algs}.length());
+
+			return false;
+			}
+		else if ( kex_orig_ == orig )
+			return false;
+
+		VectorVal* client_list = name_list_to_vector(orig ? algs             : kex_algs_cache_);
+		VectorVal* server_list = name_list_to_vector(orig ? kex_algs_cache_  : algs);
+
+		for ( unsigned int i = 0; i < client_list->Size(); ++i )
+			{
+			for ( unsigned int j = 0; j < server_list->Size(); ++j )
+				{
+				if ( *(client_list->Lookup(i)->AsStringVal()->AsString()) == *(server_list->Lookup(j)->AsStringVal()->AsString()) )
+					{
+					kex_algorithm_.init((const uint8 *) client_list->Lookup(i)->AsStringVal()->Bytes(),
+						client_list->Lookup(i)->AsStringVal()->Len());
+
+					Unref(client_list);
+					Unref(server_list);
+
+					// UNTESTED
+					if ( update_kex_state_if_equal("rsa1024-sha1", KEX_RSA) )
+						return true;
+					// UNTESTED
+					if ( update_kex_state_if_equal("rsa2048-sha256", KEX_RSA) )
+						return true;
+
+					// UNTESTED
+					if ( update_kex_state_if_equal("diffie-hellman-group1-sha1", KEX_DH) )
+						return true;
+					// UNTESTED
+					if ( update_kex_state_if_equal("diffie-hellman-group14-sha1", KEX_DH) )
+						return true;
+
+					if ( update_kex_state_if_equal("diffie-hellman-group-exchange-sha1", KEX_DH_GEX) )
+						return true;
+					if ( update_kex_state_if_equal("diffie-hellman-group-exchange-sha256", KEX_DH_GEX) )
+						return true;
+
+					if ( update_kex_state_if_startswith("gss-group1-sha1-", KEX_GSS) )
+						return true;
+					if ( update_kex_state_if_startswith("gss-group14-sha1-", KEX_GSS) )
+						return true;
+					if ( update_kex_state_if_startswith("gss-gex-sha1-", KEX_GSS) )
+						return true;
+					if ( update_kex_state_if_startswith("gss-", KEX_GSS) )
+						return true;
+
+					if ( update_kex_state_if_startswith("ecdh-sha2-", KEX_ECC) )
+						return true;
+					if ( update_kex_state_if_equal("ecmqv-sha2", KEX_ECC) )
+						return true;
+					if ( update_kex_state_if_equal("curve25519-sha256@libssh.org", KEX_ECC) )
+						return true;
+
+
+					bro_analyzer()->Weird(fmt("ssh_unknown_kex_algorithm=%s", c_str(kex_algorithm_)));
+					return true;
+
+					}
+				}
+			}
+
+		Unref(client_list);
+		Unref(server_list);
+
+		return true;
+		%}
+
+};

src/analyzer/protocol/ssh/ssh.pac

@@ -0,0 +1,33 @@
+# Generated by binpac_quickstart
+
+# Analyzer for Secure Shell
+#  - ssh-protocol.pac: describes the SSH protocol messages
+#  - ssh-analyzer.pac: describes the SSH analyzer code
+
+%include binpac.pac
+%include bro.pac
+
+%extern{
+	#include "types.bif.h"
+	#include "events.bif.h"
+%}
+
+analyzer SSH withcontext {
+	connection: SSH_Conn;
+	flow:       SSH_Flow;
+};
+
+# Our connection consists of two flows, one in each direction.
+connection SSH_Conn(bro_analyzer: BroAnalyzer) {
+	upflow   = SSH_Flow(true);
+	downflow = SSH_Flow(false);
+};
+
+%include ssh-protocol.pac
+
+# Now we define the flow:
+flow SSH_Flow(is_orig: bool) {
+	flowunit = SSH_PDU(is_orig) withcontext(connection, this);
+};
+
+%include ssh-analyzer.pac

src/analyzer/protocol/ssh/types.bif

@@ -0,0 +1,6 @@
+module SSH;
+
+type Algorithm_Prefs: record;
+type Capabilities: record;
+
+module GLOBAL;

src/analyzer/protocol/ssl/CMakeLists.txt

@@ -4,7 +4,9 @@ include(BroPlugin)
 include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
 
 bro_plugin_begin(Bro SSL)
-bro_plugin_cc(SSL.cc Plugin.cc)
+bro_plugin_cc(SSL.cc DTLS.cc Plugin.cc)
 bro_plugin_bif(events.bif)
-bro_plugin_pac(ssl.pac ssl-analyzer.pac ssl-protocol.pac ssl-defs.pac)
+bro_plugin_pac(tls-handshake.pac tls-handshake-protocol.pac tls-handshake-analyzer.pac ssl-defs.pac)
+bro_plugin_pac(ssl.pac ssl-dtls-analyzer.pac ssl-analyzer.pac ssl-dtls-protocol.pac ssl-protocol.pac ssl-defs.pac)
+bro_plugin_pac(dtls.pac ssl-dtls-analyzer.pac dtls-analyzer.pac ssl-dtls-protocol.pac dtls-protocol.pac ssl-defs.pac)
 bro_plugin_end()

src/analyzer/protocol/ssl/DTLS.cc

@@ -0,0 +1,65 @@
+
+#include "DTLS.h"
+#include "Reporter.h"
+#include "util.h"
+
+#include "events.bif.h"
+
+#include "dtls_pac.h"
+#include "tls-handshake_pac.h"
+
+using namespace analyzer::dtls;
+
+DTLS_Analyzer::DTLS_Analyzer(Connection* c)
+: analyzer::Analyzer("DTLS", c)
+	{
+	interp = new binpac::DTLS::SSL_Conn(this);
+	handshake_interp = new binpac::TLSHandshake::Handshake_Conn(this);
+	}
+
+DTLS_Analyzer::~DTLS_Analyzer()
+	{
+	delete interp;
+	delete handshake_interp;
+	}
+
+void DTLS_Analyzer::Done()
+	{
+	Analyzer::Done();
+	interp->FlowEOF(true);
+	interp->FlowEOF(false);
+	handshake_interp->FlowEOF(true);
+	handshake_interp->FlowEOF(false);
+	}
+
+void DTLS_Analyzer::DeliverPacket(int len, const u_char* data, bool orig, uint64 seq, const IP_Hdr* ip, int caplen)
+	{
+	Analyzer::DeliverPacket(len, data, orig, seq, ip, caplen);
+	interp->NewData(orig, data, data + len);
+	}
+
+void DTLS_Analyzer::EndOfData(bool is_orig)
+	{
+	Analyzer::EndOfData(is_orig);
+	interp->FlowEOF(is_orig);
+	handshake_interp->FlowEOF(is_orig);
+	}
+
+
+void DTLS_Analyzer::SendHandshake(uint8 msg_type, uint32 length, const u_char* begin, const u_char* end, bool orig)
+	{
+	try
+		{
+		handshake_interp->NewData(orig, (const unsigned char*) &msg_type, (const unsigned char*) &msg_type + 1);
+		uint32 host_length = htonl(length);
+		// the parser inspects a uint24 - since it is big-endian, it should be ok to just skip
+		// the first byte of the uint32. Since we get the data from an uint24 from the dtls-parser, this should
+		// always yield the correct result.
+		handshake_interp->NewData(orig, (const unsigned char*) &host_length + 1, (const unsigned char*) &host_length + sizeof(host_length));
+		handshake_interp->NewData(orig, begin, end);
+		}
+	catch ( const binpac::Exception& e )
+		{
+		ProtocolViolation(fmt("Binpac exception: %s", e.c_msg()));
+		}
+	}

src/analyzer/protocol/ssl/DTLS.h

@@ -0,0 +1,38 @@
+#ifndef ANALYZER_PROTOCOL_SSL_DTLS_H
+#define ANALYZER_PROTOCOL_SSL_DTLS_H
+
+#include "events.bif.h"
+
+#include "analyzer/protocol/udp/UDP.h"
+
+namespace binpac { namespace DTLS { class SSL_Conn; } }
+
+namespace binpac { namespace TLSHandshake { class Handshake_Conn; } }
+
+namespace analyzer { namespace dtls {
+
+class DTLS_Analyzer : public analyzer::Analyzer {
+public:
+	DTLS_Analyzer(Connection* conn);
+	virtual ~DTLS_Analyzer();
+
+	// Overriden from Analyzer.
+	virtual void Done();
+	virtual void DeliverPacket(int len, const u_char* data, bool orig,
+					uint64 seq, const IP_Hdr* ip, int caplen);
+	virtual void EndOfData(bool is_orig);
+
+	void SendHandshake(uint8 msg_type, uint32 length, const u_char* begin, const u_char* end, bool orig);
+
+
+	static analyzer::Analyzer* Instantiate(Connection* conn)
+		{ return new DTLS_Analyzer(conn); }
+
+protected:
+	binpac::DTLS::SSL_Conn* interp;
+	binpac::TLSHandshake::Handshake_Conn* handshake_interp;
+};
+
+} } // namespace analyzer::*
+
+#endif

src/analyzer/protocol/ssl/Plugin.cc

@@ -4,6 +4,7 @@
 #include "plugin/Plugin.h"
 
 #include "SSL.h"
+#include "DTLS.h"
 
 namespace plugin {
 namespace Bro_SSL {
@@ -13,10 +14,11 @@ public:
 	plugin::Configuration Configure()
 		{
 		AddComponent(new ::analyzer::Component("SSL", ::analyzer::ssl::SSL_Analyzer::Instantiate));
+		AddComponent(new ::analyzer::Component("DTLS", ::analyzer::dtls::DTLS_Analyzer::Instantiate));
 
 		plugin::Configuration config;
 		config.name = "Bro::SSL";
-		config.description = "SSL analyzer";
+		config.description = "SSL/TLS and DTLS analyzers";
 		return config;
 		}
 } plugin;

src/analyzer/protocol/ssl/SSL.cc

@@ -5,6 +5,8 @@
 #include "util.h"
 
 #include "events.bif.h"
+#include "ssl_pac.h"
+#include "tls-handshake_pac.h"
 
 using namespace analyzer::ssl;
 
@@ -12,12 +14,14 @@ SSL_Analyzer::SSL_Analyzer(Connection* c)
 : tcp::TCP_ApplicationAnalyzer("SSL", c)
 	{
 	interp = new binpac::SSL::SSL_Conn(this);
+	handshake_interp = new binpac::TLSHandshake::Handshake_Conn(this);
 	had_gap = false;
 	}
 
 SSL_Analyzer::~SSL_Analyzer()
 	{
 	delete interp;
+	delete handshake_interp;
 	}
 
 void SSL_Analyzer::Done()
@@ -26,12 +30,15 @@ void SSL_Analyzer::Done()
 
 	interp->FlowEOF(true);
 	interp->FlowEOF(false);
+	handshake_interp->FlowEOF(true);
+	handshake_interp->FlowEOF(false);
 	}
 
 void SSL_Analyzer::EndpointEOF(bool is_orig)
 	{
 	tcp::TCP_ApplicationAnalyzer::EndpointEOF(is_orig);
 	interp->FlowEOF(is_orig);
+	handshake_interp->FlowEOF(is_orig);
 	}
 
 void SSL_Analyzer::DeliverStream(int len, const u_char* data, bool orig)
@@ -57,6 +64,18 @@ void SSL_Analyzer::DeliverStream(int len, const u_char* data, bool orig)
 		}
 	}
 
+void SSL_Analyzer::SendHandshake(const u_char* begin, const u_char* end, bool orig)
+	{
+	try
+		{
+		handshake_interp->NewData(orig, begin, end);
+		}
+	catch ( const binpac::Exception& e )
+		{
+		ProtocolViolation(fmt("Binpac exception: %s", e.c_msg()));
+		}
+	}
+
 void SSL_Analyzer::Undelivered(uint64 seq, int len, bool orig)
 	{
 	tcp::TCP_ApplicationAnalyzer::Undelivered(seq, len, orig);

src/analyzer/protocol/ssl/SSL.h

@@ -4,7 +4,10 @@
 #include "events.bif.h"
 
 #include "analyzer/protocol/tcp/TCP.h"
-#include "ssl_pac.h"
+
+namespace binpac { namespace SSL { class SSL_Conn; } }
+
+namespace binpac { namespace TLSHandshake { class Handshake_Conn; } }
 
 namespace analyzer { namespace ssl {
 
@@ -18,6 +21,8 @@ public:
 	virtual void DeliverStream(int len, const u_char* data, bool orig);
 	virtual void Undelivered(uint64 seq, int len, bool orig);
 
+	void SendHandshake(const u_char* begin, const u_char* end, bool orig);
+
 	// Overriden from tcp::TCP_ApplicationAnalyzer.
 	virtual void EndpointEOF(bool is_orig);
 
@@ -26,6 +31,7 @@ public:
 
 protected:
 	binpac::SSL::SSL_Conn* interp;
+	binpac::TLSHandshake::Handshake_Conn* handshake_interp;
 	bool had_gap;
 
 };

src/analyzer/protocol/ssl/dtls-analyzer.pac

@@ -0,0 +1,150 @@
+
+refine connection SSL_Conn += {
+
+	%member{
+
+		struct message_info {
+			uint64 message_first_sequence; // the minumum dtls sequence number for this handshake fragment
+			bool first_sequence_seen; // did we actually see the fragment with the smallest number
+			uint64 message_last_sequence; // the mazimum dtls sequence number for this handshake fragment
+			uint16 message_handshake_sequence; // the handshake sequence number of this handshake (to identify)
+			uint32 message_length; // data length of this handshake (data in buffer)
+			uint32 message_sequence_seen; // a bitfield that shows which sequence numbers we already saw, offset from first_seq.
+			u_char* buffer;
+		} server, client;
+	%}
+
+	%init{
+		memset(&server, 0, sizeof(server));
+		memset(&client, 0, sizeof(client));
+	%}
+
+	%cleanup{
+		delete [] server.buffer;
+		delete [] client.buffer;
+	%}
+
+	function proc_dtls(pdu: SSLRecord, sequence: uint64): bool
+		%{
+		//fprintf(stderr, "Type: %d, sequence number: %d, epoch: %d\n", ${pdu.content_type}, sequence, ${pdu.epoch});
+
+		return true;
+		%}
+
+	function proc_handshake(pdu: SSLRecord, rec: Handshake): bool
+		%{
+		uint32 foffset = to_int()(${rec.fragment_offset});
+		uint32 flength = to_int()(${rec.fragment_length});
+		uint32 length = to_int()(${rec.length});
+		uint64 sequence_number = to_int()(${pdu.sequence_number});
+		//fprintf(stderr, "Handshake type: %d, length: %u, seq: %u, foffset: %u, flength: %u\n", ${rec.msg_type}, to_int()(${rec.length}), ${rec.message_seq}, to_int()(${rec.fragment_offset}), to_int()(${rec.fragment_length}));
+
+		if ( foffset == 0 && length == flength )
+			{
+			//fprintf(stderr, "Complete fragment, forwarding...\n");
+			bro_analyzer()->SendHandshake(${rec.msg_type}, length, ${rec.data}.begin(), ${rec.data}.end(), ${pdu.is_orig});
+			return true;
+			}
+
+		// if we fall through here, the message has to be reassembled. Let's first get the right info record...
+		message_info* i;
+		if ( ${pdu.is_orig} )
+			i = &client;
+		else
+			i = &server;
+
+		if ( length > MAX_DTLS_HANDSHAKE_RECORD )
+			{
+			bro_analyzer()->ProtocolViolation(fmt("DTLS record length %u larger than allowed maximum.", length));
+			return true;
+			}
+
+		if ( i->message_handshake_sequence != ${rec.message_seq} || i->message_length != length || i->buffer == 0 )
+			{
+			// cannot resume reassembling. Let's abandon the current data and try anew...
+			delete [] i->buffer;
+			memset(i, 0, sizeof(message_info));
+			i->message_handshake_sequence = ${rec.message_seq};
+			i->message_length = length;
+			i->buffer = new u_char[length];
+			// does not have to be the first sequence number - we cannot figure that out at this point. If it is not,
+			// we will fix that later...
+			i->message_first_sequence = sequence_number;
+			}
+
+		// if we arrive here, we are actually ready to resume.
+		if ( i->message_first_sequence > sequence_number )
+			{
+			if ( i->first_sequence_seen )
+				{
+				bro_analyzer()->ProtocolViolation("Saw second and different first message fragment for handshake.");
+				return true;
+				}
+			// first sequence number was incorrect, let's fix that.
+			uint64 diff = i->message_first_sequence - sequence_number;
+			i->message_sequence_seen = i->message_sequence_seen << diff;
+			i->message_first_sequence = sequence_number;
+			}
+
+		// if we have offset 0, we know the smallest number...
+		if ( foffset == 0 )
+			i->first_sequence_seen = true;
+
+		// check if we already saw the message
+		if ( ( i->message_sequence_seen & ( 1 << (sequence_number - i->message_first_sequence) ) ) != 0 )
+			return true; // do not handle same message fragment twice
+
+		// copy data from fragment to buffer
+		if ( ${rec.data}.length() != flength )
+			{
+			bro_analyzer()->ProtocolViolation(fmt("DTLS handshake record length does not match packet length"));
+			return true;
+			}
+
+		if ( foffset + flength > length )
+			{
+			bro_analyzer()->ProtocolViolation(fmt("DTLS handshake fragment trying to write past end of buffer"));
+			return true;
+			}
+
+		// store that we handled fragment
+		i->message_sequence_seen |= 1 << (sequence_number - i->message_first_sequence);
+		memcpy(i->buffer + foffset, ${rec.data}.data(), ${rec.data}.length());
+
+		//fprintf(stderr, "Copied to buffer offset %u length %u\n", foffset, ${rec.data}.length());
+
+		// store last fragment information if this is the last fragment...
+
+		// check if we saw all fragments so far. If yes, forward...
+		if ( foffset + flength == length )
+			i->message_last_sequence = sequence_number;
+
+		if ( i->message_last_sequence != 0 && i->first_sequence_seen )
+			{
+			uint64 total_length = i->message_last_sequence - i->message_first_sequence;
+			if ( total_length > 30 )
+				{
+				bro_analyzer()->ProtocolViolation(fmt("DTLS Message fragmented over more than 30 pieces. Cannot reassemble."));
+				return true;
+				}
+
+			if ( ( ~(i->message_sequence_seen) & ( ( 1<<(total_length+1) ) -1 ) ) == 0 )
+				{
+				//fprintf(stderr, "ALl fragments here. Total length %u\n", length);
+				bro_analyzer()->SendHandshake(${rec.msg_type}, length, i->buffer, i->buffer + length, ${pdu.is_orig});
+				}
+			}
+
+
+		return true;
+		%}
+};
+
+refine typeattr SSLRecord += &let {
+	proc: bool = $context.connection.proc_dtls(this, to_int()(sequence_number));
+};
+
+refine typeattr Handshake += &let {
+	proc: bool = $context.connection.proc_handshake(rec, this);
+};
+

src/analyzer/protocol/ssl/dtls-protocol.pac

@@ -0,0 +1,60 @@
+
+######################################################################
+# initial datatype for binpac
+######################################################################
+
+type DTLSPDU(is_orig: bool) = record {
+	records: SSLRecord(is_orig)[] &transient;
+};
+
+type SSLRecord(is_orig: bool) = record {
+	content_type: uint8;
+	version: uint16;
+# the epoch signalizes that a changecipherspec message has been received. Hence, everything with
+# an epoch > 0 should be encrypted
+	epoch: uint16;
+	sequence_number: uint48;
+	length: uint16;
+	cont: case valid of {
+		true -> rec: RecordText(this)[] &length=length;
+    false -> swallow: bytestring &restofdata;
+	};
+} &byteorder = bigendian, &let {
+# Do not parse body if packet version invalid
+	valid: bool = $context.connection.dtls_version_ok(version);
+};
+
+type RecordText(rec: SSLRecord) = case rec.epoch of {
+	0	-> plaintext : PlaintextRecord(rec);
+	default -> ciphertext : CiphertextRecord(rec);
+};
+
+refine casetype PlaintextRecord += {
+	HANDSHAKE		-> handshake : Handshake(rec);
+};
+
+type Handshake(rec: SSLRecord) = record {
+	msg_type: uint8;
+	length: uint24;
+	message_seq: uint16;
+	fragment_offset: uint24;
+	fragment_length: uint24;
+	data: bytestring &restofdata;
+}
+
+refine connection SSL_Conn += {
+
+	function dtls_version_ok(version: uint16): uint16
+		%{
+		switch ( version ) {
+		case DTLSv10:
+		case DTLSv12:
+			return true;
+
+		default:
+			bro_analyzer()->ProtocolViolation(fmt("Invalid version in DTLS connection. Packet reported version: %d", version));			
+			return false;
+		}
+		%}
+
+};

src/analyzer/protocol/ssl/dtls.pac

@@ -0,0 +1,36 @@
+# binpac file for SSL analyzer
+
+%include binpac.pac
+%include bro.pac
+
+%extern{
+#include "events.bif.h"
+
+namespace analyzer { namespace dtls { class DTLS_Analyzer; } }
+typedef analyzer::dtls::DTLS_Analyzer* DTLSAnalyzer;
+
+#include "DTLS.h"
+%}
+
+extern type DTLSAnalyzer;
+
+analyzer DTLS withcontext {
+	connection: SSL_Conn;
+	flow:       DTLS_Flow;
+};
+
+connection SSL_Conn(bro_analyzer: DTLSAnalyzer) {
+	upflow = DTLS_Flow(true);
+	downflow = DTLS_Flow(false);
+};
+
+%include ssl-dtls-protocol.pac
+%include dtls-protocol.pac
+
+flow DTLS_Flow(is_orig: bool) {
+  datagram = DTLSPDU(is_orig) withcontext(connection, this);
+}
+
+%include ssl-dtls-analyzer.pac
+%include dtls-analyzer.pac
+%include ssl-defs.pac

src/analyzer/protocol/ssl/proc-certificate.pac

@@ -0,0 +1,30 @@
+	function proc_certificate(is_orig: bool, certificates : bytestring[]) : bool
+		%{
+		if ( certificates->size() == 0 )
+			return true;
+
+		ODesc common;
+		common.AddRaw("Analyzer::ANALYZER_SSL");
+		common.Add(bro_analyzer()->Conn()->StartTime());
+		common.AddRaw(is_orig ? "T" : "F", 1);
+		bro_analyzer()->Conn()->IDString(&common);
+
+		for ( unsigned int i = 0; i < certificates->size(); ++i )
+			{
+			const bytestring& cert = (*certificates)[i];
+
+			ODesc file_handle;
+			file_handle.Add(common.Description());
+			file_handle.Add(i);
+
+			string file_id = file_mgr->HashHandle(file_handle.Description());
+
+			file_mgr->DataIn(reinterpret_cast<const u_char*>(cert.data()),
+			                 cert.length(), bro_analyzer()->GetAnalyzerTag(),
+			                 bro_analyzer()->Conn(), is_orig, file_id);
+			file_mgr->EndOfFile(file_id);
+			}
+		return true;
+		%}
+
+

src/analyzer/protocol/ssl/proc-client-hello.pac

@@ -0,0 +1,42 @@
+	function proc_client_hello(
+					version : uint16, ts : double,
+					client_random : bytestring,
+					session_id : uint8[],
+					cipher_suites16 : uint16[],
+					cipher_suites24 : uint24[]) : bool
+		%{
+		if ( ! version_ok(version) )
+			{
+			bro_analyzer()->ProtocolViolation(fmt("unsupported client SSL version 0x%04x", version));
+			bro_analyzer()->SetSkip(true);
+			}
+		else
+			bro_analyzer()->ProtocolConfirmation();
+
+		if ( ssl_client_hello )
+			{
+			vector<int>* cipher_suites = new vector<int>();
+			if ( cipher_suites16 )
+				std::copy(cipher_suites16->begin(), cipher_suites16->end(), std::back_inserter(*cipher_suites));
+			else
+				std::transform(cipher_suites24->begin(), cipher_suites24->end(), std::back_inserter(*cipher_suites), to_int());
+
+			VectorVal* cipher_vec = new VectorVal(internal_type("index_vec")->AsVectorType());
+			for ( unsigned int i = 0; i < cipher_suites->size(); ++i )
+				{
+				Val* ciph = new Val((*cipher_suites)[i], TYPE_COUNT);
+				cipher_vec->Assign(i, ciph);
+				}
+
+			BifEvent::generate_ssl_client_hello(bro_analyzer(), bro_analyzer()->Conn(),
+							version, ts, new StringVal(client_random.length(),
+							(const char*) client_random.data()),
+							to_string_val(session_id),
+							cipher_vec);
+
+			delete cipher_suites;
+			}
+
+		return true;
+		%}
+

src/analyzer/protocol/ssl/proc-server-hello.pac

@@ -0,0 +1,36 @@
+	function proc_server_hello(
+					version : uint16, ts : double,
+					server_random : bytestring,
+					session_id : uint8[],
+					cipher_suites16 : uint16[],
+					cipher_suites24 : uint24[],
+					comp_method : uint8) : bool
+		%{
+		if ( ! version_ok(version) )
+			{
+			bro_analyzer()->ProtocolViolation(fmt("unsupported server SSL version 0x%04x", version));
+			bro_analyzer()->SetSkip(true);
+			}
+
+		if ( ssl_server_hello )
+			{
+			vector<int>* ciphers = new vector<int>();
+
+			if ( cipher_suites16 )
+				std::copy(cipher_suites16->begin(), cipher_suites16->end(), std::back_inserter(*ciphers));
+			else
+				std::transform(cipher_suites24->begin(), cipher_suites24->end(), std::back_inserter(*ciphers), to_int());
+
+			BifEvent::generate_ssl_server_hello(bro_analyzer(),
+							bro_analyzer()->Conn(),
+							version, ts, new StringVal(server_random.length(),
+							(const char*) server_random.data()),
+							to_string_val(session_id),
+							ciphers->size()==0 ? 0 : ciphers->at(0), comp_method);
+
+			delete ciphers;
+			}
+
+		return true;
+		%}
+

src/analyzer/protocol/ssl/ssl-analyzer.pac

@@ -1,352 +1,19 @@
 # Analyzer for SSL (Bro-specific part).
 
-%extern{
-#include <vector>
-#include <algorithm>
-#include <iostream>
-#include <iterator>
-
-#include "util.h"
-
-#include "file_analysis/Manager.h"
-%}
-
-
-%header{
-	class extract_certs {
-	public:
-		bytestring const& operator() (X509Certificate* cert) const
-			{
-			return cert->certificate();
-			}
-	};
-
-	string orig_label(bool is_orig);
-	string handshake_type_label(int type);
-	%}
-
-%code{
-string orig_label(bool is_orig)
-		{
-		return string(is_orig ? "originator" :"responder");
-		}
-
-	string handshake_type_label(int type)
-		{
-		switch ( type ) {
-		case HELLO_REQUEST: return string("HELLO_REQUEST");
-		case CLIENT_HELLO: return string("CLIENT_HELLO");
-		case SERVER_HELLO: return string("SERVER_HELLO");
-		case SESSION_TICKET: return string("SESSION_TICKET");
-		case CERTIFICATE: return string("CERTIFICATE");
-		case SERVER_KEY_EXCHANGE: return string("SERVER_KEY_EXCHANGE");
-		case CERTIFICATE_REQUEST: return string("CERTIFICATE_REQUEST");
-		case SERVER_HELLO_DONE: return string("SERVER_HELLO_DONE");
-		case CERTIFICATE_VERIFY: return string("CERTIFICATE_VERIFY");
-		case CLIENT_KEY_EXCHANGE: return string("CLIENT_KEY_EXCHANGE");
-		case FINISHED: return string("FINISHED");
-		case CERTIFICATE_URL: return string("CERTIFICATE_URL");
-		case CERTIFICATE_STATUS: return string("CERTIFICATE_STATUS");
-		default: return string(fmt("UNKNOWN (%d)", type));
-		}
-		}
-
-%}
-
-
-function to_string_val(data : uint8[]) : StringVal
-	%{
-	char tmp[32];
-	memset(tmp, 0, sizeof(tmp));
-
-	// Just return an empty string if the string is longer than 32 bytes
-	if ( data && data->size() <= 32 )
-		{
-		for ( unsigned int i = data->size(); i > 0; --i )
-			tmp[i-1] = (*data)[i-1];
-		}
-
-	return new StringVal(32, tmp);
-	%}
-
-function version_ok(vers : uint16) : bool
-	%{
-	switch ( vers ) {
-	case SSLv20:
-	case SSLv30:
-	case TLSv10:
-	case TLSv11:
-	case TLSv12:
-		return true;
-
-	default:
-		return false;
-	}
-	%}
-
 refine connection SSL_Conn += {
 
-	%member{
-		int established_;
-	%}
+	%include proc-client-hello.pac
+	%include proc-server-hello.pac
+	%include proc-certificate.pac
 
-	%init{
-		established_ = false;
-	%}
-
-	%cleanup{
-	%}
-
-	function proc_alert(rec: SSLRecord, level : int, desc : int) : bool
-		%{
-		BifEvent::generate_ssl_alert(bro_analyzer(), bro_analyzer()->Conn(),
-						${rec.is_orig}, level, desc);
-		return true;
-		%}
-
-	function proc_client_hello(rec: SSLRecord,
-					version : uint16, ts : double,
-					client_random : bytestring,
-					session_id : uint8[],
-					cipher_suites16 : uint16[],
-					cipher_suites24 : uint24[]) : bool
-		%{
-		if ( ! version_ok(version) )
-			{
-			bro_analyzer()->ProtocolViolation(fmt("unsupported client SSL version 0x%04x", version));
-			bro_analyzer()->SetSkip(true);
-			}
-		else
-			bro_analyzer()->ProtocolConfirmation();
-
-		if ( ssl_client_hello )
-			{
-			vector<int>* cipher_suites = new vector<int>();
-			if ( cipher_suites16 )
-				std::copy(cipher_suites16->begin(), cipher_suites16->end(), std::back_inserter(*cipher_suites));
-			else
-				std::transform(cipher_suites24->begin(), cipher_suites24->end(), std::back_inserter(*cipher_suites), to_int());
-
-			VectorVal* cipher_vec = new VectorVal(internal_type("index_vec")->AsVectorType());
-			for ( unsigned int i = 0; i < cipher_suites->size(); ++i )
-				{
-				Val* ciph = new Val((*cipher_suites)[i], TYPE_COUNT);
-				cipher_vec->Assign(i, ciph);
-				}
-
-			BifEvent::generate_ssl_client_hello(bro_analyzer(), bro_analyzer()->Conn(),
-							version, ts, new StringVal(client_random.length(),
-							(const char*) client_random.data()),
-							to_string_val(session_id),
-							cipher_vec);
-
-			delete cipher_suites;
-			}
-
-		return true;
-		%}
-
-	function proc_server_hello(rec: SSLRecord,
-					version : uint16, ts : double,
-					server_random : bytestring,
-					session_id : uint8[],
-					cipher_suites16 : uint16[],
-					cipher_suites24 : uint24[],
-					comp_method : uint8) : bool
-		%{
-		if ( ! version_ok(version) )
-			{
-			bro_analyzer()->ProtocolViolation(fmt("unsupported server SSL version 0x%04x", version));
-			bro_analyzer()->SetSkip(true);
-			}
-
-		if ( ssl_server_hello )
-			{
-			vector<int>* ciphers = new vector<int>();
-
-			if ( cipher_suites16 )
-				std::copy(cipher_suites16->begin(), cipher_suites16->end(), std::back_inserter(*ciphers));
-			else
-				std::transform(cipher_suites24->begin(), cipher_suites24->end(), std::back_inserter(*ciphers), to_int());
-
-			BifEvent::generate_ssl_server_hello(bro_analyzer(),
-							bro_analyzer()->Conn(),
-							version, ts, new StringVal(server_random.length(),
-							(const char*) server_random.data()),
-							to_string_val(session_id),
-							ciphers->size()==0 ? 0 : ciphers->at(0), comp_method);
-
-			delete ciphers;
-			}
-
-		return true;
-		%}
-
-	function proc_session_ticket_handshake(rec: SessionTicketHandshake, is_orig: bool): bool
-		%{
-		if ( ssl_session_ticket_handshake )
-			{
-			BifEvent::generate_ssl_session_ticket_handshake(bro_analyzer(),
-							bro_analyzer()->Conn(),
-							${rec.ticket_lifetime_hint},
-							new StringVal(${rec.data}.length(), (const char*) ${rec.data}.data()));
-			}
-		return true;
-		%}
-
-	function proc_ssl_extension(rec: SSLRecord, type: int, sourcedata: const_bytestring) : bool
-		%{
-		// We cheat a little bit here. We want to throw this event
-		// for every extension we encounter, even those that are
-		// handled by more specialized events later. To access the
-		// parsed data, we use sourcedata, which contains the whole
-		// data blob of the extension, including headers. We skip
-		// over those (4 bytes).
-		size_t length = sourcedata.length();
-		if ( length < 4 )
-			{
-			// This should be impossible due to the binpac parser
-			// and protocol description
-			bro_analyzer()->ProtocolViolation(fmt("Impossible extension length: %lu", length));
-			bro_analyzer()->SetSkip(true);
-			return true;
-			}
-
-		length -= 4;
-		const unsigned char* data = sourcedata.begin() + 4;
-
-		if ( ssl_extension )
-			BifEvent::generate_ssl_extension(bro_analyzer(),
-						bro_analyzer()->Conn(), ${rec.is_orig}, type,
-						new StringVal(length, reinterpret_cast<const char*>(data)));
-		return true;
-		%}
-
-	function proc_ec_point_formats(rec: SSLRecord, point_format_list: uint8[]) : bool
-		%{
-		VectorVal* points = new VectorVal(internal_type("index_vec")->AsVectorType());
-
-		if ( point_format_list )
-			{
-			for ( unsigned int i = 0; i < point_format_list->size(); ++i )
-				points->Assign(i, new Val((*point_format_list)[i], TYPE_COUNT));
-			}
-
-		BifEvent::generate_ssl_extension_ec_point_formats(bro_analyzer(), bro_analyzer()->Conn(),
-		   ${rec.is_orig}, points);
-
-		return true;
-		%}
-
-	function proc_elliptic_curves(rec: SSLRecord, list: uint16[]) : bool
-		%{
-		VectorVal* curves = new VectorVal(internal_type("index_vec")->AsVectorType());
-
-		if ( list )
-			{
-			for ( unsigned int i = 0; i < list->size(); ++i )
-				curves->Assign(i, new Val((*list)[i], TYPE_COUNT));
-			}
-
-		BifEvent::generate_ssl_extension_elliptic_curves(bro_analyzer(), bro_analyzer()->Conn(),
-		   ${rec.is_orig}, curves);
-
-		return true;
-		%}
-
-	function proc_apnl(rec: SSLRecord, protocols: ProtocolName[]) : bool
-		%{
-		VectorVal* plist = new VectorVal(internal_type("string_vec")->AsVectorType());
-
-		if ( protocols )
-			{
-			for ( unsigned int i = 0; i < protocols->size(); ++i )
-				plist->Assign(i, new StringVal((*protocols)[i]->name().length(), (const char*) (*protocols)[i]->name().data()));
-			}
-
-		BifEvent::generate_ssl_extension_application_layer_protocol_negotiation(bro_analyzer(), bro_analyzer()->Conn(),
-											${rec.is_orig}, plist);
-
-		return true;
-		%}
-
-	function proc_server_name(rec: SSLRecord, list: ServerName[]) : bool
-		%{
-		VectorVal* servers = new VectorVal(internal_type("string_vec")->AsVectorType());
-
-		if ( list )
-			{
-			for ( unsigned int i = 0, j = 0; i < list->size(); ++i )
-				{
-				ServerName* servername = (*list)[i];
-				if ( servername->name_type() != 0 )
-					{
-					bro_analyzer()->Weird(fmt("Encountered unknown type in server name ssl extension: %d", servername->name_type()));
-					continue;
-					}
-
-				if ( servername->host_name() )
-					servers->Assign(j++, new StringVal(servername->host_name()->host_name().length(), (const char*) servername->host_name()->host_name().data()));
-				else
-					bro_analyzer()->Weird("Empty server_name extension in ssl connection");
-				}
-			}
-
-		BifEvent::generate_ssl_extension_server_name(bro_analyzer(), bro_analyzer()->Conn(),
-		   ${rec.is_orig}, servers);
-
-		return true;
-		%}
-
-	function proc_certificate(rec: SSLRecord, certificates : bytestring[]) : bool
-		%{
-		if ( certificates->size() == 0 )
-			return true;
-
-		ODesc common;
-		common.AddRaw("Analyzer::ANALYZER_SSL");
-		common.Add(bro_analyzer()->Conn()->StartTime());
-		common.AddRaw(${rec.is_orig} ? "T" : "F", 1);
-		bro_analyzer()->Conn()->IDString(&common);
-
-		for ( unsigned int i = 0; i < certificates->size(); ++i )
-			{
-			const bytestring& cert = (*certificates)[i];
-
-			ODesc file_handle;
-			file_handle.Add(common.Description());
-			file_handle.Add(i);
-
-			string file_id = file_mgr->HashHandle(file_handle.Description());
-
-			file_mgr->DataIn(reinterpret_cast<const u_char*>(cert.data()),
-			                 cert.length(), bro_analyzer()->GetAnalyzerTag(),
-			                 bro_analyzer()->Conn(), ${rec.is_orig}, file_id);
-			file_mgr->EndOfFile(file_id);
-			}
-		return true;
-		%}
-
-	function proc_v2_certificate(rec: SSLRecord, cert : bytestring) : bool
+	function proc_v2_certificate(is_orig: bool, cert : bytestring) : bool
 		%{
 		vector<bytestring>* cert_list = new vector<bytestring>(1,cert);
-		bool ret = proc_certificate(rec, cert_list);
+		bool ret = proc_certificate(is_orig, cert_list);
 		delete cert_list;
 		return ret;
 		%}
 
-	function proc_v3_certificate(rec: SSLRecord, cl : X509Certificate[]) : bool
-		%{
-		vector<X509Certificate*>* certs = cl;
-		vector<bytestring>* cert_list = new vector<bytestring>();
-
-		std::transform(certs->begin(), certs->end(),
-		std::back_inserter(*cert_list), extract_certs());
-
-		bool ret = proc_certificate(rec, cert_list);
-		delete cert_list;
-		return ret;
-		%}
 
 	function proc_v2_client_master_key(rec: SSLRecord, cipher_kind: int) : bool
 		%{
@@ -356,209 +23,38 @@ refine connection SSL_Conn += {
 		return true;
 		%}
 
-	function proc_unknown_handshake(hs: Handshake, is_orig: bool) : bool
+	function proc_handshake(rec: SSLRecord, data: bytestring, is_orig: bool) : bool
 		%{
-		bro_analyzer()->ProtocolViolation(fmt("unknown handshake message (%d) from %s",
-			${hs.msg_type}, orig_label(is_orig).c_str()));
+		bro_analyzer()->SendHandshake(data.begin(), data.end(), is_orig);
 		return true;
 		%}
-
-	function proc_unknown_record(rec: SSLRecord) : bool
-		%{
-		bro_analyzer()->ProtocolViolation(fmt("unknown SSL record type (%d) from %s",
-				${rec.content_type},
-				orig_label(${rec.is_orig}).c_str()));
-		return true;
-		%}
-
-	function proc_ciphertext_record(rec : SSLRecord) : bool
-		%{
-		 if ( client_state_ == STATE_ENCRYPTED &&
-		      server_state_ == STATE_ENCRYPTED &&
-		      established_ == false )
-			{
-			established_ = true;
-			BifEvent::generate_ssl_established(bro_analyzer(),
-							bro_analyzer()->Conn());
-			}
-
-		BifEvent::generate_ssl_encrypted_data(bro_analyzer(),
-			bro_analyzer()->Conn(), ${rec.is_orig}, ${rec.content_type}, ${rec.length});
-
-		return true;
-		%}
-
-	function proc_heartbeat(rec : SSLRecord, type: uint8, payload_length: uint16, data: bytestring) : bool
-		%{
-		BifEvent::generate_ssl_heartbeat(bro_analyzer(),
-			bro_analyzer()->Conn(), ${rec.is_orig}, ${rec.length}, type, payload_length,
-			new StringVal(data.length(), (const char*) data.data()));
-		return true;
-		%}
-
-	function proc_check_v2_server_hello_version(version: uint16) : bool
-		%{
-		if ( version != SSLv20 )
-			{
-			bro_analyzer()->ProtocolViolation(fmt("Invalid version in SSL server hello. Version: %d", version));
-			bro_analyzer()->SetSkip(true);
-			return false;
-			}
-
-		return true;
-		%}
-
-	function proc_certificate_status(rec : SSLRecord, status_type: uint8, response: bytestring) : bool
-		%{
-		 if ( status_type == 1 ) // ocsp
-			{
-			BifEvent::generate_ssl_stapled_ocsp(bro_analyzer(),
-							    bro_analyzer()->Conn(), ${rec.is_orig},
-							    new StringVal(response.length(),
-							    (const char*) response.data()));
-			}
-
-		return true;
-		%}
-
-	function proc_ec_server_key_exchange(rec: SSLRecord, curve_type: uint8, curve: uint16) : bool
-		%{
-		if ( curve_type == NAMED_CURVE )
-			BifEvent::generate_ssl_server_curve(bro_analyzer(),
-			  bro_analyzer()->Conn(), curve);
-
-		return true;
-		%}
-
-	function proc_dh_server_key_exchange(rec: SSLRecord, p: bytestring, g: bytestring, Ys: bytestring) : bool
-		%{
-		BifEvent::generate_ssl_dh_server_params(bro_analyzer(),
-			bro_analyzer()->Conn(),
-		  new StringVal(p.length(), (const char*) p.data()),
-		  new StringVal(g.length(), (const char*) g.data()),
-		  new StringVal(Ys.length(), (const char*) Ys.data())
-		  );
-
-		return true;
-		%}
-
-	function proc_ccs(rec: SSLRecord) : bool
-		%{
-		BifEvent::generate_ssl_change_cipher_spec(bro_analyzer(),
-			bro_analyzer()->Conn(), ${rec.is_orig});
-
-		return true;
-		%}
-
-	function proc_handshake(rec: SSLRecord, msg_type: uint8, length: uint24) : bool
-		%{
-		BifEvent::generate_ssl_handshake_message(bro_analyzer(),
-			bro_analyzer()->Conn(), ${rec.is_orig}, msg_type, to_int()(length));
-
-		return true;
-		%}
-
 };
 
-refine typeattr Alert += &let {
-	proc : bool = $context.connection.proc_alert(rec, level, description);
-};
 
 refine typeattr V2Error += &let {
 	proc : bool = $context.connection.proc_alert(rec, -1, error_code);
 };
 
-refine typeattr Heartbeat += &let {
-	proc : bool = $context.connection.proc_heartbeat(rec, type, payload_length, data);
-};
-
-refine typeattr ClientHello += &let {
-	proc : bool = $context.connection.proc_client_hello(rec, client_version,
-				gmt_unix_time, random_bytes,
-				session_id, csuits, 0);
-};
 
 refine typeattr V2ClientHello += &let {
-	proc : bool = $context.connection.proc_client_hello(rec, client_version, 0,
+	proc : bool = $context.connection.proc_client_hello(client_version, 0,
 				challenge, session_id, 0, ciphers);
 };
 
-refine typeattr ServerHello += &let {
-	proc : bool = $context.connection.proc_server_hello(rec, server_version,
-			gmt_unix_time, random_bytes, session_id, cipher_suite, 0,
-			compression_method);
-};
-
 refine typeattr V2ServerHello += &let {
 	check_v2 : bool = $context.connection.proc_check_v2_server_hello_version(server_version);
 
-	proc : bool = $context.connection.proc_server_hello(rec, server_version, 0,
+	proc : bool = $context.connection.proc_server_hello(server_version, 0,
 				conn_id_data, 0, 0, ciphers, 0) &requires(check_v2) &if(check_v2 == true);
 
-	cert : bool = $context.connection.proc_v2_certificate(rec, cert_data)
+	cert : bool = $context.connection.proc_v2_certificate(rec.is_orig, cert_data)
 		&requires(proc) &requires(check_v2) &if(check_v2 == true);
 };
 
-refine typeattr Certificate += &let {
-	proc : bool = $context.connection.proc_v3_certificate(rec, certificates);
-};
-
 refine typeattr V2ClientMasterKey += &let {
 	proc : bool = $context.connection.proc_v2_client_master_key(rec, cipher_kind);
 };
 
-refine typeattr UnknownHandshake += &let {
-	proc : bool = $context.connection.proc_unknown_handshake(hs, is_orig);
-};
-
-refine typeattr SessionTicketHandshake += &let {
-	proc : bool = $context.connection.proc_session_ticket_handshake(this, rec.is_orig);
-}
-
-refine typeattr UnknownRecord += &let {
-	proc : bool = $context.connection.proc_unknown_record(rec);
-};
-
-refine typeattr CiphertextRecord += &let {
-	proc : bool = $context.connection.proc_ciphertext_record(rec);
-}
-
-refine typeattr SSLExtension += &let {
-	proc : bool = $context.connection.proc_ssl_extension(rec, type, sourcedata);
-};
-
-refine typeattr EcPointFormats += &let {
-	proc : bool = $context.connection.proc_ec_point_formats(rec, point_format_list);
-};
-
-refine typeattr EllipticCurves += &let {
-	proc : bool = $context.connection.proc_elliptic_curves(rec, elliptic_curve_list);
-};
-
-refine typeattr ApplicationLayerProtocolNegotiationExtension += &let {
-	proc : bool = $context.connection.proc_apnl(rec, protocol_name_list);
-};
-
-refine typeattr ServerNameExt += &let {
-	proc : bool = $context.connection.proc_server_name(rec, server_names);
-};
-
-refine typeattr CertificateStatus += &let {
-	proc : bool = $context.connection.proc_certificate_status(rec, status_type, response);
-};
-
-refine typeattr EcServerKeyExchange += &let {
-	proc : bool = $context.connection.proc_ec_server_key_exchange(rec, curve_type, curve);
-};
-
-refine typeattr DhServerKeyExchange += &let {
-	proc : bool = $context.connection.proc_dh_server_key_exchange(rec, dh_p, dh_g, dh_Ys);
-};
-
-refine typeattr ChangeCipherSpec += &let {
-	proc : bool = $context.connection.proc_ccs(rec);
-};
-
 refine typeattr Handshake += &let {
-	proc : bool = $context.connection.proc_handshake(rec, msg_type, length);
+	proc : bool = $context.connection.proc_handshake(rec, data, rec.is_orig);
 };

src/analyzer/protocol/ssl/ssl-defs.pac

@@ -1,5 +1,86 @@
 # Some common definitions for the SSL and SSL record-layer analyzers.
 
+type uint24 = record {
+	byte1 : uint8;
+	byte2 : uint8;
+	byte3 : uint8;
+};
+
+type uint48 = record {
+	byte1 : uint8;
+	byte2 : uint8;
+	byte3 : uint8;
+	byte4 : uint8;
+	byte5 : uint8;
+	byte6 : uint8;
+};
+
+
+%header{
+	string orig_label(bool is_orig);
+	%}
+
+
+%code{
+string orig_label(bool is_orig)
+		{
+		return string(is_orig ? "originator" :"responder");
+		}
+%}
+
+%header{
+	class to_int {
+	public:
+		int operator()(uint24 * num) const
+		{
+		return (num->byte1() << 16) | (num->byte2() << 8) | num->byte3();
+		}
+
+		uint64 operator()(uint48 * num) const
+		{
+		return ((uint64)num->byte1() << 40) | ((uint64)num->byte2() << 32) | ((uint64)num->byte3() << 24) |
+		  ((uint64)num->byte4() << 16) | ((uint64)num->byte5() << 8) | (uint64)num->byte6();
+		}
+	};
+
+	string state_label(int state_nr);
+%}
+
+extern type to_int;
+
+function to_string_val(data : uint8[]) : StringVal
+	%{
+	char tmp[32];
+	memset(tmp, 0, sizeof(tmp));
+
+	// Just return an empty string if the string is longer than 32 bytes
+	if ( data && data->size() <= 32 )
+		{
+		for ( unsigned int i = data->size(); i > 0; --i )
+			tmp[i-1] = (*data)[i-1];
+		}
+
+	return new StringVal(32, tmp);
+	%}
+
+function version_ok(vers : uint16) : bool
+	%{
+	switch ( vers ) {
+	case SSLv20:
+	case SSLv30:
+	case TLSv10:
+	case TLSv11:
+	case TLSv12:
+	case DTLSv10:
+	case DTLSv12:
+		return true;
+
+	default:
+		return false;
+	}
+	%}
+
+
 %extern{
 #include <string>
 using std::string;
@@ -7,6 +88,9 @@ using std::string;
 #include "events.bif.h"
 %}
 
+# a maximum of 100k for one record seems safe 
+let MAX_DTLS_HANDSHAKE_RECORD: uint32 = 100000;
+
 enum ContentType {
 	CHANGE_CIPHER_SPEC = 20,
 	ALERT = 21,
@@ -27,7 +111,11 @@ enum SSLVersions {
 	SSLv30		= 0x0300,
 	TLSv10		= 0x0301,
 	TLSv11		= 0x0302,
-	TLSv12		= 0x0303
+	TLSv12		= 0x0303,
+
+	DTLSv10   = 0xFEFF,
+	# DTLSv11 does not exist.
+	DTLSv12   = 0xFEFD
 };
 
 enum SSLExtensions {

src/analyzer/protocol/ssl/ssl-dtls-analyzer.pac

@@ -0,0 +1,106 @@
+
+%extern{
+#include <vector>
+#include <algorithm>
+#include <iostream>
+#include <iterator>
+
+#include "util.h"
+
+#include "file_analysis/Manager.h"
+%}
+
+refine connection SSL_Conn += {
+
+	%member{
+		int established_;
+	%}
+
+	%init{
+		established_ = false;
+	%}
+
+	%cleanup{
+	%}
+
+	function proc_alert(rec: SSLRecord, level : int, desc : int) : bool
+		%{
+		BifEvent::generate_ssl_alert(bro_analyzer(), bro_analyzer()->Conn(),
+						${rec.is_orig}, level, desc);
+		return true;
+		%}
+	function proc_unknown_record(rec: SSLRecord) : bool
+		%{
+		bro_analyzer()->ProtocolViolation(fmt("unknown SSL record type (%d) from %s",
+				${rec.content_type},
+				orig_label(${rec.is_orig}).c_str()));
+		return true;
+		%}
+
+	function proc_ciphertext_record(rec : SSLRecord) : bool
+		%{
+		 if ( client_state_ == STATE_ENCRYPTED &&
+		      server_state_ == STATE_ENCRYPTED &&
+		      established_ == false )
+			{
+			established_ = true;
+			BifEvent::generate_ssl_established(bro_analyzer(),
+							bro_analyzer()->Conn());
+			}
+
+		BifEvent::generate_ssl_encrypted_data(bro_analyzer(),
+			bro_analyzer()->Conn(), ${rec.is_orig}, ${rec.content_type}, ${rec.length});
+
+		return true;
+		%}
+
+	function proc_heartbeat(rec : SSLRecord, type: uint8, payload_length: uint16, data: bytestring) : bool
+		%{
+		BifEvent::generate_ssl_heartbeat(bro_analyzer(),
+			bro_analyzer()->Conn(), ${rec.is_orig}, ${rec.length}, type, payload_length,
+			new StringVal(data.length(), (const char*) data.data()));
+		return true;
+		%}
+
+	function proc_check_v2_server_hello_version(version: uint16) : bool
+		%{
+		if ( version != SSLv20 )
+			{
+			bro_analyzer()->ProtocolViolation(fmt("Invalid version in SSL server hello. Version: %d", version));
+			bro_analyzer()->SetSkip(true);
+			return false;
+			}
+
+		return true;
+		%}
+
+
+	function proc_ccs(rec: SSLRecord) : bool
+		%{
+		BifEvent::generate_ssl_change_cipher_spec(bro_analyzer(),
+			bro_analyzer()->Conn(), ${rec.is_orig});
+
+		return true;
+		%}
+
+};
+
+refine typeattr Alert += &let {
+	proc : bool = $context.connection.proc_alert(rec, level, description);
+};
+
+refine typeattr Heartbeat += &let {
+	proc : bool = $context.connection.proc_heartbeat(rec, type, payload_length, data);
+};
+
+refine typeattr UnknownRecord += &let {
+	proc : bool = $context.connection.proc_unknown_record(rec);
+};
+
+refine typeattr CiphertextRecord += &let {
+	proc : bool = $context.connection.proc_ciphertext_record(rec);
+}
+
+refine typeattr ChangeCipherSpec += &let {
+	proc : bool = $context.connection.proc_ccs(rec);
+};

src/analyzer/protocol/ssl/ssl-dtls-protocol.pac

@@ -0,0 +1,134 @@
+
+######################################################################
+# General definitions
+######################################################################
+
+type PlaintextRecord(rec: SSLRecord) = case rec.content_type of {
+	CHANGE_CIPHER_SPEC	-> ch_cipher : ChangeCipherSpec(rec);
+	ALERT			-> alert : Alert(rec);
+	HEARTBEAT -> heartbeat: Heartbeat(rec);
+	APPLICATION_DATA	-> app_data : ApplicationData(rec);
+	default			-> unknown_record : UnknownRecord(rec);
+};
+
+
+######################################################################
+# Encryption Tracking
+######################################################################
+
+enum AnalyzerState {
+	STATE_CLEAR,
+	STATE_ENCRYPTED
+};
+
+%code{
+	string state_label(int state_nr)
+		{
+		switch ( state_nr ) {
+		case STATE_CLEAR:
+			return string("CLEAR");
+
+		case STATE_ENCRYPTED:
+			return string("ENCRYPTED");
+
+		default:
+			return string(fmt("UNKNOWN (%d)", state_nr));
+		}
+		}
+%}
+
+######################################################################
+# Change Cipher Spec Protocol (7.1.)
+######################################################################
+
+type ChangeCipherSpec(rec: SSLRecord) = record {
+	type : uint8;
+} &length = 1, &let {
+	state_changed : bool =
+		$context.connection.startEncryption(rec.is_orig);
+};
+
+
+######################################################################
+# Alert Protocol (7.2.)
+######################################################################
+
+type Alert(rec: SSLRecord) = record {
+	level : uint8;
+	description: uint8;
+};
+
+
+######################################################################
+# V3 Application Data
+######################################################################
+
+# Application data should always be encrypted, so we should not
+# reach this point.
+type ApplicationData(rec: SSLRecord) = record {
+	data : bytestring &restofdata &transient;
+};
+
+######################################################################
+# V3 Heartbeat
+######################################################################
+
+type Heartbeat(rec: SSLRecord) = record {
+	type : uint8;
+	payload_length : uint16;
+	data : bytestring &restofdata;
+};
+
+
+
+######################################################################
+# Fragmentation (6.2.1.)
+######################################################################
+
+type UnknownRecord(rec: SSLRecord) = record {
+	cont : bytestring &restofdata &transient;
+};
+
+type CiphertextRecord(rec: SSLRecord) = record {
+	cont : bytestring &restofdata &transient;
+};
+
+######################################################################
+# binpac analyzer for SSL including
+######################################################################
+
+refine connection SSL_Conn += {
+
+	%member{
+		int client_state_;
+		int server_state_;
+		int record_layer_version_;
+	%}
+
+	%init{
+		server_state_ = STATE_CLEAR;
+		client_state_ = STATE_CLEAR;
+		record_layer_version_ = UNKNOWN_VERSION;
+	%}
+
+	function client_state() : int %{ return client_state_; %}
+
+	function server_state() : int %{ return client_state_; %}
+
+	function state(is_orig: bool) : int
+		%{
+		if ( is_orig )
+			return client_state_;
+		else
+			return server_state_;
+		%}
+
+	function startEncryption(is_orig: bool) : bool
+		%{
+		if ( is_orig )
+			client_state_ = STATE_ENCRYPTED;
+		else
+			server_state_ = STATE_ENCRYPTED;
+		return true;
+		%}
+};

src/analyzer/protocol/ssl/ssl-protocol.pac

@@ -2,30 +2,6 @@
 # To be used in conjunction with an SSL record-layer analyzer.
 # Separation is necessary due to possible fragmentation of SSL records.
 
-######################################################################
-# General definitions
-######################################################################
-
-type uint24 = record {
-	byte1 : uint8;
-	byte2 : uint8;
-	byte3 : uint8;
-};
-
-%header{
-	class to_int {
-	public:
-		int operator()(uint24 * num) const
-		{
-		return (num->byte1() << 16) | (num->byte2() << 8) | num->byte3();
-		}
-	};
-
-	string state_label(int state_nr);
-%}
-
-extern type to_int;
-
 type SSLRecord(is_orig: bool) = record {
 	head0 : uint8;
 	head1 : uint8;
@@ -58,161 +34,19 @@ type RecordText(rec: SSLRecord) = case $context.connection.state(rec.is_orig) of
 		-> plaintext : PlaintextRecord(rec);
 };
 
-type PlaintextRecord(rec: SSLRecord) = case rec.content_type of {
-	CHANGE_CIPHER_SPEC	-> ch_cipher : ChangeCipherSpec(rec);
-	ALERT			-> alert : Alert(rec);
+refine casetype PlaintextRecord += {
 	HANDSHAKE		-> handshake : Handshake(rec);
-	HEARTBEAT -> heartbeat: Heartbeat(rec);
-	APPLICATION_DATA	-> app_data : ApplicationData(rec);
 	V2_ERROR		-> v2_error : V2Error(rec);
 	V2_CLIENT_HELLO		-> v2_client_hello : V2ClientHello(rec);
 	V2_CLIENT_MASTER_KEY	-> v2_client_master_key : V2ClientMasterKey(rec);
 	V2_SERVER_HELLO		-> v2_server_hello : V2ServerHello(rec);
-	default			-> unknown_record : UnknownRecord(rec);
 };
 
-######################################################################
-# TLS Extensions
-######################################################################
-
-type SSLExtension(rec: SSLRecord) = record {
-	type: uint16;
-	data_len: uint16;
-
-	# Pretty code ahead. Deal with the fact that perhaps extensions are
-	# not really present and we do not want to fail because of that.
-	ext: case type of {
-		EXT_APPLICATION_LAYER_PROTOCOL_NEGOTIATION -> apnl: ApplicationLayerProtocolNegotiationExtension(rec)[] &until($element == 0 || $element != 0);
-		EXT_ELLIPTIC_CURVES -> elliptic_curves: EllipticCurves(rec)[] &until($element == 0 || $element != 0);
-		EXT_EC_POINT_FORMATS -> ec_point_formats: EcPointFormats(rec)[] &until($element == 0 || $element != 0);
-#		EXT_STATUS_REQUEST -> status_request: StatusRequest(rec)[] &until($element == 0 || $element != 0);
-		EXT_SERVER_NAME -> server_name: ServerNameExt(rec)[] &until($element == 0 || $element != 0);
-		default -> data: bytestring &restofdata;
-	};
-} &length=data_len+4 &exportsourcedata;
-
-type ServerNameHostName() = record {
-	length: uint16;
-	host_name: bytestring &length=length;
+# Handshakes are parsed by the handshake analyzer.
+type Handshake(rec: SSLRecord) = record {
+	data: bytestring &restofdata;
 };
 
-type ServerName() = record {
-	name_type: uint8; # has to be 0 for host-name
-	name: case name_type of {
-		0 -> host_name: ServerNameHostName;
-		default -> data : bytestring &restofdata &transient; # unknown name
-	};
-};
-
-type ServerNameExt(rec: SSLRecord) = record {
-	length: uint16;
-	server_names: ServerName[] &until($input.length() == 0);
-} &length=length+2;
-
-# Do not parse for now. Structure is correct, but only contains asn.1 data that we would not use further.
-#type OcspStatusRequest(rec: SSLRecord) = record {
-#	responder_id_list_length: uint16;
-#	responder_id_list: bytestring &length=responder_id_list_length;
-#	request_extensions_length: uint16;
-#	request_extensions: bytestring &length=request_extensions_length;
-#};
-#
-#type StatusRequest(rec: SSLRecord) = record {
-#	status_type: uint8; # 1 -> ocsp
-#	req: case status_type of {
-#		1 -> ocsp_status_request: OcspStatusRequest(rec);
-#		default -> data : bytestring &restofdata &transient; # unknown
-#	};
-#};
-
-type EcPointFormats(rec: SSLRecord) = record {
-	length: uint8;
-	point_format_list: uint8[length];
-};
-
-type EllipticCurves(rec: SSLRecord) = record {
-	length: uint16;
-	elliptic_curve_list: uint16[length/2];
-};
-
-type ProtocolName() = record {
-  length: uint8;
-	name: bytestring &length=length;
-};
-
-type ApplicationLayerProtocolNegotiationExtension(rec: SSLRecord) = record {
-	length: uint16;
-	protocol_name_list: ProtocolName[] &until($input.length() == 0);
-} &length=length+2;
-
-######################################################################
-# Encryption Tracking
-######################################################################
-
-enum AnalyzerState {
-	STATE_CLEAR,
-	STATE_ENCRYPTED
-};
-
-%code{
-	string state_label(int state_nr)
-		{
-		switch ( state_nr ) {
-		case STATE_CLEAR:
-			return string("CLEAR");
-
-		case STATE_ENCRYPTED:
-			return string("ENCRYPTED");
-
-		default:
-			return string(fmt("UNKNOWN (%d)", state_nr));
-		}
-		}
-%}
-
-######################################################################
-# SSLv3 Handshake Protocols (7.)
-######################################################################
-
-enum HandshakeType {
-	HELLO_REQUEST       = 0,
-	CLIENT_HELLO        = 1,
-	SERVER_HELLO        = 2,
-	SESSION_TICKET      = 4, # RFC 5077
-	CERTIFICATE         = 11,
-	SERVER_KEY_EXCHANGE = 12,
-	CERTIFICATE_REQUEST = 13,
-	SERVER_HELLO_DONE   = 14,
-	CERTIFICATE_VERIFY  = 15,
-	CLIENT_KEY_EXCHANGE = 16,
-	FINISHED            = 20,
-	CERTIFICATE_URL     = 21, # RFC 3546
-	CERTIFICATE_STATUS  = 22, # RFC 3546
-};
-
-
-######################################################################
-# V3 Change Cipher Spec Protocol (7.1.)
-######################################################################
-
-type ChangeCipherSpec(rec: SSLRecord) = record {
-	type : uint8;
-} &length = 1, &let {
-	state_changed : bool =
-		$context.connection.startEncryption(rec.is_orig);
-};
-
-
-######################################################################
-# V3 Alert Protocol (7.2.)
-######################################################################
-
-type Alert(rec: SSLRecord) = record {
-	level : uint8;
-	description: uint8;
-};
-
-
 ######################################################################
 # V2 Error Records (SSLv2 2.7.)
 ######################################################################
@@ -224,53 +58,6 @@ type V2Error(rec: SSLRecord) = record {
 };
 
 
-######################################################################
-# V3 Application Data
-######################################################################
-
-# Application data should always be encrypted, so we should not
-# reach this point.
-type ApplicationData(rec: SSLRecord) = record {
-	data : bytestring &restofdata &transient;
-};
-
-######################################################################
-# V3 Heartbeat
-######################################################################
-
-type Heartbeat(rec: SSLRecord) = record {
-	type : uint8;
-	payload_length : uint16;
-	data : bytestring &restofdata;
-};
-
-######################################################################
-# V3 Hello Request (7.4.1.1.)
-######################################################################
-
-# Hello Request is empty
-type HelloRequest(rec: SSLRecord) = empty;
-
-
-######################################################################
-# V3 Client Hello (7.4.1.2.)
-######################################################################
-
-type ClientHello(rec: SSLRecord) = record {
-	client_version : uint16;
-	gmt_unix_time : uint32;
-	random_bytes : bytestring &length = 28;
-	session_len : uint8;
-	session_id : uint8[session_len];
-	csuit_len : uint16 &check(csuit_len > 1 && csuit_len % 2 == 0);
-	csuits : uint16[csuit_len/2];
-	cmeth_len : uint8 &check(cmeth_len > 0);
-	cmeths : uint8[cmeth_len];
-	# This weirdness is to deal with the possible existence or absence
-	# of the following fields.
-	ext_len: uint16[] &until($element == 0 || $element != 0);
-	extensions : SSLExtension(rec)[] &until($input.length() == 0);
-};
 
 ######################################################################
 # V2 Client Hello (SSLv2 2.5.)
@@ -288,26 +75,6 @@ type V2ClientHello(rec: SSLRecord) = record {
 };
 
 
-######################################################################
-# V3 Server Hello (7.4.1.3.)
-######################################################################
-
-type ServerHello(rec: SSLRecord) = record {
-	server_version : uint16;
-	gmt_unix_time : uint32;
-	random_bytes : bytestring &length = 28;
-	session_len : uint8;
-	session_id : uint8[session_len];
-	cipher_suite : uint16[1];
-	compression_method : uint8;
-	# This weirdness is to deal with the possible existence or absence
-	# of the following fields.
-	ext_len: uint16[] &until($element == 0 || $element != 0);
-	extensions : SSLExtension(rec)[] &until($input.length() == 0);
-} &let {
-	cipher_set : bool =
-		$context.connection.set_cipher(cipher_suite[0]);
-};
 
 ######################################################################
 # V2 Server Hello (SSLv2 2.6.)
@@ -329,298 +96,6 @@ type V2ServerHello(rec: SSLRecord) = record {
 };
 
 
-######################################################################
-# V3 Server Certificate (7.4.2.)
-######################################################################
-
-type X509Certificate = record {
-	length : uint24;
-	certificate : bytestring &length = to_int()(length);
-};
-
-type Certificate(rec: SSLRecord) = record {
-	length : uint24;
-	certificates : X509Certificate[] &until($input.length() == 0);
-} &length = to_int()(length)+3;
-
-# OCSP Stapling
-
-type CertificateStatus(rec: SSLRecord) = record {
-	status_type: uint8; # 1 = ocsp, everything else is undefined
-	length : uint24;
-	response: bytestring &restofdata;
-};
-
-######################################################################
-# V3 Server Key Exchange Message (7.4.3.)
-######################################################################
-
-# Usually, the server key exchange does not contain any information
-# that we are interested in.
-#
-# The exception is when we are using an ECDHE, DHE or DH-Anon suite.
-# In this case, we can extract information about the chosen cipher from
-# here.
-type ServerKeyExchange(rec: SSLRecord) = case $context.connection.chosen_cipher() of {
-	TLS_ECDH_ECDSA_WITH_NULL_SHA,
-	TLS_ECDH_ECDSA_WITH_RC4_128_SHA,
-	TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA,
-	TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA,
-	TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA,
-	TLS_ECDHE_ECDSA_WITH_NULL_SHA,
-	TLS_ECDHE_ECDSA_WITH_RC4_128_SHA,
-	TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA,
-	TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
-	TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
-	TLS_ECDH_RSA_WITH_NULL_SHA,
-	TLS_ECDH_RSA_WITH_RC4_128_SHA,
-	TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA,
-	TLS_ECDH_RSA_WITH_AES_128_CBC_SHA,
-	TLS_ECDH_RSA_WITH_AES_256_CBC_SHA,
-	TLS_ECDHE_RSA_WITH_NULL_SHA,
-	TLS_ECDHE_RSA_WITH_RC4_128_SHA,
-	TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,
-	TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
-	TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
-	TLS_ECDH_ANON_WITH_NULL_SHA,
-	TLS_ECDH_ANON_WITH_RC4_128_SHA,
-	TLS_ECDH_ANON_WITH_3DES_EDE_CBC_SHA,
-	TLS_ECDH_ANON_WITH_AES_128_CBC_SHA,
-	TLS_ECDH_ANON_WITH_AES_256_CBC_SHA,
-	TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,
-	TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384,
-	TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256,
-	TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384,
-	TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256,
-	TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384,
-	TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256,
-	TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384,
-	TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
-	TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
-	TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256,
-	TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384,
-	TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
-	TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
-	TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256,
-	TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384,
-	TLS_ECDHE_PSK_WITH_RC4_128_SHA,
-	TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA,
-	TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA,
-	TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA,
-	TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256,
-	TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384,
-	TLS_ECDHE_PSK_WITH_NULL_SHA,
-	TLS_ECDHE_PSK_WITH_NULL_SHA256,
-	TLS_ECDHE_PSK_WITH_NULL_SHA384,
-	TLS_ECDHE_ECDSA_WITH_ARIA_128_CBC_SHA256,
-	TLS_ECDHE_ECDSA_WITH_ARIA_256_CBC_SHA384,
-	TLS_ECDH_ECDSA_WITH_ARIA_128_CBC_SHA256,
-	TLS_ECDH_ECDSA_WITH_ARIA_256_CBC_SHA384,
-	TLS_ECDHE_RSA_WITH_ARIA_128_CBC_SHA256,
-	TLS_ECDHE_RSA_WITH_ARIA_256_CBC_SHA384,
-	TLS_ECDH_RSA_WITH_ARIA_128_CBC_SHA256,
-	TLS_ECDH_RSA_WITH_ARIA_256_CBC_SHA384,
-	TLS_ECDHE_ECDSA_WITH_ARIA_128_GCM_SHA256,
-	TLS_ECDHE_ECDSA_WITH_ARIA_256_GCM_SHA384,
-	TLS_ECDH_ECDSA_WITH_ARIA_128_GCM_SHA256,
-	TLS_ECDH_ECDSA_WITH_ARIA_256_GCM_SHA384,
-	TLS_ECDHE_RSA_WITH_ARIA_128_GCM_SHA256,
-	TLS_ECDHE_RSA_WITH_ARIA_256_GCM_SHA384,
-	TLS_ECDH_RSA_WITH_ARIA_128_GCM_SHA256,
-	TLS_ECDH_RSA_WITH_ARIA_256_GCM_SHA384,
-	TLS_ECDHE_PSK_WITH_ARIA_128_CBC_SHA256,
-	TLS_ECDHE_PSK_WITH_ARIA_256_CBC_SHA384,
-	TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_CBC_SHA256,
-	TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_CBC_SHA384,
-	TLS_ECDH_ECDSA_WITH_CAMELLIA_128_CBC_SHA256,
-	TLS_ECDH_ECDSA_WITH_CAMELLIA_256_CBC_SHA384,
-	TLS_ECDHE_RSA_WITH_CAMELLIA_128_CBC_SHA256,
-	TLS_ECDHE_RSA_WITH_CAMELLIA_256_CBC_SHA384,
-	TLS_ECDH_RSA_WITH_CAMELLIA_128_CBC_SHA256,
-	TLS_ECDH_RSA_WITH_CAMELLIA_256_CBC_SHA384,
-	TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_GCM_SHA256,
-	TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_GCM_SHA384,
-	TLS_ECDH_ECDSA_WITH_CAMELLIA_128_GCM_SHA256,
-	TLS_ECDH_ECDSA_WITH_CAMELLIA_256_GCM_SHA384,
-	TLS_ECDHE_RSA_WITH_CAMELLIA_128_GCM_SHA256,
-	TLS_ECDHE_RSA_WITH_CAMELLIA_256_GCM_SHA384,
-	TLS_ECDH_RSA_WITH_CAMELLIA_128_GCM_SHA256,
-	TLS_ECDH_RSA_WITH_CAMELLIA_256_GCM_SHA384,
-	TLS_ECDHE_PSK_WITH_CAMELLIA_128_CBC_SHA256,
-	TLS_ECDHE_PSK_WITH_CAMELLIA_256_CBC_SHA384,
-	TLS_ECDHE_ECDSA_WITH_AES_128_CCM,
-	TLS_ECDHE_ECDSA_WITH_AES_256_CCM,
-	TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8,
-	TLS_ECDHE_ECDSA_WITH_AES_256_CCM_8,
-	TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,
-	TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256
-		-> ec_server_key_exchange : EcServerKeyExchange(rec);
-
-	# DHE suites
-	TLS_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA,
-	TLS_DHE_DSS_WITH_DES_CBC_SHA,
-	TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA,
-	TLS_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA,
-	TLS_DHE_RSA_WITH_DES_CBC_SHA,
-	TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA,
-	TLS_DHE_DSS_WITH_AES_128_CBC_SHA,
-	TLS_DHE_RSA_WITH_AES_128_CBC_SHA,
-	TLS_DHE_DSS_WITH_AES_256_CBC_SHA,
-	TLS_DHE_RSA_WITH_AES_256_CBC_SHA,
-	TLS_DHE_DSS_WITH_AES_128_CBC_SHA256,
-	TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA,
-	TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA,
-	TLS_DHE_DSS_EXPORT1024_WITH_DES_CBC_SHA,
-	TLS_DHE_DSS_EXPORT1024_WITH_RC4_56_SHA,
-	TLS_DHE_DSS_WITH_RC4_128_SHA,
-	TLS_DHE_RSA_WITH_AES_128_CBC_SHA256,
-	TLS_DHE_DSS_WITH_AES_256_CBC_SHA256,
-	TLS_DHE_RSA_WITH_AES_256_CBC_SHA256,
-	TLS_DHE_DSS_WITH_3DES_EDE_CBC_RMD,
-	TLS_DHE_DSS_WITH_AES_128_CBC_RMD,
-	TLS_DHE_DSS_WITH_AES_256_CBC_RMD,
-	TLS_DHE_RSA_WITH_3DES_EDE_CBC_RMD,
-	TLS_DHE_RSA_WITH_AES_128_CBC_RMD,
-	TLS_DHE_RSA_WITH_AES_256_CBC_RMD,
-	TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA,
-	TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA,
-	TLS_DHE_PSK_WITH_RC4_128_SHA,
-	TLS_DHE_PSK_WITH_3DES_EDE_CBC_SHA,
-	TLS_DHE_PSK_WITH_AES_128_CBC_SHA,
-	TLS_DHE_PSK_WITH_AES_256_CBC_SHA,
-	TLS_DHE_DSS_WITH_SEED_CBC_SHA,
-	TLS_DHE_RSA_WITH_SEED_CBC_SHA,
-	TLS_DHE_RSA_WITH_AES_128_GCM_SHA256,
-	TLS_DHE_RSA_WITH_AES_256_GCM_SHA384,
-	TLS_DHE_DSS_WITH_AES_128_GCM_SHA256,
-	TLS_DHE_DSS_WITH_AES_256_GCM_SHA384,
-	TLS_DHE_PSK_WITH_AES_128_GCM_SHA256,
-	TLS_DHE_PSK_WITH_AES_256_GCM_SHA384,
-	TLS_DHE_PSK_WITH_AES_128_CBC_SHA256,
-	TLS_DHE_PSK_WITH_AES_256_CBC_SHA384,
-	TLS_DHE_PSK_WITH_NULL_SHA256,
-	TLS_DHE_PSK_WITH_NULL_SHA384,
-	TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA256,
-	TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA256,
-	TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA256,
-	TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256,
-	TLS_DHE_DSS_WITH_ARIA_128_CBC_SHA256,
-	TLS_DHE_DSS_WITH_ARIA_256_CBC_SHA384,
-	TLS_DHE_RSA_WITH_ARIA_128_CBC_SHA256,
-	TLS_DHE_RSA_WITH_ARIA_256_CBC_SHA384,
-	TLS_DHE_RSA_WITH_ARIA_128_GCM_SHA256,
-	TLS_DHE_RSA_WITH_ARIA_256_GCM_SHA384,
-	TLS_DHE_DSS_WITH_ARIA_128_GCM_SHA256,
-	TLS_DHE_DSS_WITH_ARIA_256_GCM_SHA384,
-	TLS_DHE_PSK_WITH_ARIA_128_CBC_SHA256,
-	TLS_DHE_PSK_WITH_ARIA_256_CBC_SHA384,
-	TLS_DHE_PSK_WITH_ARIA_128_GCM_SHA256,
-	TLS_DHE_PSK_WITH_ARIA_256_GCM_SHA384,
-	TLS_DHE_RSA_WITH_CAMELLIA_128_GCM_SHA256,
-	TLS_DHE_RSA_WITH_CAMELLIA_256_GCM_SHA384,
-	TLS_DHE_DSS_WITH_CAMELLIA_128_GCM_SHA256,
-	TLS_DHE_DSS_WITH_CAMELLIA_256_GCM_SHA384,
-	TLS_DHE_PSK_WITH_CAMELLIA_128_GCM_SHA256,
-	TLS_DHE_PSK_WITH_CAMELLIA_256_GCM_SHA384,
-	TLS_DHE_PSK_WITH_CAMELLIA_128_CBC_SHA256,
-	TLS_DHE_PSK_WITH_CAMELLIA_256_CBC_SHA384,
-	TLS_DHE_RSA_WITH_AES_128_CCM,
-	TLS_DHE_RSA_WITH_AES_256_CCM,
-	TLS_DHE_RSA_WITH_AES_128_CCM_8,
-	TLS_DHE_RSA_WITH_AES_256_CCM_8,
-	TLS_DHE_PSK_WITH_AES_128_CCM,
-	TLS_DHE_PSK_WITH_AES_256_CCM,
-	TLS_PSK_DHE_WITH_AES_128_CCM_8,
-	TLS_PSK_DHE_WITH_AES_256_CCM_8,
-	TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256,
-	# DH-anon suites
-	TLS_DH_ANON_EXPORT_WITH_RC4_40_MD5,
-	TLS_DH_ANON_WITH_RC4_128_MD5,
-	TLS_DH_ANON_EXPORT_WITH_DES40_CBC_SHA,
-	TLS_DH_ANON_WITH_DES_CBC_SHA,
-	TLS_DH_ANON_WITH_3DES_EDE_CBC_SHA,
-	TLS_DH_ANON_WITH_AES_128_CBC_SHA,
-	TLS_DH_ANON_WITH_AES_256_CBC_SHA,
-	TLS_DH_ANON_WITH_CAMELLIA_128_CBC_SHA,
-	TLS_DH_ANON_WITH_AES_128_CBC_SHA256,
-	TLS_DH_ANON_WITH_AES_256_CBC_SHA256,
-	TLS_DH_ANON_WITH_CAMELLIA_256_CBC_SHA,
-	TLS_DH_ANON_WITH_SEED_CBC_SHA,
-	TLS_DH_ANON_WITH_AES_128_GCM_SHA256,
-	TLS_DH_ANON_WITH_AES_256_GCM_SHA384,
-	TLS_DH_ANON_WITH_CAMELLIA_128_CBC_SHA256,
-	TLS_DH_ANON_WITH_CAMELLIA_256_CBC_SHA256,
-	TLS_DH_ANON_WITH_ARIA_128_CBC_SHA256,
-	TLS_DH_ANON_WITH_ARIA_256_CBC_SHA384,
-	TLS_DH_ANON_WITH_ARIA_128_GCM_SHA256,
-	TLS_DH_ANON_WITH_ARIA_256_GCM_SHA384,
-	TLS_DH_ANON_WITH_CAMELLIA_128_GCM_SHA256,
-	TLS_DH_ANON_WITH_CAMELLIA_256_GCM_SHA384
-	# DH non-anon suites do not send a ServerKeyExchange
-		-> dh_server_key_exchange : DhServerKeyExchange(rec);
-
-	default
-		-> key : bytestring &restofdata &transient;
-};
-
-# For the moment, we really only are interested in the curve name. If it
-# is not set (if the server sends explicit parameters), we do not bother.
-# We also do not parse the actual signature data following the named curve.
-type EcServerKeyExchange(rec: SSLRecord) = record {
-	curve_type: uint8;
-	curve: uint16; # only if curve_type = 3 (NAMED_CURVE)
-	data: bytestring &restofdata &transient;
-};
-
-# For both, dh_anon and dhe the ServerKeyExchange starts with a ServerDHParams
-# structure. After that, they start to differ, but we do not care about that.
-type DhServerKeyExchange(rec: SSLRecord) = record {
-	dh_p_length: uint16;
-	dh_p: bytestring &length=dh_p_length;
-	dh_g_length: uint16;
-	dh_g: bytestring &length=dh_g_length;
-	dh_Ys_length: uint16;
-	dh_Ys: bytestring &length=dh_Ys_length;
-	data: bytestring &restofdata &transient;
-};
-
-
-######################################################################
-# V3 Certificate Request (7.4.4.)
-######################################################################
-
-# For now, ignore Certificate Request Details; just eat up message.
-type CertificateRequest(rec: SSLRecord) = record {
-	cont : bytestring &restofdata &transient;
-};
-
-
-######################################################################
-# V3 Server Hello Done (7.4.5.)
-######################################################################
-
-# Server Hello Done is empty
-type ServerHelloDone(rec: SSLRecord) = empty;
-
-
-######################################################################
-# V3 Client Certificate (7.4.6.)
-######################################################################
-
-# Client Certificate is identical to Server Certificate;
-# no further definition here
-
-
-######################################################################
-# V3 Client Key Exchange Message (7.4.7.)
-######################################################################
-
-# For now ignore details of ClientKeyExchange (most of it is
-# encrypted anyway); just eat up message.
-type ClientKeyExchange(rec: SSLRecord) = record {
-	key : bytestring &restofdata &transient;
-};
-
 ######################################################################
 # V2 Client Master Key (SSLv2 2.5.)
 ######################################################################
@@ -641,75 +116,6 @@ type V2ClientMasterKey(rec: SSLRecord) = record {
 };
 
 
-######################################################################
-# V3 Certificate Verify (7.4.8.)
-######################################################################
-
-# For now, ignore Certificate Verify; just eat up the message.
-type CertificateVerify(rec: SSLRecord) = record {
-	cont : bytestring &restofdata &transient;
-};
-
-
-######################################################################
-# V3 Finished (7.4.9.)
-######################################################################
-
-# The finished messages are always sent after encryption is in effect,
-# so we will not be able to read those messages.
-type Finished(rec: SSLRecord) = record {
-	cont : bytestring &restofdata &transient;
-};
-
-type SessionTicketHandshake(rec: SSLRecord) = record {
-	ticket_lifetime_hint: uint32;
-	data:                 bytestring &restofdata;
-};
-
-######################################################################
-# V3 Handshake Protocol (7.)
-######################################################################
-
-type UnknownHandshake(hs: Handshake, is_orig: bool) = record {
-	data : bytestring &restofdata &transient;
-};
-
-type Handshake(rec: SSLRecord) = record {
-	msg_type : uint8;
-	length : uint24;
-
-	body : case msg_type of {
-		HELLO_REQUEST       -> hello_request       : HelloRequest(rec);
-		CLIENT_HELLO        -> client_hello        : ClientHello(rec);
-		SERVER_HELLO        -> server_hello        : ServerHello(rec);
-		SESSION_TICKET      -> session_ticket      : SessionTicketHandshake(rec);
-		CERTIFICATE         -> certificate         : Certificate(rec);
-		SERVER_KEY_EXCHANGE -> server_key_exchange : ServerKeyExchange(rec);
-		CERTIFICATE_REQUEST -> certificate_request : CertificateRequest(rec);
-		SERVER_HELLO_DONE   -> server_hello_done   : ServerHelloDone(rec);
-		CERTIFICATE_VERIFY  -> certificate_verify  : CertificateVerify(rec);
-		CLIENT_KEY_EXCHANGE -> client_key_exchange : ClientKeyExchange(rec);
-		FINISHED            -> finished            : Finished(rec);
-		CERTIFICATE_URL     -> certificate_url     : bytestring &restofdata &transient;
-		CERTIFICATE_STATUS  -> certificate_status  : CertificateStatus(rec);
-		default             -> unknown_handshake   : UnknownHandshake(this, rec.is_orig);
-	} &length = to_int()(length);
-};
-
-
-######################################################################
-# Fragmentation (6.2.1.)
-######################################################################
-
-type UnknownRecord(rec: SSLRecord) = record {
-	cont : bytestring &restofdata &transient;
-};
-
-type CiphertextRecord(rec: SSLRecord) = record {
-	cont : bytestring &restofdata &transient;
-};
-
-
 ######################################################################
 # initial datatype for binpac
 ######################################################################
@@ -725,31 +131,15 @@ type SSLPDU(is_orig: bool) = record {
 
 refine connection SSL_Conn += {
 
-	%member{
-		int client_state_;
-		int server_state_;
-		int record_layer_version_;
-		uint32 chosen_cipher_;
-	%}
-
-	%init{
-		server_state_ = STATE_CLEAR;
-		client_state_ = STATE_CLEAR;
-		record_layer_version_ = UNKNOWN_VERSION;
-		chosen_cipher_ = NO_CHOSEN_CIPHER;
-	%}
-
-	function chosen_cipher() : int %{ return chosen_cipher_; %}
-
-	function set_cipher(cipher: uint32) : bool
-		%{
-		chosen_cipher_ = cipher;
-		return true;
-		%}
-
 	function determine_ssl_record_layer(head0 : uint8, head1 : uint8,
 					head2 : uint8, head3: uint8, head4: uint8, is_orig: bool) : int
 		%{
+		// stop processing if we already had a protocol violation or otherwhise
+		// decided that we do not want to parse anymore. Just setting skip is not
+		// enough for the data that is already in the pipe.
+		if ( bro_analyzer()->Skipping() )
+			return UNKNOWN_VERSION;
+
 		// re-check record layer version to be sure that we still are synchronized with
 		// the data stream
 		if ( record_layer_version_ != UNKNOWN_VERSION && record_layer_version_ != SSLv20 )
@@ -818,24 +208,4 @@ refine connection SSL_Conn += {
 		return UNKNOWN_VERSION;
 		%}
 
-	function client_state() : int %{ return client_state_; %}
-
-	function server_state() : int %{ return client_state_; %}
-
-	function state(is_orig: bool) : int
-		%{
-		if ( is_orig )
-			return client_state_;
-		else
-			return server_state_;
-		%}
-
-	function startEncryption(is_orig: bool) : bool
-		%{
-		if ( is_orig )
-			client_state_ = STATE_ENCRYPTED;
-		else
-			server_state_ = STATE_ENCRYPTED;
-		return true;
-		%}
 };

src/analyzer/protocol/ssl/ssl.pac

@@ -10,23 +10,32 @@
 
 %extern{
 #include "events.bif.h"
+
+namespace analyzer { namespace ssl { class SSL_Analyzer; } }
+typedef analyzer::ssl::SSL_Analyzer* SSLAnalyzer;
+
+#include "SSL.h"
 %}
 
+extern type SSLAnalyzer;
+
 analyzer SSL withcontext {
 	connection: SSL_Conn;
 	flow:       SSL_Flow;
 };
 
-connection SSL_Conn(bro_analyzer: BroAnalyzer) {
+connection SSL_Conn(bro_analyzer: SSLAnalyzer) {
 	upflow = SSL_Flow(true);
 	downflow = SSL_Flow(false);
 };
 
+%include ssl-dtls-protocol.pac
 %include ssl-protocol.pac
 
 flow SSL_Flow(is_orig: bool) {
 	flowunit = SSLPDU(is_orig) withcontext(connection, this);
 }
 
+%include ssl-dtls-analyzer.pac
 %include ssl-analyzer.pac
 %include ssl-defs.pac

src/analyzer/protocol/ssl/tls-handshake-analyzer.pac

@@ -0,0 +1,273 @@
+# Analyzer for SSL/TLS Handshake protocol (Bro-specific part).
+
+%extern{
+#include <vector>
+#include <algorithm>
+#include <iostream>
+#include <iterator>
+
+#include "util.h"
+
+#include "file_analysis/Manager.h"
+%}
+
+%header{
+	class extract_certs {
+	public:
+		bytestring const& operator() (X509Certificate* cert) const
+			{
+			return cert->certificate();
+			}
+	};
+
+	string orig_label(bool is_orig);
+	string handshake_type_label(int type);
+	%}
+
+refine connection Handshake_Conn += {
+
+	%include proc-client-hello.pac
+	%include proc-server-hello.pac
+	%include proc-certificate.pac
+
+	function proc_session_ticket_handshake(rec: SessionTicketHandshake, is_orig: bool): bool
+		%{
+		if ( ssl_session_ticket_handshake )
+			{
+			BifEvent::generate_ssl_session_ticket_handshake(bro_analyzer(),
+							bro_analyzer()->Conn(),
+							${rec.ticket_lifetime_hint},
+							new StringVal(${rec.data}.length(), (const char*) ${rec.data}.data()));
+			}
+		return true;
+		%}
+
+	function proc_ssl_extension(rec: HandshakeRecord, type: int, sourcedata: const_bytestring) : bool
+		%{
+		// We cheat a little bit here. We want to throw this event
+		// for every extension we encounter, even those that are
+		// handled by more specialized events later. To access the
+		// parsed data, we use sourcedata, which contains the whole
+		// data blob of the extension, including headers. We skip
+		// over those (4 bytes).
+		size_t length = sourcedata.length();
+		if ( length < 4 )
+			{
+			// This should be impossible due to the binpac parser
+			// and protocol description
+			bro_analyzer()->ProtocolViolation(fmt("Impossible extension length: %zu", length));
+			bro_analyzer()->SetSkip(true);
+			return true;
+			}
+
+		length -= 4;
+		const unsigned char* data = sourcedata.begin() + 4;
+
+		if ( ssl_extension )
+			BifEvent::generate_ssl_extension(bro_analyzer(),
+						bro_analyzer()->Conn(), ${rec.is_orig}, type,
+						new StringVal(length, reinterpret_cast<const char*>(data)));
+		return true;
+		%}
+
+	function proc_ec_point_formats(rec: HandshakeRecord, point_format_list: uint8[]) : bool
+		%{
+		VectorVal* points = new VectorVal(internal_type("index_vec")->AsVectorType());
+
+		if ( point_format_list )
+			{
+			for ( unsigned int i = 0; i < point_format_list->size(); ++i )
+				points->Assign(i, new Val((*point_format_list)[i], TYPE_COUNT));
+			}
+
+		BifEvent::generate_ssl_extension_ec_point_formats(bro_analyzer(), bro_analyzer()->Conn(),
+		   ${rec.is_orig}, points);
+
+		return true;
+		%}
+
+	function proc_elliptic_curves(rec: HandshakeRecord, list: uint16[]) : bool
+		%{
+		VectorVal* curves = new VectorVal(internal_type("index_vec")->AsVectorType());
+
+		if ( list )
+			{
+			for ( unsigned int i = 0; i < list->size(); ++i )
+				curves->Assign(i, new Val((*list)[i], TYPE_COUNT));
+			}
+
+		BifEvent::generate_ssl_extension_elliptic_curves(bro_analyzer(), bro_analyzer()->Conn(),
+		   ${rec.is_orig}, curves);
+
+		return true;
+		%}
+
+	function proc_apnl(rec: HandshakeRecord, protocols: ProtocolName[]) : bool
+		%{
+		VectorVal* plist = new VectorVal(internal_type("string_vec")->AsVectorType());
+
+		if ( protocols )
+			{
+			for ( unsigned int i = 0; i < protocols->size(); ++i )
+				plist->Assign(i, new StringVal((*protocols)[i]->name().length(), (const char*) (*protocols)[i]->name().data()));
+			}
+
+		BifEvent::generate_ssl_extension_application_layer_protocol_negotiation(bro_analyzer(), bro_analyzer()->Conn(),
+											${rec.is_orig}, plist);
+
+		return true;
+		%}
+
+	function proc_server_name(rec: HandshakeRecord, list: ServerName[]) : bool
+		%{
+		VectorVal* servers = new VectorVal(internal_type("string_vec")->AsVectorType());
+
+		if ( list )
+			{
+			for ( unsigned int i = 0, j = 0; i < list->size(); ++i )
+				{
+				ServerName* servername = (*list)[i];
+				if ( servername->name_type() != 0 )
+					{
+					bro_analyzer()->Weird(fmt("Encountered unknown type in server name ssl extension: %d", servername->name_type()));
+					continue;
+					}
+
+				if ( servername->host_name() )
+					servers->Assign(j++, new StringVal(servername->host_name()->host_name().length(), (const char*) servername->host_name()->host_name().data()));
+				else
+					bro_analyzer()->Weird("Empty server_name extension in ssl connection");
+				}
+			}
+
+		BifEvent::generate_ssl_extension_server_name(bro_analyzer(), bro_analyzer()->Conn(),
+		   ${rec.is_orig}, servers);
+
+		return true;
+		%}
+
+	function proc_v3_certificate(is_orig: bool, cl : X509Certificate[]) : bool
+		%{
+		vector<X509Certificate*>* certs = cl;
+		vector<bytestring>* cert_list = new vector<bytestring>();
+
+		std::transform(certs->begin(), certs->end(),
+		std::back_inserter(*cert_list), extract_certs());
+
+		bool ret = proc_certificate(is_orig, cert_list);
+		delete cert_list;
+		return ret;
+		%}
+
+	function proc_unknown_handshake(hs: HandshakeRecord, is_orig: bool) : bool
+		%{
+		bro_analyzer()->ProtocolViolation(fmt("unknown handshake message (%d) from %s",
+			${hs.msg_type}, orig_label(is_orig).c_str()));
+		return true;
+		%}
+
+	function proc_certificate_status(rec : HandshakeRecord, status_type: uint8, response: bytestring) : bool
+		%{
+		 if ( status_type == 1 ) // ocsp
+			{
+			BifEvent::generate_ssl_stapled_ocsp(bro_analyzer(),
+							    bro_analyzer()->Conn(), ${rec.is_orig},
+							    new StringVal(response.length(),
+							    (const char*) response.data()));
+			}
+
+		return true;
+		%}
+
+	function proc_ec_server_key_exchange(rec: HandshakeRecord, curve_type: uint8, curve: uint16) : bool
+		%{
+		if ( curve_type == NAMED_CURVE )
+			BifEvent::generate_ssl_server_curve(bro_analyzer(),
+			  bro_analyzer()->Conn(), curve);
+
+		return true;
+		%}
+
+	function proc_dh_server_key_exchange(rec: HandshakeRecord, p: bytestring, g: bytestring, Ys: bytestring) : bool
+		%{
+		BifEvent::generate_ssl_dh_server_params(bro_analyzer(),
+			bro_analyzer()->Conn(),
+		  new StringVal(p.length(), (const char*) p.data()),
+		  new StringVal(g.length(), (const char*) g.data()),
+		  new StringVal(Ys.length(), (const char*) Ys.data())
+		  );
+
+		return true;
+		%}
+
+	function proc_handshake(is_orig: bool, msg_type: uint8, length: uint24) : bool
+		%{
+		BifEvent::generate_ssl_handshake_message(bro_analyzer(),
+			bro_analyzer()->Conn(), is_orig, msg_type, to_int()(length));
+
+		return true;
+		%}
+
+
+};
+
+refine typeattr ClientHello += &let {
+	proc : bool = $context.connection.proc_client_hello(client_version,
+				gmt_unix_time, random_bytes,
+				session_id, csuits, 0);
+};
+
+refine typeattr ServerHello += &let {
+	proc : bool = $context.connection.proc_server_hello(server_version,
+			gmt_unix_time, random_bytes, session_id, cipher_suite, 0,
+			compression_method);
+};
+
+refine typeattr Certificate += &let {
+	proc : bool = $context.connection.proc_v3_certificate(rec.is_orig, certificates);
+};
+
+refine typeattr UnknownHandshake += &let {
+	proc : bool = $context.connection.proc_unknown_handshake(hs, is_orig);
+};
+
+refine typeattr SessionTicketHandshake += &let {
+	proc : bool = $context.connection.proc_session_ticket_handshake(this, rec.is_orig);
+}
+
+refine typeattr SSLExtension += &let {
+	proc : bool = $context.connection.proc_ssl_extension(rec, type, sourcedata);
+};
+
+refine typeattr EcPointFormats += &let {
+	proc : bool = $context.connection.proc_ec_point_formats(rec, point_format_list);
+};
+
+refine typeattr EllipticCurves += &let {
+	proc : bool = $context.connection.proc_elliptic_curves(rec, elliptic_curve_list);
+};
+
+refine typeattr ApplicationLayerProtocolNegotiationExtension += &let {
+	proc : bool = $context.connection.proc_apnl(rec, protocol_name_list);
+};
+
+refine typeattr ServerNameExt += &let {
+	proc : bool = $context.connection.proc_server_name(rec, server_names);
+};
+
+refine typeattr CertificateStatus += &let {
+	proc : bool = $context.connection.proc_certificate_status(rec, status_type, response);
+};
+
+refine typeattr EcServerKeyExchange += &let {
+	proc : bool = $context.connection.proc_ec_server_key_exchange(rec, curve_type, curve);
+};
+
+refine typeattr DhServerKeyExchange += &let {
+	proc : bool = $context.connection.proc_dh_server_key_exchange(rec, dh_p, dh_g, dh_Ys);
+};
+
+refine typeattr Handshake += &let {
+	proc : bool = $context.connection.proc_handshake(rec.is_orig, rec.msg_type, rec.msg_length);
+};
+

src/analyzer/protocol/ssl/tls-handshake-protocol.pac

@@ -0,0 +1,538 @@
+######################################################################
+# Handshake Protocols (7.)
+######################################################################
+
+enum HandshakeType {
+	HELLO_REQUEST       = 0,
+	CLIENT_HELLO        = 1,
+	SERVER_HELLO        = 2,
+	HELLO_VERIFY_REQUEST = 3, # DTLS
+	SESSION_TICKET      = 4, # RFC 5077
+	CERTIFICATE         = 11,
+	SERVER_KEY_EXCHANGE = 12,
+	CERTIFICATE_REQUEST = 13,
+	SERVER_HELLO_DONE   = 14,
+	CERTIFICATE_VERIFY  = 15,
+	CLIENT_KEY_EXCHANGE = 16,
+	FINISHED            = 20,
+	CERTIFICATE_URL     = 21, # RFC 3546
+	CERTIFICATE_STATUS  = 22, # RFC 3546
+};
+
+
+######################################################################
+# V3 Handshake Protocol (7.)
+######################################################################
+
+type HandshakeRecord(is_orig: bool) = record {
+  msg_type: uint8;
+	msg_length: uint24;
+	rec: Handshake(this);
+} &length=(to_int()(msg_length) + 4);
+
+type Handshake(rec: HandshakeRecord) = case rec.msg_type of {
+	HELLO_REQUEST        -> hello_request        : HelloRequest(rec);
+	CLIENT_HELLO         -> client_hello         : ClientHello(rec);
+	SERVER_HELLO         -> server_hello         : ServerHello(rec);
+	HELLO_VERIFY_REQUEST -> hello_verify_request : HelloVerifyRequest(rec);
+	SESSION_TICKET       -> session_ticket       : SessionTicketHandshake(rec);
+	CERTIFICATE          -> certificate          : Certificate(rec);
+	SERVER_KEY_EXCHANGE  -> server_key_exchange  : ServerKeyExchange(rec);
+	CERTIFICATE_REQUEST  -> certificate_request  : CertificateRequest(rec);
+	SERVER_HELLO_DONE    -> server_hello_done    : ServerHelloDone(rec);
+	CERTIFICATE_VERIFY   -> certificate_verify   : CertificateVerify(rec);
+	CLIENT_KEY_EXCHANGE  -> client_key_exchange  : ClientKeyExchange(rec);
+	FINISHED             -> finished             : Finished(rec);
+	CERTIFICATE_URL      -> certificate_url      : bytestring &restofdata &transient;
+	CERTIFICATE_STATUS   -> certificate_status   : CertificateStatus(rec);
+	default              -> unknown_handshake    : UnknownHandshake(rec, rec.is_orig);
+}
+
+type HandshakePDU(is_orig: bool) = record {
+	records: HandshakeRecord(is_orig)[] &transient;
+} &byteorder = bigendian;
+
+type UnknownHandshake(hs: HandshakeRecord, is_orig: bool) = record {
+	data : bytestring &restofdata &transient;
+};
+
+######################################################################
+# V3 Hello Request (7.4.1.1.)
+######################################################################
+
+# Hello Request is empty
+type HelloRequest(rec: HandshakeRecord) = empty;
+
+
+######################################################################
+# V3 Client Hello (7.4.1.2.)
+######################################################################
+
+type ClientHello(rec: HandshakeRecord) = record {
+	client_version : uint16;
+	gmt_unix_time : uint32;
+	random_bytes : bytestring &length = 28;
+	session_len : uint8;
+	session_id : uint8[session_len];
+	dtls_cookie: case client_version of {
+		DTLSv10 -> cookie: ClientHelloCookie(rec);
+		default -> nothing: bytestring &length=0;
+	};
+	csuit_len : uint16 &check(csuit_len > 1 && csuit_len % 2 == 0);
+	csuits : uint16[csuit_len/2];
+	cmeth_len : uint8 &check(cmeth_len > 0);
+	cmeths : uint8[cmeth_len];
+	# This weirdness is to deal with the possible existence or absence
+	# of the following fields.
+	ext_len: uint16[] &until($element == 0 || $element != 0);
+	extensions : SSLExtension(rec)[] &until($input.length() == 0);
+};
+
+type ClientHelloCookie(rec: HandshakeRecord) = record {
+	cookie_len : uint8;
+	cookie : bytestring &length = cookie_len;
+};
+
+######################################################################
+# V3 Server Hello (7.4.1.3.)
+######################################################################
+
+type ServerHello(rec: HandshakeRecord) = record {
+	server_version : uint16;
+	gmt_unix_time : uint32;
+	random_bytes : bytestring &length = 28;
+	session_len : uint8;
+	session_id : uint8[session_len];
+	cipher_suite : uint16[1];
+	compression_method : uint8;
+	# This weirdness is to deal with the possible existence or absence
+	# of the following fields.
+	ext_len: uint16[] &until($element == 0 || $element != 0);
+	extensions : SSLExtension(rec)[] &until($input.length() == 0);
+} &let {
+	cipher_set : bool =
+		$context.connection.set_cipher(cipher_suite[0]);
+};
+
+######################################################################
+# DTLS Hello Verify Request
+######################################################################
+
+type HelloVerifyRequest(rec: HandshakeRecord) = record {
+	version: uint16;
+	cookie_length: uint8;
+	cookie: bytestring &length=cookie_length;
+};
+
+######################################################################
+# V3 Server Certificate (7.4.2.)
+######################################################################
+
+type X509Certificate = record {
+	length : uint24;
+	certificate : bytestring &length = to_int()(length);
+};
+
+type Certificate(rec: HandshakeRecord) = record {
+	length : uint24;
+	certificates : X509Certificate[] &until($input.length() == 0);
+} &length = to_int()(length)+3;
+
+# OCSP Stapling
+
+type CertificateStatus(rec: HandshakeRecord) = record {
+	status_type: uint8; # 1 = ocsp, everything else is undefined
+	length : uint24;
+	response: bytestring &restofdata;
+};
+
+######################################################################
+# V3 Server Key Exchange Message (7.4.3.)
+######################################################################
+
+# Usually, the server key exchange does not contain any information
+# that we are interested in.
+#
+# The exception is when we are using an ECDHE, DHE or DH-Anon suite.
+# In this case, we can extract information about the chosen cipher from
+# here.
+type ServerKeyExchange(rec: HandshakeRecord) = case $context.connection.chosen_cipher() of {
+	TLS_ECDH_ECDSA_WITH_NULL_SHA,
+	TLS_ECDH_ECDSA_WITH_RC4_128_SHA,
+	TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA,
+	TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA,
+	TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA,
+	TLS_ECDHE_ECDSA_WITH_NULL_SHA,
+	TLS_ECDHE_ECDSA_WITH_RC4_128_SHA,
+	TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA,
+	TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
+	TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
+	TLS_ECDH_RSA_WITH_NULL_SHA,
+	TLS_ECDH_RSA_WITH_RC4_128_SHA,
+	TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA,
+	TLS_ECDH_RSA_WITH_AES_128_CBC_SHA,
+	TLS_ECDH_RSA_WITH_AES_256_CBC_SHA,
+	TLS_ECDHE_RSA_WITH_NULL_SHA,
+	TLS_ECDHE_RSA_WITH_RC4_128_SHA,
+	TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,
+	TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
+	TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
+	TLS_ECDH_ANON_WITH_NULL_SHA,
+	TLS_ECDH_ANON_WITH_RC4_128_SHA,
+	TLS_ECDH_ANON_WITH_3DES_EDE_CBC_SHA,
+	TLS_ECDH_ANON_WITH_AES_128_CBC_SHA,
+	TLS_ECDH_ANON_WITH_AES_256_CBC_SHA,
+	TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,
+	TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384,
+	TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256,
+	TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384,
+	TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256,
+	TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384,
+	TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256,
+	TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384,
+	TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+	TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
+	TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256,
+	TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384,
+	TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+	TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+	TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256,
+	TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384,
+	TLS_ECDHE_PSK_WITH_RC4_128_SHA,
+	TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA,
+	TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA,
+	TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA,
+	TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256,
+	TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384,
+	TLS_ECDHE_PSK_WITH_NULL_SHA,
+	TLS_ECDHE_PSK_WITH_NULL_SHA256,
+	TLS_ECDHE_PSK_WITH_NULL_SHA384,
+	TLS_ECDHE_ECDSA_WITH_ARIA_128_CBC_SHA256,
+	TLS_ECDHE_ECDSA_WITH_ARIA_256_CBC_SHA384,
+	TLS_ECDH_ECDSA_WITH_ARIA_128_CBC_SHA256,
+	TLS_ECDH_ECDSA_WITH_ARIA_256_CBC_SHA384,
+	TLS_ECDHE_RSA_WITH_ARIA_128_CBC_SHA256,
+	TLS_ECDHE_RSA_WITH_ARIA_256_CBC_SHA384,
+	TLS_ECDH_RSA_WITH_ARIA_128_CBC_SHA256,
+	TLS_ECDH_RSA_WITH_ARIA_256_CBC_SHA384,
+	TLS_ECDHE_ECDSA_WITH_ARIA_128_GCM_SHA256,
+	TLS_ECDHE_ECDSA_WITH_ARIA_256_GCM_SHA384,
+	TLS_ECDH_ECDSA_WITH_ARIA_128_GCM_SHA256,
+	TLS_ECDH_ECDSA_WITH_ARIA_256_GCM_SHA384,
+	TLS_ECDHE_RSA_WITH_ARIA_128_GCM_SHA256,
+	TLS_ECDHE_RSA_WITH_ARIA_256_GCM_SHA384,
+	TLS_ECDH_RSA_WITH_ARIA_128_GCM_SHA256,
+	TLS_ECDH_RSA_WITH_ARIA_256_GCM_SHA384,
+	TLS_ECDHE_PSK_WITH_ARIA_128_CBC_SHA256,
+	TLS_ECDHE_PSK_WITH_ARIA_256_CBC_SHA384,
+	TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_CBC_SHA256,
+	TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_CBC_SHA384,
+	TLS_ECDH_ECDSA_WITH_CAMELLIA_128_CBC_SHA256,
+	TLS_ECDH_ECDSA_WITH_CAMELLIA_256_CBC_SHA384,
+	TLS_ECDHE_RSA_WITH_CAMELLIA_128_CBC_SHA256,
+	TLS_ECDHE_RSA_WITH_CAMELLIA_256_CBC_SHA384,
+	TLS_ECDH_RSA_WITH_CAMELLIA_128_CBC_SHA256,
+	TLS_ECDH_RSA_WITH_CAMELLIA_256_CBC_SHA384,
+	TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_GCM_SHA256,
+	TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_GCM_SHA384,
+	TLS_ECDH_ECDSA_WITH_CAMELLIA_128_GCM_SHA256,
+	TLS_ECDH_ECDSA_WITH_CAMELLIA_256_GCM_SHA384,
+	TLS_ECDHE_RSA_WITH_CAMELLIA_128_GCM_SHA256,
+	TLS_ECDHE_RSA_WITH_CAMELLIA_256_GCM_SHA384,
+	TLS_ECDH_RSA_WITH_CAMELLIA_128_GCM_SHA256,
+	TLS_ECDH_RSA_WITH_CAMELLIA_256_GCM_SHA384,
+	TLS_ECDHE_PSK_WITH_CAMELLIA_128_CBC_SHA256,
+	TLS_ECDHE_PSK_WITH_CAMELLIA_256_CBC_SHA384,
+	TLS_ECDHE_ECDSA_WITH_AES_128_CCM,
+	TLS_ECDHE_ECDSA_WITH_AES_256_CCM,
+	TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8,
+	TLS_ECDHE_ECDSA_WITH_AES_256_CCM_8,
+	TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,
+	TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256
+		-> ec_server_key_exchange : EcServerKeyExchange(rec);
+
+	# DHE suites
+	TLS_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA,
+	TLS_DHE_DSS_WITH_DES_CBC_SHA,
+	TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA,
+	TLS_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA,
+	TLS_DHE_RSA_WITH_DES_CBC_SHA,
+	TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA,
+	TLS_DHE_DSS_WITH_AES_128_CBC_SHA,
+	TLS_DHE_RSA_WITH_AES_128_CBC_SHA,
+	TLS_DHE_DSS_WITH_AES_256_CBC_SHA,
+	TLS_DHE_RSA_WITH_AES_256_CBC_SHA,
+	TLS_DHE_DSS_WITH_AES_128_CBC_SHA256,
+	TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA,
+	TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA,
+	TLS_DHE_DSS_EXPORT1024_WITH_DES_CBC_SHA,
+	TLS_DHE_DSS_EXPORT1024_WITH_RC4_56_SHA,
+	TLS_DHE_DSS_WITH_RC4_128_SHA,
+	TLS_DHE_RSA_WITH_AES_128_CBC_SHA256,
+	TLS_DHE_DSS_WITH_AES_256_CBC_SHA256,
+	TLS_DHE_RSA_WITH_AES_256_CBC_SHA256,
+	TLS_DHE_DSS_WITH_3DES_EDE_CBC_RMD,
+	TLS_DHE_DSS_WITH_AES_128_CBC_RMD,
+	TLS_DHE_DSS_WITH_AES_256_CBC_RMD,
+	TLS_DHE_RSA_WITH_3DES_EDE_CBC_RMD,
+	TLS_DHE_RSA_WITH_AES_128_CBC_RMD,
+	TLS_DHE_RSA_WITH_AES_256_CBC_RMD,
+	TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA,
+	TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA,
+	TLS_DHE_PSK_WITH_RC4_128_SHA,
+	TLS_DHE_PSK_WITH_3DES_EDE_CBC_SHA,
+	TLS_DHE_PSK_WITH_AES_128_CBC_SHA,
+	TLS_DHE_PSK_WITH_AES_256_CBC_SHA,
+	TLS_DHE_DSS_WITH_SEED_CBC_SHA,
+	TLS_DHE_RSA_WITH_SEED_CBC_SHA,
+	TLS_DHE_RSA_WITH_AES_128_GCM_SHA256,
+	TLS_DHE_RSA_WITH_AES_256_GCM_SHA384,
+	TLS_DHE_DSS_WITH_AES_128_GCM_SHA256,
+	TLS_DHE_DSS_WITH_AES_256_GCM_SHA384,
+	TLS_DHE_PSK_WITH_AES_128_GCM_SHA256,
+	TLS_DHE_PSK_WITH_AES_256_GCM_SHA384,
+	TLS_DHE_PSK_WITH_AES_128_CBC_SHA256,
+	TLS_DHE_PSK_WITH_AES_256_CBC_SHA384,
+	TLS_DHE_PSK_WITH_NULL_SHA256,
+	TLS_DHE_PSK_WITH_NULL_SHA384,
+	TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA256,
+	TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA256,
+	TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA256,
+	TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256,
+	TLS_DHE_DSS_WITH_ARIA_128_CBC_SHA256,
+	TLS_DHE_DSS_WITH_ARIA_256_CBC_SHA384,
+	TLS_DHE_RSA_WITH_ARIA_128_CBC_SHA256,
+	TLS_DHE_RSA_WITH_ARIA_256_CBC_SHA384,
+	TLS_DHE_RSA_WITH_ARIA_128_GCM_SHA256,
+	TLS_DHE_RSA_WITH_ARIA_256_GCM_SHA384,
+	TLS_DHE_DSS_WITH_ARIA_128_GCM_SHA256,
+	TLS_DHE_DSS_WITH_ARIA_256_GCM_SHA384,
+	TLS_DHE_PSK_WITH_ARIA_128_CBC_SHA256,
+	TLS_DHE_PSK_WITH_ARIA_256_CBC_SHA384,
+	TLS_DHE_PSK_WITH_ARIA_128_GCM_SHA256,
+	TLS_DHE_PSK_WITH_ARIA_256_GCM_SHA384,
+	TLS_DHE_RSA_WITH_CAMELLIA_128_GCM_SHA256,
+	TLS_DHE_RSA_WITH_CAMELLIA_256_GCM_SHA384,
+	TLS_DHE_DSS_WITH_CAMELLIA_128_GCM_SHA256,
+	TLS_DHE_DSS_WITH_CAMELLIA_256_GCM_SHA384,
+	TLS_DHE_PSK_WITH_CAMELLIA_128_GCM_SHA256,
+	TLS_DHE_PSK_WITH_CAMELLIA_256_GCM_SHA384,
+	TLS_DHE_PSK_WITH_CAMELLIA_128_CBC_SHA256,
+	TLS_DHE_PSK_WITH_CAMELLIA_256_CBC_SHA384,
+	TLS_DHE_RSA_WITH_AES_128_CCM,
+	TLS_DHE_RSA_WITH_AES_256_CCM,
+	TLS_DHE_RSA_WITH_AES_128_CCM_8,
+	TLS_DHE_RSA_WITH_AES_256_CCM_8,
+	TLS_DHE_PSK_WITH_AES_128_CCM,
+	TLS_DHE_PSK_WITH_AES_256_CCM,
+	TLS_PSK_DHE_WITH_AES_128_CCM_8,
+	TLS_PSK_DHE_WITH_AES_256_CCM_8,
+	TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256,
+	# DH-anon suites
+	TLS_DH_ANON_EXPORT_WITH_RC4_40_MD5,
+	TLS_DH_ANON_WITH_RC4_128_MD5,
+	TLS_DH_ANON_EXPORT_WITH_DES40_CBC_SHA,
+	TLS_DH_ANON_WITH_DES_CBC_SHA,
+	TLS_DH_ANON_WITH_3DES_EDE_CBC_SHA,
+	TLS_DH_ANON_WITH_AES_128_CBC_SHA,
+	TLS_DH_ANON_WITH_AES_256_CBC_SHA,
+	TLS_DH_ANON_WITH_CAMELLIA_128_CBC_SHA,
+	TLS_DH_ANON_WITH_AES_128_CBC_SHA256,
+	TLS_DH_ANON_WITH_AES_256_CBC_SHA256,
+	TLS_DH_ANON_WITH_CAMELLIA_256_CBC_SHA,
+	TLS_DH_ANON_WITH_SEED_CBC_SHA,
+	TLS_DH_ANON_WITH_AES_128_GCM_SHA256,
+	TLS_DH_ANON_WITH_AES_256_GCM_SHA384,
+	TLS_DH_ANON_WITH_CAMELLIA_128_CBC_SHA256,
+	TLS_DH_ANON_WITH_CAMELLIA_256_CBC_SHA256,
+	TLS_DH_ANON_WITH_ARIA_128_CBC_SHA256,
+	TLS_DH_ANON_WITH_ARIA_256_CBC_SHA384,
+	TLS_DH_ANON_WITH_ARIA_128_GCM_SHA256,
+	TLS_DH_ANON_WITH_ARIA_256_GCM_SHA384,
+	TLS_DH_ANON_WITH_CAMELLIA_128_GCM_SHA256,
+	TLS_DH_ANON_WITH_CAMELLIA_256_GCM_SHA384
+	# DH non-anon suites do not send a ServerKeyExchange
+		-> dh_server_key_exchange : DhServerKeyExchange(rec);
+
+	default
+		-> key : bytestring &restofdata &transient;
+};
+
+# For the moment, we really only are interested in the curve name. If it
+# is not set (if the server sends explicit parameters), we do not bother.
+# We also do not parse the actual signature data following the named curve.
+type EcServerKeyExchange(rec: HandshakeRecord) = record {
+	curve_type: uint8;
+	curve: uint16; # only if curve_type = 3 (NAMED_CURVE)
+	data: bytestring &restofdata &transient;
+};
+
+# For both, dh_anon and dhe the ServerKeyExchange starts with a ServerDHParams
+# structure. After that, they start to differ, but we do not care about that.
+type DhServerKeyExchange(rec: HandshakeRecord) = record {
+	dh_p_length: uint16;
+	dh_p: bytestring &length=dh_p_length;
+	dh_g_length: uint16;
+	dh_g: bytestring &length=dh_g_length;
+	dh_Ys_length: uint16;
+	dh_Ys: bytestring &length=dh_Ys_length;
+	data: bytestring &restofdata &transient;
+};
+
+
+######################################################################
+# V3 Certificate Request (7.4.4.)
+######################################################################
+
+# For now, ignore Certificate Request Details; just eat up message.
+type CertificateRequest(rec: HandshakeRecord) = record {
+	cont : bytestring &restofdata &transient;
+};
+
+
+######################################################################
+# V3 Server Hello Done (7.4.5.)
+######################################################################
+
+# Server Hello Done is empty
+type ServerHelloDone(rec: HandshakeRecord) = empty;
+
+
+######################################################################
+# V3 Client Certificate (7.4.6.)
+######################################################################
+
+# Client Certificate is identical to Server Certificate;
+# no further definition here
+
+
+######################################################################
+# V3 Client Key Exchange Message (7.4.7.)
+######################################################################
+
+# For now ignore details of ClientKeyExchange (most of it is
+# encrypted anyway); just eat up message.
+type ClientKeyExchange(rec: HandshakeRecord) = record {
+	key : bytestring &restofdata &transient;
+};
+
+
+######################################################################
+# V3 Certificate Verify (7.4.8.)
+######################################################################
+
+# For now, ignore Certificate Verify; just eat up the message.
+type CertificateVerify(rec: HandshakeRecord) = record {
+	cont : bytestring &restofdata &transient;
+};
+
+
+######################################################################
+# V3 Finished (7.4.9.)
+######################################################################
+
+# The finished messages are always sent after encryption is in effect,
+# so we will not be able to read those messages.
+type Finished(rec: HandshakeRecord) = record {
+	cont : bytestring &restofdata &transient;
+};
+
+type SessionTicketHandshake(rec: HandshakeRecord) = record {
+	ticket_lifetime_hint: uint32;
+	data:                 bytestring &restofdata;
+};
+
+######################################################################
+# TLS Extensions
+######################################################################
+
+type SSLExtension(rec: HandshakeRecord) = record {
+	type: uint16;
+	data_len: uint16;
+
+	# Pretty code ahead. Deal with the fact that perhaps extensions are
+	# not really present and we do not want to fail because of that.
+	ext: case type of {
+		EXT_APPLICATION_LAYER_PROTOCOL_NEGOTIATION -> apnl: ApplicationLayerProtocolNegotiationExtension(rec)[] &until($element == 0 || $element != 0);
+		EXT_ELLIPTIC_CURVES -> elliptic_curves: EllipticCurves(rec)[] &until($element == 0 || $element != 0);
+		EXT_EC_POINT_FORMATS -> ec_point_formats: EcPointFormats(rec)[] &until($element == 0 || $element != 0);
+#		EXT_STATUS_REQUEST -> status_request: StatusRequest(rec)[] &until($element == 0 || $element != 0);
+		EXT_SERVER_NAME -> server_name: ServerNameExt(rec)[] &until($element == 0 || $element != 0);
+		default -> data: bytestring &restofdata;
+	};
+} &length=data_len+4 &exportsourcedata;
+
+type ServerNameHostName() = record {
+	length: uint16;
+	host_name: bytestring &length=length;
+};
+
+type ServerName() = record {
+	name_type: uint8; # has to be 0 for host-name
+	name: case name_type of {
+		0 -> host_name: ServerNameHostName;
+		default -> data : bytestring &restofdata &transient; # unknown name
+	};
+};
+
+type ServerNameExt(rec: HandshakeRecord) = record {
+	length: uint16;
+	server_names: ServerName[] &until($input.length() == 0);
+} &length=length+2;
+
+# Do not parse for now. Structure is correct, but only contains asn.1 data that we would not use further.
+#type OcspStatusRequest(rec: HandshakeRecord) = record {
+#	responder_id_list_length: uint16;
+#	responder_id_list: bytestring &length=responder_id_list_length;
+#	request_extensions_length: uint16;
+#	request_extensions: bytestring &length=request_extensions_length;
+#};
+#
+#type StatusRequest(rec: HandshakeRecord) = record {
+#	status_type: uint8; # 1 -> ocsp
+#	req: case status_type of {
+#		1 -> ocsp_status_request: OcspStatusRequest(rec);
+#		default -> data : bytestring &restofdata &transient; # unknown
+#	};
+#};
+
+type EcPointFormats(rec: HandshakeRecord) = record {
+	length: uint8;
+	point_format_list: uint8[length];
+};
+
+type EllipticCurves(rec: HandshakeRecord) = record {
+	length: uint16;
+	elliptic_curve_list: uint16[length/2];
+};
+
+type ProtocolName() = record {
+  length: uint8;
+	name: bytestring &length=length;
+};
+
+type ApplicationLayerProtocolNegotiationExtension(rec: HandshakeRecord) = record {
+	length: uint16;
+	protocol_name_list: ProtocolName[] &until($input.length() == 0);
+} &length=length+2;
+
+refine connection Handshake_Conn += {
+
+	%member{
+		uint32 chosen_cipher_;
+	%}
+
+	%init{
+		chosen_cipher_ = NO_CHOSEN_CIPHER;
+	%}
+
+	function chosen_cipher() : int %{ return chosen_cipher_; %}
+
+	function set_cipher(cipher: uint32) : bool
+		%{
+		chosen_cipher_ = cipher;
+		return true;
+		%}
+};
+
+

src/analyzer/protocol/ssl/tls-handshake.pac

@@ -0,0 +1,23 @@
+# Binpac analyzer just for the TLS handshake protocol and nothing else
+
+%include binpac.pac
+%include bro.pac
+
+analyzer TLSHandshake withcontext {
+	connection: Handshake_Conn;
+	flow:       Handshake_Flow;
+};
+
+connection Handshake_Conn(bro_analyzer: BroAnalyzer) {
+	upflow = Handshake_Flow(true);
+	downflow = Handshake_Flow(false);
+};
+
+%include ssl-defs.pac
+%include tls-handshake-protocol.pac
+
+flow Handshake_Flow(is_orig: bool) {
+	flowunit = HandshakePDU(is_orig) withcontext(connection, this);
+}
+
+%include tls-handshake-analyzer.pac

src/analyzer/protocol/tcp/TCP.cc

@@ -367,6 +367,41 @@ void TCP_Analyzer::Done()
 	finished = 1;
 	}
 
+analyzer::Analyzer* TCP_Analyzer::FindChild(ID arg_id)
+	{
+	analyzer::Analyzer* child = analyzer::TransportLayerAnalyzer::FindChild(arg_id);
+
+	if ( child )
+		return child;
+
+	LOOP_OVER_GIVEN_CHILDREN(i, packet_children)
+		{
+		analyzer::Analyzer* child = (*i)->FindChild(arg_id);
+		if ( child )
+			return child;
+		}
+
+	return 0;
+	}
+
+analyzer::Analyzer* TCP_Analyzer::FindChild(Tag arg_tag)
+	{
+	analyzer::Analyzer* child = analyzer::TransportLayerAnalyzer::FindChild(arg_tag);
+
+	if ( child )
+		return child;
+
+	LOOP_OVER_GIVEN_CHILDREN(i, packet_children)
+		{
+		analyzer::Analyzer* child = (*i)->FindChild(arg_tag);
+		if ( child )
+			return child;
+		}
+
+	return 0;
+	}
+
+
 void TCP_Analyzer::EnableReassembly()
 	{
 	SetReassembler(new TCP_Reassembler(this, this,
@@ -1764,6 +1799,15 @@ bool TCP_Analyzer::HadGap(bool is_orig) const
 	return endp && endp->HadGap();
 	}
 
+void TCP_Analyzer::AddChildPacketAnalyzer(analyzer::Analyzer* a)
+	{
+	DBG_LOG(DBG_ANALYZER, "%s added packet child %s",
+			this->GetAnalyzerName(), a->GetAnalyzerName());
+
+	packet_children.push_back(a);
+	a->SetParent(this);
+	}
+
 int TCP_Analyzer::DataPending(TCP_Endpoint* closing_endp)
 	{
 	if ( Skipping() )

src/analyzer/protocol/tcp/TCP.h

@@ -47,8 +47,10 @@ public:
 
 	// Add a child analyzer that will always get the packets,
 	// independently of whether we do any reassembly.
-	void AddChildPacketAnalyzer(analyzer::Analyzer* a)
-		{ packet_children.push_back(a); a->SetParent(this); }
+	void AddChildPacketAnalyzer(analyzer::Analyzer* a);
+
+	virtual Analyzer* FindChild(ID id);
+	virtual Analyzer* FindChild(Tag tag);
 
 	// True if the connection has closed in some sense, false otherwise.
 	int IsClosed() const	{ return orig->did_close || resp->did_close; }

src/bro.bif

@@ -1675,6 +1675,7 @@ function net_stats%(%): NetStats
 	unsigned int recv = 0;
 	unsigned int drop = 0;
 	unsigned int link = 0;
+	unsigned int bytes_recv = 0;
 
 	const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs());
 
@@ -1688,12 +1689,14 @@ function net_stats%(%): NetStats
 		recv += stat.received;
 		drop += stat.dropped;
 		link += stat.link;
+		bytes_recv += stat.bytes_received;
 		}
 
 	RecordVal* ns = new RecordVal(net_stats);
 	ns->Assign(0, new Val(recv, TYPE_COUNT));
 	ns->Assign(1, new Val(drop, TYPE_COUNT));
 	ns->Assign(2, new Val(link, TYPE_COUNT));
+	ns->Assign(3, new Val(bytes_recv, TYPE_COUNT));
 
 	return ns;
 	%}
@@ -2244,7 +2247,7 @@ function to_count%(str: string%): count
 	const char* s = str->CheckString();
 	char* end_s;
 
-	uint64 u = (uint64) strtoll(s, &end_s, 10);
+	uint64 u = (uint64) strtoull(s, &end_s, 10);
 
 	if ( s[0] == '\0' || end_s[0] != '\0' )
     	{

src/broker-dummy/CMakeLists.txt

@@ -0,0 +1,13 @@
+# Placeholder for Broker-based communication functionality, not enabled
+# by default.  This helps satisfy coverage unit tests pass regardless of
+# whether Broker is enabled or not.
+
+include(BroSubdir)
+
+bif_target(comm.bif)
+bif_target(data.bif)
+bif_target(messaging.bif)
+bif_target(store.bif)
+
+bro_add_subdir_library(broker_dummy ${BIF_OUTPUT_CC})
+add_dependencies(bro_broker_dummy generate_outputs)

src/broker-dummy/comm.bif

@@ -0,0 +1,3 @@
+
+##! Placeholder for Broker-based communication functionality, not enabled
+##! by default.