Merge branch 'master' into topic/seth/fix-compiler-warnings

CHANGES

@@ -1,8 +1,50 @@
-@(#) $Id: CHANGES 7076 2010-09-13 02:42:27Z vern $
+1.6-dev.21 Wed Jan 19 17:36:02 PST 2011
 
--+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    * Added 4 new BiFs and a new record type for testing the entropy
+      of strings. (Seth Hall)
 
-1.6-dev.4 Sat Jan 15 14:14:21 PST 2011
+        find_entropy(data: string): entropy_test_result
+                This is a one shot function that accepts a string
+                and returns the result of the entropy calculations.
+
+        entropy_test_init(index: any): bool
+                This and the next two functions are for calculating
+                entropy piece-wise. It only needs an index which can
+                be any type of variable. It needs to be something
+                that uniquely identifies the data stream that is
+                currently having it's entropy calculated.
+
+        entropy_test_add(index: any, data: string): bool
+                This function is used to add data into the entropy
+                calculation. It takes the index used in the function
+                above and the data that you are adding and returns
+                true if everything seemed to work, false otherwise.
+
+        entropy_test_finish(index: any): entropy_test_result
+                Calling this function indicates that all of the
+                desired data has been inserted into the
+                entropy_test_add function and the entropy should be
+                calculated. This function *must* be called in order
+                to clean up an internal state tracking variable.  If
+                this is never called on an index, it will result in
+                a memory leak.
+
+    The entropy_test_result values have several measures of the
+    entropy, but a good one to work with is the "entropy" attribute.
+    It's a double and as the value approaches 8.0 it can be
+    considered more and more random.  For example, a value of 7.832
+    would be quite random but a value of 4.671 is not very random.
+
+1.6-dev.20 Wed Jan 19 17:30:11 PST 2011
+
+    * BRO_DNS_FAKE is now listed in the --help output. (Seth Hall)
+
+1.6-dev.18 Wed Jan 19 16:37:13 PST 2011
+
+    * Removing unnecessary expire timer from http_sessions. (Gregor
+      Maier)
+
+1.6-dev.16 Sat Jan 15 14:14:21 PST 2011
 
 - Updates to the build system (Jonathan Siwek)
 

CMakeLists.txt

@@ -57,6 +57,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_COMPILE_FLAGS}")
 ########################################################################
 ## Dependency Configuration
 
+include(MacDependencyPaths)
 include(FindRequiredPackage)
 
 # Check cache value first to avoid displaying "Found sed" messages everytime

VERSION

@@ -1 +1 @@
-1.6-dev.4
+1.6-dev.21

aux/binpac

@@ -1 +1 @@
-Subproject commit 7bbd3b14c02321ff2a63d7267e9ae022bda4f5bc
+Subproject commit 898cfd5ddc8cd356e4052c0bd699e51812a91e98

aux/bro-aux

@@ -1 +1 @@
-Subproject commit 62b2f1bdd52d355fb0384c3f0e8f1879c7c17724
+Subproject commit d741ee2ebd6576d9329218bfb53941b4de5375b6

aux/broccoli

@@ -1 +1 @@
-Subproject commit cd02839cf07d4db34f30bbdbb41711bc8b06b1ac
+Subproject commit c745d747ec65f608bead605fc26f84ca44be21c9

aux/broctl

@@ -1 +1 @@
-Subproject commit 798ea05965318333af4df8fe9f0296eb3a669f4f
+Subproject commit 41336359365238036fd63f8bf8d2624da71c200b

cmake/ConfigurePackaging.cmake

@@ -140,14 +140,12 @@ macro(SetPackageMetadata)
     set(CPACK_RPM_PACKAGE_LICENSE "BSD")
 endmacro(SetPackageMetadata)
 
-# Sets pre and post install scripts for PackageMaker and RPM packages.
+# Sets pre and post install scripts for PackageMaker packages.
 # The main functionality that such scripts offer is a way to make backups
 # of "configuration" files that a user may have modified.
-# A better way to prevent an RPM from not overwriting config files is
+# Note that RPMs already have a robust mechanism for dealing with
-# with the %config(noreplace) .spec attribute, but CPack does not have any
+# user-modified files, so we do not need this additional functionality
-# good hooks into using that yet, so we re-use the pre/post install scripts
+macro(SetPackageInstallScripts VERSION)
-# See also: http://public.kitware.com/Bug/view.php?id=10294
-macro(SetPackageInstallScripts)
 
     if (INSTALLED_CONFIG_FILES)
         # Remove duplicates from the list of installed config files
@@ -160,6 +158,14 @@ macro(SetPackageInstallScripts)
         set(INSTALLED_CONFIG_FILES "${_tmp}" CACHE STRING "" FORCE)
     endif ()
 
+    if (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+        # Leaving the set of installed config files empty will just
+        # bypass the logic in the pre/post install scripts and let
+        # the RPM do their own thing (regarding backups, etc.)
+        # when upgrading packages.
+        set (INSTALLED_CONFIG_FILES "")
+    endif ()
+
     if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/cmake/package_preinstall.sh.in)
         configure_file(
             ${CMAKE_CURRENT_SOURCE_DIR}/cmake/package_preinstall.sh.in
@@ -189,7 +195,7 @@ macro(ConfigurePackaging _version)
     SetPackageGenerators()
     SetPackageFileName(${_version})
     SetPackageMetadata()
-    SetPackageInstallScripts()
+    SetPackageInstallScripts(${_version})
 
     set(CPACK_SET_DESTDIR true)
     set(CPACK_PACKAGING_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX})

cmake/MAC_PACKAGE_INTRO

@@ -15,10 +15,6 @@ destination as the one that contains the root filesystem.
 
 If you have existing configuration files that are modified or
 otherwise different from the version included in the package,
-this installer will attempt to prevent clobbering them by
+this installer will attempt to prevent overwirting them,
-backing them up like:
-
-    @CMAKE_INSTALL_PREFIX@/etc/<file>.<nextAvailableNumber>
-
 but its also advisable to make your own backups of important
 files before proceeding.

cmake/MacDependencyPaths.cmake

@@ -0,0 +1,10 @@
+if (NOT _MAC_DEPENDENCY_PATHS)
+set(_MAC_DEPENDENCY_PATHS)
+    # As of CMake 2.8.3, Fink and MacPorts search paths are appended to the
+    # default search prefix paths, but the nicer thing would be if they are
+    # prepended to the default, so that is fixed here.
+    if (APPLE) 
+        list(INSERT CMAKE_SYSTEM_PREFIX_PATH 0 /opt/local) # MacPorts
+        list(INSERT CMAKE_SYSTEM_PREFIX_PATH 0 /sw)        # Fink
+    endif ()
+endif ()

cmake/package_postupgrade.sh.in

@@ -3,8 +3,9 @@
 # This script is meant to be used by binary packages post-installation.
 # Variables between @ symbols are replaced by CMake at configure time.
 
-backupDesc="# Backup made by install of @CMAKE_PROJECT_NAME@ version @VERSION@"
 backupNamesFile=/tmp/bro_install_backups
+version=@VERSION@
+newFiles=""
 
 # check whether it's safe to remove backup configuration files that
 # the most recent package install created
@@ -13,22 +14,25 @@ if [ -e ${backupNamesFile} ]; then
     backupFileList=`cat ${backupNamesFile}`
 
     for backupFile in ${backupFileList}; do
-        origFile=`echo ${backupFile} | sed 's/\(.*\)\..*/\1/'`
+        origFileName=`echo ${backupFile} | sed 's/\(.*\)\..*/\1/'`
 
-        diff ${origFile} ${backupFile} > /dev/null 2>&1
+        diff ${origFileName} ${backupFile} > /dev/null 2>&1
 
         if [ $? -eq 0 ]; then
             # if the installed version and the backup version don't differ
             # then we can remove the backup version
             rm ${backupFile}
         else
-            # keep the backup, prepend text explaining what created it
+            # The backup file differs from the newly installed version,
-            tmpfile=/tmp/bro_install_tmp$$
+            # since we can't tell if the backup version has been modified
-            echo ${backupDesc} > ${tmpfile}
+            # by the user, we should restore it to its original location
-            echo "" >> ${tmpfile}
+            # and rename the new version appropriately.
-            cat ${backupFile} >> ${tmpfile}
+
-            cp ${tmpfile} ${backupFile}
+            newFileName=${origFileName}.${version}
-            rm ${tmpfile}
+            newFiles="${newFiles}\n${newFileName}"
+
+            mv ${origFileName} ${newFileName}
+            mv ${backupFile} ${origFileName}
         fi
 
     done
@@ -36,6 +40,16 @@ if [ -e ${backupNamesFile} ]; then
     rm ${backupNamesFile}
 fi
 
+if [ -n "${newFiles}" ]; then
+# Use some apple script to display a message to user
+/usr/bin/osascript << EOF
+    tell application "System Events"
+        activate
+        display alert "Existing configuration files differ from the ones that would be installed by this package.  To avoid overwriting configuration which you may have modified, the following new config files have been installed:\n${newFiles}\n\nIf you have previously modified configuration files, please make sure that they are still compatible, else you should update your config files to the new versions."
+    end tell
+EOF
+fi
+
 # make sure that world-writeable dirs have the sticky bit set
 # so that unprivileged can't rename/remove files within
 

policy/bro.init

@@ -264,6 +264,14 @@ type geo_location: record {
 	longitude: double;
 };
 
+type entropy_test_result: record {
+	entropy: double;
+	chi_square: double;
+	mean: double;
+	monte_carlo_pi: double;
+	serial_correlation: double;
+};
+
 # Prototypes of Bro built-in functions.
 @load strings.bif.bro
 @load bro.bif.bro

policy/heavy.http.bro

@@ -1,3 +0,0 @@
-# $Id: heavy.http.bro 4723 2007-08-07 18:14:35Z vern $
-
-redef http_sessions &write_expire = 5 hrs;

policy/http.bro

@@ -79,17 +79,8 @@ type http_session_info: record {
 
 const http_log = open_log_file("http") &redef;
 
-# Called when an HTTP session times out.
-global expire_http_session:
-	function(t: table[conn_id] of http_session_info, id: conn_id)
-		: interval;
-
 export {
-	# Indexed by conn_id.
+	global http_sessions: table[conn_id] of http_session_info;
-	# (Exported so that we can define a timeout on it.)
-	global http_sessions: table[conn_id] of http_session_info
-			&expire_func = expire_http_session
-			&read_expire = 15 min;
 }
 
 global http_session_id = 0;
@@ -202,30 +193,6 @@ event connection_state_remove(c: connection)
 	delete http_sessions[c$id];
 	}
 
-function expire_http_session(t: table[conn_id] of http_session_info,
-				id: conn_id): interval
-	{
-	### FIXME: not really clear that we need this function at all ...
-	#
-	# One would think that connection_state_remove() already takes care
-	# of everything.  However, without this expire-handler, some requests
-	# don't show up with the test-suite (but haven't reproduced with
-	# smaller traces) - Robin.
-
-	local s = http_sessions[id];
-	finish_stream(id, s$id, s$request_stream);
-	return 0 sec;
-	}
-
-# event connection_timeout(c: connection)
-# 	{
-# 	if ( ! maintain_http_sessions )
-# 		{
-# 		local id = c$id;
-# 		if ( [id$orig_h, id$resp_h] in http_sessions )
-# 			delete http_sessions[id$orig_h, id$resp_h];
-# 		}
-# 	}
 
 # event http_stats(c: connection, stats: http_stats_rec)
 # 	{

policy/portmapper.bro

@@ -4,6 +4,7 @@
 @load hot
 @load conn
 @load weird
+@load scan
 
 module Portmapper;
 

src/CMakeLists.txt

@@ -320,6 +320,7 @@ set(bro_SRCS
     PrefixTable.cc
     PriorityQueue.cc
     Queue.cc
+    RandTest.cc
     RE.cc
     RPC.cc
     Reassem.cc

src/ConnCompressor.cc

@@ -391,26 +391,6 @@ Connection* ConnCompressor::NextFromOrig(PendingConn* pending, double t,
 			{
 			if ( (tp->th_flags & TH_ACK) && ! pending->ACK )
 				Weird(pending, t, "repeated_SYN_with_ack");
-			else
-				{
-				// We adjust the start-time. Unfortunately
-				// this means that we have to create a new
-				// PendingConn as all of them need to be
-				// monotonically increasing in time. This
-				// leads to some inconsistencies with TCP.cc,
-				// as by doing this we basically restart our
-				// attempt_timer.
-
-				pending = MoveState(t, pending);
-
-				// Removing is necessary because the key
-				// will be destroyed at some point.
-				conns.Remove(&pending->key, sizeof(pending->key),
-						pending->hash, true);
-				conns.Dictionary::Insert(&pending->key,
-					sizeof(pending->key), pending->hash,
-					MakeMapPtr(pending), 0);
-				}
 			}
 
 		else
@@ -715,17 +695,6 @@ uint8 ConnCompressor::MakeFlags(const PendingConn* c) const
 	return tcp_flags;
 	}
 
-ConnCompressor::PendingConn* ConnCompressor::MoveState(double time,
-							PendingConn* c)
-	{
-	PendingConn* nc = MakeNewState(time);
-	memcpy(nc, c, sizeof(PendingConn));
-	c->invalid = 1;
-	nc->time = time;
-	++sizes.pending_in_mem;
-	return nc;
-	}
-
 ConnCompressor::PendingConn* ConnCompressor::MakeNewState(double t)
 	{
 	// See if there is enough space in the current block.

src/ConnCompressor.h

@@ -138,10 +138,6 @@ private:
 	// Fakes a TCP packet based on the available information.
 	const IP_Hdr* PendingConnToPacket(const PendingConn* c);
 
-	// For changing the timestamp of PendingConn - allocates a new one,
-	// sets the given time, and copies all other data from old.
-	PendingConn* MoveState(double time, PendingConn* old);
-
 	// Construct a TCP-flags byte.
 	uint8 MakeFlags(const PendingConn* c) const;
 

src/Expr.cc

@@ -2611,7 +2611,6 @@ Val* AssignExpr::Eval(Frame* f) const
 	if ( v )
 		{
 		op1->Assign(f, v);
-		//### op1->SetAttribs();
 		return val ? val->Ref() : v->Ref();
 		}
 	else
@@ -3062,13 +3061,6 @@ FieldExpr::FieldExpr(Expr* arg_op, const char* arg_field_name)
 	if ( IsError() )
 		return;
 
-	if ( streq(arg_field_name, "attr") )
-		{
-		field = -1;
-		SetType(op->Type()->AttributesType()->Ref());
-		return;
-		}
-
 	if ( ! IsRecord(op->Type()->Tag()) )
 		ExprError("not a record");
 	else
@@ -3106,12 +3098,7 @@ void FieldExpr::Assign(Frame* f, Val* v, Opcode opcode)
 		return;
 
 	if ( field < 0 )
-		{
+		ExprError("no such field in record");
-		Val* lhs = op->Eval(f);
-		lhs->SetAttribs(v->AsRecordVal());
-		Unref(lhs);
-		return;
-		}
 
 	Val* op_v = op->Eval(f);
 	if ( op_v )
@@ -3124,9 +3111,6 @@ void FieldExpr::Assign(Frame* f, Val* v, Opcode opcode)
 
 Val* FieldExpr::Fold(Val* v) const
 	{
-	if ( field < 0 )
-		return v->GetAttribs(true)->Ref();
-
 	Val* result = v->AsRecordVal()->Lookup(field);
 	if ( result )
 		return result->Ref();
@@ -3179,24 +3163,20 @@ bool FieldExpr::DoUnserialize(UnserialInfo* info)
 	return td != 0;
 	}
 
-HasFieldExpr::HasFieldExpr(Expr* arg_op, const char* arg_field_name,
+HasFieldExpr::HasFieldExpr(Expr* arg_op, const char* arg_field_name)
-			   bool arg_is_attr)
 : UnaryExpr(EXPR_HAS_FIELD, arg_op)
 	{
 	field_name = arg_field_name;
-	is_attr = arg_is_attr;
 	field = 0;
 
 	if ( IsError() )
 		return;
 
-	if ( ! is_attr && ! IsRecord(op->Type()->Tag()) )
+	if ( ! IsRecord(op->Type()->Tag()) )
 		ExprError("not a record");
 	else
 		{
-		RecordType* rt = is_attr ?
+		RecordType* rt = op->Type()->AsRecordType();
-					op->Type()->AttributesType() :
-					op->Type()->AsRecordType();
 		field = rt->FieldOffset(field_name);
 
 		if ( field < 0 )
@@ -3215,10 +3195,7 @@ Val* HasFieldExpr::Fold(Val* v) const
 	{
 	RecordVal* rec_to_look_at;
 
-	if ( is_attr )
+	rec_to_look_at = v->AsRecordVal();
-		rec_to_look_at = v->GetAttribs(false);
-	else
-		rec_to_look_at = v->AsRecordVal();
 
 	if ( ! rec_to_look_at )
 		return new Val(0, TYPE_BOOL);
@@ -3235,12 +3212,7 @@ void HasFieldExpr::ExprDescribe(ODesc* d) const
 	op->Describe(d);
 
 	if ( d->IsReadable() )
-		{
+		d->Add("?$");
-		if ( is_attr )
-			d->Add("?$$");
-		else
-			d->Add("?$");
-		}
 
 	if ( IsError() )
 		d->Add("<error>");
@@ -3255,13 +3227,17 @@ IMPLEMENT_SERIAL(HasFieldExpr, SER_HAS_FIELD_EXPR);
 bool HasFieldExpr::DoSerialize(SerialInfo* info) const
 	{
 	DO_SERIALIZE(SER_HAS_FIELD_EXPR, UnaryExpr);
-	return SERIALIZE(is_attr) && SERIALIZE(field_name) && SERIALIZE(field);
+
+	// Serialize the former "bool is_attr" first for backwards compatibility.
+	return SERIALIZE(false) && SERIALIZE(field_name) && SERIALIZE(field);
 	}
 
 bool HasFieldExpr::DoUnserialize(UnserialInfo* info)
 	{
 	DO_UNSERIALIZE(UnaryExpr);
-	return UNSERIALIZE(&is_attr) && UNSERIALIZE_STR(&field_name, 0) && UNSERIALIZE(&field);
+	// Unserialize the former "bool is_attr" first for backwards compatibility.
+	bool not_used;
+	return UNSERIALIZE(&not_used) && UNSERIALIZE_STR(&field_name, 0) && UNSERIALIZE(&field);
 	}
 
 RecordConstructorExpr::RecordConstructorExpr(ListExpr* constructor_list)
@@ -3507,8 +3483,6 @@ Val* SetConstructorExpr::Eval(Frame* f) const
 		aggr->Assign(element, 0);
 		}
 
-	aggr->AsTableVal()->SetAttrs(attrs);
-
 	return aggr;
 	}
 

src/Expr.h

@@ -709,7 +709,7 @@ protected:
 // "rec?$$attrname" is true if the attribute attrname is not nil.
 class HasFieldExpr : public UnaryExpr {
 public:
-	HasFieldExpr(Expr* op, const char* field_name, bool is_attr);
+	HasFieldExpr(Expr* op, const char* field_name);
 	~HasFieldExpr();
 
 protected:

src/File.cc

@@ -195,10 +195,9 @@ bool BroFile::Open(FILE* file)
 	InstallRotateTimer();
 
 	if ( ! f )
-		{
 		f = fopen(name, access);
-		SetBuf(buffered);
+
-		}
+	SetBuf(buffered);
 
 	if ( f )
 		{

src/NetVar.cc

@@ -126,6 +126,8 @@ TableType* smb_negotiate;
 
 RecordType* geo_location;
 
+RecordType* entropy_test_result;
+
 TableType* dhcp_router_list;
 RecordType* dhcp_msg;
 
@@ -460,6 +462,8 @@ void init_net_var()
 
 	geo_location = internal_type("geo_location")->AsRecordType();
 
+	entropy_test_result = internal_type("entropy_test_result")->AsRecordType();
+
 	dhcp_router_list = internal_type("dhcp_router_list")->AsTableType();
 	dhcp_msg = internal_type("dhcp_msg")->AsRecordType();
 

src/NetVar.h

@@ -133,6 +133,8 @@ extern TableType* smb_negotiate;
 
 extern RecordType* geo_location;
 
+extern RecordType* entropy_test_result;
+
 extern TableType* dhcp_router_list;
 extern RecordType* dhcp_msg;
 

src/PersistenceSerializer.cc

@@ -348,9 +348,6 @@ bool PersistenceSerializer::RunSerialization(SerialStatus* status)
 			status->conn_cookie = status->conns->InitForIteration();
 			status->conns->MakeRobustCookie(status->conn_cookie);
 			}
-
-		if ( status->info.may_suspend )
-			bro_logger->Log("Starting incremental serialization...");
 		}
 
 	else if ( cont->ChildSuspended() )
@@ -480,9 +477,6 @@ bool PersistenceSerializer::RunSerialization(SerialStatus* status)
 			}
 		}
 
-	if ( status->info.may_suspend )
-		bro_logger->Log("Finished incremental serialization.");
-
 	delete status;
 	return ret;
 	}

src/RandTest.cc

@@ -0,0 +1,132 @@
+/*
+	Apply various randomness tests to a stream of bytes
+
+		by John Walker  --  September 1996
+			http://www.fourmilab.ch/random
+
+		This software is in the public domain. Permission to use, copy, modify,
+		and distribute this software and its documentation for any purpose and
+		without fee is hereby granted, without any conditions or restrictions.
+		This software is provided “as is” without express or implied warranty.
+
+	Modified for Bro by Seth Hall - July 2010
+*/
+
+#include <RandTest.h>
+
+RandTest::RandTest()
+	{
+	totalc = 0;
+	mp = 0;
+	sccfirst = 1;
+	inmont = mcount = 0;
+	cexp = montex = montey = montepi = sccu0 = scclast = scct1 = scct2 = scct3 = 0.0;
+
+	for (int i = 0; i < 256; i++)
+		{
+		ccount[i] = 0;
+		}
+	}
+
+void RandTest::add(void *buf, int bufl)
+	{
+	unsigned char *bp = (unsigned char*)buf;
+	int oc;
+
+	while (bufl-- > 0)
+		{
+		oc = *bp++;
+		ccount[oc]++;   /* Update counter for this bin */
+		totalc++;
+
+		/* Update inside / outside circle counts for Monte Carlo
+ 		   computation of PI */
+		monte[mp++] = oc;  /* Save character for Monte Carlo */
+		if (mp >= RT_MONTEN)  /* Calculate every RT_MONTEN character */
+			{
+			mp = 0;
+			mcount++;
+			montex = 0;
+			montey = 0;
+			for (int mj=0; mj < RT_MONTEN/2; mj++)
+				{
+				montex = (montex * 256.0) + monte[mj];
+				montey = (montey * 256.0) + monte[(RT_MONTEN / 2) + mj];
+				}
+			if (montex*montex + montey*montey <= RT_INCIRC)
+				{
+				inmont++;
+				}
+			}
+
+		/* Update calculation of serial correlation coefficient */
+		if (sccfirst)
+			{
+			sccfirst = 0;
+			scclast = 0;
+			sccu0 = oc;
+			}
+		else
+			{
+			scct1 = scct1 + scclast * oc;
+			}
+
+		scct2 = scct2 + oc;
+		scct3 = scct3 + (oc * oc);
+		scclast = oc;
+		oc <<= 1;
+		}
+	}
+
+void RandTest::end(double *r_ent, double *r_chisq,
+                   double *r_mean, double *r_montepicalc, double *r_scc)
+	{
+	int i;
+	double ent, chisq, scc, datasum;
+	ent = 0.0; chisq = 0.0; scc = 0.0; datasum = 0.0;
+	double prob[256];    /* Probabilities per bin for entropy */
+
+	/* Complete calculation of serial correlation coefficient */
+	scct1 = scct1 + scclast * sccu0;
+	scct2 = scct2 * scct2;
+	scc = totalc * scct3 - scct2;
+	if (scc == 0.0)
+	   scc = -100000;
+	else
+	   scc = (totalc * scct1 - scct2) / scc;
+
+	/* Scan bins and calculate probability for each bin and
+	   Chi-Square distribution.  The probability will be reused
+	   in the entropy calculation below.  While we're at it,
+	   we sum of all the data which will be used to compute the
+	   mean. */
+	cexp = totalc / 256.0;  /* Expected count per bin */
+	for (i = 0; i < 256; i++)
+		{
+		double a = ccount[i] - cexp;
+
+		prob[i] = ((double) ccount[i]) / totalc;
+		chisq += (a * a) / cexp;
+		datasum += ((double) i) * ccount[i];
+		}
+
+	/* Calculate entropy */
+	for (i = 0; i < 256; i++)
+		{
+		if (prob[i] > 0.0)
+			{
+			ent += prob[i] * rt_log2(1 / prob[i]);
+			}
+		}
+
+	/* Calculate Monte Carlo value for PI from percentage of hits
+	   within the circle */
+	montepi = 4.0 * (((double) inmont) / mcount);
+
+	/* Return results through arguments */
+	*r_ent = ent;
+	*r_chisq = chisq;
+	*r_mean = datasum / totalc;
+	*r_montepicalc = montepi;
+	*r_scc = scc;
+	}

src/RandTest.h

@@ -0,0 +1,34 @@
+#include <math.h>
+
+#define log2of10 3.32192809488736234787
+/*  RT_LOG2  --  Calculate log to the base 2  */
+static double rt_log2(double x)
+{
+    return log2of10 * log10(x);
+}
+
+#define RT_MONTEN 6  /* Bytes used as Monte Carlo
+                        co-ordinates. This should be no more
+                        bits than the mantissa of your "double"
+                        floating point type. */
+
+// RT_INCIRC = pow(pow(256.0, (double) (RT_MONTEN / 2)) - 1, 2.0);
+#define RT_INCIRC 281474943156225.0
+
+class RandTest {
+	public:
+		RandTest();
+		void add(void *buf, int bufl);
+		void end(double *r_ent, double *r_chisq, double *r_mean,
+		         double *r_montepicalc, double *r_scc);
+
+	private:
+		long ccount[256];  /* Bins to count occurrences of values */
+		long totalc;       /* Total bytes counted */
+		int mp;
+		int sccfirst;
+		unsigned int monte[RT_MONTEN];
+		long inmont, mcount;
+		double cexp, montex, montey, montepi,
+		       sccu0, scclast, scct1, scct2, scct3;
+	};

src/Trigger.cc

@@ -130,11 +130,17 @@ Trigger::Trigger(Expr* arg_cond, Stmt* arg_body, Stmt* arg_timeout_stmts,
 
 	Val* timeout = arg_timeout ? arg_timeout->ExprVal() : 0;
 
+	// Make sure we don't get deleted if somebody calls a method like
+	// Timeout() while evaluating the trigger. 
+	Ref(this);
+
 	if ( ! Eval() && timeout )
 		{
 		timer = new TriggerTimer(timeout->AsInterval(), this);
 		timer_mgr->Add(timer);
 		}
+
+	Unref(this);
 	}
 
 Trigger::~Trigger()

src/Type.cc

@@ -10,20 +10,6 @@
 #include "Scope.h"
 #include "Serializer.h"
 
-RecordType* init_global_attrs();
-
-bool in_global_attr_decl = false;
-RecordType* global_attributes_type = init_global_attrs();
-
-RecordType* init_global_attrs()
-	{
-	in_global_attr_decl = true;
-	RecordType* rt = new RecordType(new type_decl_list);
-	in_global_attr_decl = false;
-	rt->MakeGlobalAttributeType();
-	return rt;
-	}
-
 const char* type_name(TypeTag t)
 	{
 	static char errbuf[512];
@@ -58,7 +44,6 @@ BroType::BroType(TypeTag t, bool arg_base_type)
 	tag = t;
 	is_network_order = 0;
 	base_type = arg_base_type;
-	is_global_attributes_type = false;
 
 	switch ( tag ) {
 	case TYPE_VOID:
@@ -118,28 +103,6 @@ BroType::BroType(TypeTag t, bool arg_base_type)
 		break;
 	}
 
-	// Kind of hacky; we don't want an error while we're defining
-	// the global attrs!
-	if ( in_global_attr_decl )
-		{
-		attributes_type = 0;
-		return;
-		}
-
-	if ( ! global_attributes_type )
-		SetError();
-	else
-		attributes_type = global_attributes_type;
-	}
-
-bool BroType::SetAttributesType(type_decl_list* attr_types)
-	{
-	TypeList* global = new TypeList();
-	global->Append(global_attributes_type);
-
-	attributes_type = refine_type(global, attr_types)->AsRecordType();
-
-	return (attributes_type != 0);
 	}
 
 int BroType::MatchesIndex(ListExpr*& /* index */) const
@@ -241,16 +204,6 @@ BroType* BroType::Unserialize(UnserialInfo* info, TypeTag want)
 		return t2;
 		}
 
-	// For the global_attribute_type, we also return our current instance.
-	if ( t->is_global_attributes_type )
-		{
-		BroType* t2 = global_attributes_type;
-		Unref(t);
-		t2->Ref();
-		assert(t2);
-		return t2;
-		}
-
 	assert(t);
 	return t;
 	}
@@ -267,10 +220,15 @@ bool BroType::DoSerialize(SerialInfo* info) const
 		return false;
 
 	if ( ! (SERIALIZE(is_network_order) && SERIALIZE(base_type) &&
-		SERIALIZE(is_global_attributes_type)) )
+		// Serialize the former "bool is_global_attributes_type" for
+		// backwards compatibility.
+		SERIALIZE(false)) )
 		return false;
 
-	SERIALIZE_OPTIONAL(attributes_type);
+	// Likewise, serialize the former optional "RecordType* attributes_type"
+	// for backwards compatibility.
+	void* null = NULL;
+	SERIALIZE(null);
 
 	info->s->WriteCloseTag("Type");
 
@@ -288,13 +246,19 @@ bool BroType::DoUnserialize(UnserialInfo* info)
 	tag = (TypeTag) c1;
 	internal_tag = (InternalTypeTag) c2;
 
+	bool not_used;
+
 	if ( ! (UNSERIALIZE(&is_network_order) && UNSERIALIZE(&base_type)
-			&& UNSERIALIZE(&is_global_attributes_type)) )
+			// Unerialize the former "bool is_global_attributes_type" for
+			// backwards compatibility.
+			&& UNSERIALIZE(&not_used)) )
 		return 0;
 
-	BroType* type;
+	BroType* not_used_either;
-	UNSERIALIZE_OPTIONAL(type, BroType::Unserialize(info, TYPE_RECORD));
+
-	attributes_type = (RecordType*) type;
+	// Likewise, unserialize the former optional "RecordType*
+	// attributes_type" for backwards compatibility.
+	UNSERIALIZE_OPTIONAL(not_used_either, BroType::Unserialize(info, TYPE_RECORD));
 
 	return true;
 	}
@@ -721,9 +685,6 @@ TypeDecl::TypeDecl(BroType* t, const char* i, attr_list* arg_attrs)
 	type = t;
 	attrs = arg_attrs ? new Attributes(arg_attrs, t) : 0;
 	id = i;
-
-	if ( in_global_attr_decl && ! attrs->FindAttr(ATTR_DEFAULT) )
-		error("global attribute types must have default values");
 	}
 
 TypeDecl::~TypeDecl()

src/Type.h

@@ -60,9 +60,6 @@ class EnumType;
 class Serializer;
 class VectorType;
 
-extern bool in_global_attr_decl;
-extern RecordType* global_attributes_type;
-
 const int DOES_NOT_MATCH_INDEX = 0;
 const int MATCHES_INDEX_SCALAR = 1;
 const int MATCHES_INDEX_VECTOR = 2;
@@ -74,15 +71,6 @@ public:
 	TypeTag Tag() const		{ return tag; }
 	InternalTypeTag InternalType() const	{ return internal_tag; }
 
-	// Type for the attributes (metadata) on this type.
-	RecordType* AttributesType()
-		{
-		if ( ! attributes_type )
-			attributes_type = global_attributes_type;
-		return attributes_type;
-		}
-	bool SetAttributesType(type_decl_list* attr_types);
-
 	// Whether it's stored in network order.
 	int IsNetworkOrder() const	{ return is_network_order; }
 
@@ -211,8 +199,6 @@ public:
 
 	BroType* Ref()		{ ::Ref(this); return this; }
 
-	void MakeGlobalAttributeType()	{ is_global_attributes_type = true; }
-
 	virtual void Describe(ODesc* d) const;
 
 	virtual unsigned MemoryAllocation() const;
@@ -221,7 +207,7 @@ public:
 	static BroType* Unserialize(UnserialInfo* info, TypeTag want = TYPE_ANY);
 
 protected:
-	BroType()	{ attributes_type = 0; }
+	BroType()	{ }
 
 	void SetError();
 
@@ -232,8 +218,6 @@ private:
 	InternalTypeTag internal_tag;
 	bool is_network_order;
 	bool base_type;
-	bool is_global_attributes_type;
-	RecordType* attributes_type;
 };
 
 class TypeList : public BroType {

src/Val.cc

@@ -414,15 +414,6 @@ bool Val::DoUnserialize(UnserialInfo* info)
 	return false;
 	}
 
-RecordVal* Val::GetAttribs(bool instantiate)
-	{
-	if ( ! instantiate || attribs )
-		return attribs;
-
-	attribs = new RecordVal(type->AttributesType());
-	return attribs;
-	}
-
 int Val::IsZero() const
 	{
 	switch ( type->InternalType() ) {

src/Val.h

@@ -178,13 +178,6 @@ public:
 	Val* Ref()			{ ::Ref(this); return this; }
 	virtual Val* Clone() const;
 
-	RecordVal* GetAttribs(bool instantiate);
-	void SetAttribs(RecordVal* arg_attribs)
-		{
-		Unref((Val*) attribs);
-		attribs = arg_attribs;
-		}
-
 	int IsZero() const;
 	int IsOne() const;
 

src/bro.bif

@@ -1725,7 +1725,7 @@ function md5_hmac%(...%): string
 %%{
 static map<BroString, md5_state_s> md5_states;
 
-BroString* convert_md5_index_to_string(Val* index)
+BroString* convert_index_to_string(Val* index)
 	{
 	ODesc d;
 	index->Describe(&d);
@@ -1735,7 +1735,7 @@ BroString* convert_md5_index_to_string(Val* index)
 
 function md5_hash_init%(index: any%): bool
 	%{
-	BroString* s = convert_md5_index_to_string(index);
+	BroString* s = convert_index_to_string(index);
 	int status = 0;
 
 	if ( md5_states.count(*s) < 1 )
@@ -1752,7 +1752,7 @@ function md5_hash_init%(index: any%): bool
 
 function md5_hash_update%(index: any, data: string%): bool
 	%{
-	BroString* s = convert_md5_index_to_string(index);
+	BroString* s = convert_index_to_string(index);
 	int status = 0;
 
 	if ( md5_states.count(*s) > 0 )
@@ -1767,7 +1767,7 @@ function md5_hash_update%(index: any, data: string%): bool
 
 function md5_hash_finish%(index: any%): string
 	%{
-	BroString* s = convert_md5_index_to_string(index);
+	BroString* s = convert_index_to_string(index);
 	StringVal* printable_digest;
 
 	if ( md5_states.count(*s) > 0 )
@@ -1848,15 +1848,15 @@ function uuid_to_string%(uuid: string%): string
 	%}
 
 
-# The following functions are attempts to convert strings into
+# The following functions convert strings into patterns at run-time. As the
-# patterns at run-time. These attempts were later *abandoned* because
+# computed NFAs and DFAs cannot be cleanly deallocated (at least for now),
-# NFA and DFA cannot be cleanly deallocated.
+# they can only be used at initialization time.
 
 function merge_pattern%(p1: pattern, p2: pattern%): pattern
 	%{
-	if ( reading_live )
+	if ( bro_start_network_time != 0.0 )
 		{
-		builtin_run_time("should not call merge_pattern while reading live traffic");
+		builtin_run_time("merge_pattern can only be called at init time");
 		return 0;
 		}
 
@@ -1900,9 +1900,9 @@ function convert_for_pattern%(s: string%): string
 
 function string_to_pattern%(s: string, convert: bool%): pattern
 	%{
-	if ( reading_live )
+	if ( bro_start_network_time != 0.0 )
 		{
-		builtin_run_time("should not call merge_pattern while reading live traffic");
+		builtin_run_time("string_to_pattern can only be called at init time");
 		return 0;
 		}
 
@@ -2808,7 +2808,7 @@ private:
 # function result.  Therefore, they can only be called inside a when-condition.
 function lookup_addr%(host: addr%) : string
 	%{
-	// FIXME: Is should be easy to adapt the function to synchronous
+	// FIXME: It should be easy to adapt the function to synchronous
 	// lookups if we're reading a trace.
 	Trigger* trigger = frame->GetTrigger();
 
@@ -2824,8 +2824,18 @@ function lookup_addr%(host: addr%) : string
 #ifdef BROv6
 	if ( ! is_v4_addr(host) )
 		{
-		builtin_run_time("lookup_addr() only supports IPv4 addresses");
+		// FIXME: This is a temporary work-around until we get this
-		return new StringVal("<ipv6-address>");
+		// fixed. We warn the user once, and always trigger a timeout.
+		// Ticket #355 records the problem.
+		static bool warned = false;
+		if ( ! warned )
+			{
+			warn("lookup_addr() only supports IPv4 addresses currently");
+			warned = true;
+			}
+
+		trigger->Timeout();
+		return 0;
 		}
 
 	dns_mgr->AsyncLookupAddr(to_v4_addr(host),
@@ -3196,3 +3206,83 @@ function disable_event_group%(group: string%) : any
 	event_registry->EnableGroup(group->CheckString(), false);
 	return 0;
 	%}
+
+
+%%{
+#include <RandTest.h>
+static map<BroString, RandTest*> entropy_states;
+%%}
+
+function find_entropy%(data: string%): entropy_test_result
+	%{
+	double montepi, scc, ent, mean, chisq;
+	montepi = scc = ent = mean = chisq = 0.0;
+	RecordVal* ent_result = new RecordVal(entropy_test_result);
+	RandTest *rt = new RandTest();
+
+	rt->add((char*) data->Bytes(), data->Len());
+	rt->end(&ent, &chisq, &mean, &montepi, &scc);
+	delete rt;
+
+	ent_result->Assign(0, new Val(ent,     TYPE_DOUBLE));
+	ent_result->Assign(1, new Val(chisq,   TYPE_DOUBLE));
+	ent_result->Assign(2, new Val(mean,    TYPE_DOUBLE));
+	ent_result->Assign(3, new Val(montepi, TYPE_DOUBLE));
+	ent_result->Assign(4, new Val(scc,     TYPE_DOUBLE));
+	return ent_result;
+	%}
+
+function entropy_test_init%(index: any%): bool
+	%{
+	BroString* s = convert_index_to_string(index);
+	int status = 0;
+
+	if ( entropy_states.count(*s) < 1 )
+		{
+		entropy_states[*s] = new RandTest();
+		status = 1;
+		}
+
+	delete s;
+	return new Val(status, TYPE_BOOL);
+	%}
+
+function entropy_test_add%(index: any, data: string%): bool
+	%{
+	BroString* s = convert_index_to_string(index);
+	int status = 0;
+
+	if ( entropy_states.count(*s) > 0 )
+		{
+		entropy_states[*s]->add((char*) data->Bytes(), data->Len());
+		status = 1;
+		}
+
+	delete s;
+	return new Val(status, TYPE_BOOL);
+	%}
+
+function entropy_test_finish%(index: any%): entropy_test_result
+	%{
+	BroString* s = convert_index_to_string(index);
+	double montepi, scc, ent, mean, chisq;
+	montepi = scc = ent = mean = chisq = 0.0;
+	RecordVal* ent_result = new RecordVal(entropy_test_result);
+
+	if ( entropy_states.count(*s) > 0 )
+		{
+		RandTest *rt = entropy_states[*s];
+		rt->end(&ent, &chisq, &mean, &montepi, &scc);
+		entropy_states.erase(*s);
+		delete rt;
+		}
+
+	ent_result->Assign(0, new Val(ent,     TYPE_DOUBLE));
+	ent_result->Assign(1, new Val(chisq,   TYPE_DOUBLE));
+	ent_result->Assign(2, new Val(mean,    TYPE_DOUBLE));
+	ent_result->Assign(3, new Val(montepi, TYPE_DOUBLE));
+	ent_result->Assign(4, new Val(scc,     TYPE_DOUBLE));
+
+	delete s;
+	return ent_result;
+	%}

src/main.cc

@@ -122,6 +122,14 @@ const char* bro_version()
 #endif
 	}
 
+const char* bro_dns_fake()
+	{
+	if ( ! getenv("BRO_DNS_FAKE") )
+		return "off";
+	else 
+		return "on";
+	}
+
 void usage()
 	{
 	fprintf(stderr, "bro version %s\n", bro_version());
@@ -184,6 +192,7 @@ void usage()
 
 	fprintf(stderr, "    $BROPATH                       | file search path (%s)\n", bro_path());
 	fprintf(stderr, "    $BRO_PREFIXES                  | prefix list (%s)\n", bro_prefixes());
+	fprintf(stderr, "    $BRO_DNS_FAKE                  | disable DNS lookups (%s)\n", bro_dns_fake());
 
 	exit(1);
 	}

src/net_util.cc

@@ -97,7 +97,9 @@ int udp6_checksum(const struct ip6_hdr* ip6, const struct udphdr* up, int len)
 	sum = ones_complement_checksum((void*) ip6->ip6_src.s6_addr, 16, sum);
 	sum = ones_complement_checksum((void*) ip6->ip6_dst.s6_addr, 16, sum);
 
-	sum = ones_complement_checksum((void*) &len, 4, sum);
+	uint32 l = htonl(len);
+	sum = ones_complement_checksum((void*) &l, 4, sum);
+
 	uint32 addl_pseudo = htons(IPPROTO_UDP);
 	sum = ones_complement_checksum((void*) &addl_pseudo, 4, sum);
 	sum = ones_complement_checksum((void*) up, len, sum);

src/parse.y

@@ -10,7 +10,7 @@
 %token TOK_BOOL TOK_BREAK TOK_CASE TOK_CONST
 %token TOK_CONSTANT TOK_COPY TOK_COUNT TOK_COUNTER TOK_DEFAULT TOK_DELETE
 %token TOK_DOUBLE TOK_ELSE TOK_ENUM TOK_EVENT TOK_EXPORT TOK_FILE TOK_FOR
-%token TOK_FUNCTION TOK_GLOBAL TOK_GLOBAL_ATTR TOK_ID TOK_IF TOK_INT
+%token TOK_FUNCTION TOK_GLOBAL TOK_ID TOK_IF TOK_INT
 %token TOK_INTERVAL TOK_LIST TOK_LOCAL TOK_MODULE TOK_MATCH TOK_NET
 %token TOK_NEXT TOK_OF TOK_PATTERN TOK_PATTERN_TEXT
 %token TOK_PORT TOK_PRINT TOK_RECORD TOK_REDEF
@@ -55,7 +55,7 @@
 %type <func_type> func_hdr func_params
 %type <type_l> type_list
 %type <type_decl> type_decl formal_args_decl
-%type <type_decl_l> type_decl_list formal_args_decl_list opt_attr_attr
+%type <type_decl_l> type_decl_list formal_args_decl_list
 %type <record> formal_args
 %type <list> expr_list opt_expr_list
 %type <c_case> case
@@ -419,13 +419,7 @@ expr:
 	|	expr TOK_HAS_FIELD TOK_ID
 			{
 			set_location(@1, @3);
-			$$ = new HasFieldExpr($1, $3, false);
+			$$ = new HasFieldExpr($1, $3);
-			}
-
-	|	expr TOK_HAS_ATTR TOK_ID
-			{
-			set_location(@1, @3);
-			$$ = new HasFieldExpr($1, $3, true);
 			}
 
 	|	anonymous_function
@@ -823,17 +817,9 @@ decl:
 				}
 			}
 
-	|	TOK_TYPE global_id ':' refined_type opt_attr opt_attr_attr ';'
+	|	TOK_TYPE global_id ':' refined_type opt_attr ';'
 			{
 			add_type($2, $4, $5, 0);
-			if ( $6 )
-				$2->AsType()->SetAttributesType($6);
-			}
-
-	|	TOK_GLOBAL_ATTR ':' { in_global_attr_decl = true; }
-		'{' type_decl_list '}' ';' { in_global_attr_decl = false; }
-			{
-			global_attributes_type = new RecordType($5);
 			}
 
 	|	TOK_EVENT event_id ':' refined_type opt_attr ';'
@@ -858,13 +844,6 @@ conditional:
 			{ do_atelse(); }
 	;
 
-opt_attr_attr:
-		TOK_ATTR_ATTR '=' '{' type_decl_list '}'
-			{ $$ = $4; }
-	|
-			{ $$ = 0; }
-	;
-
 func_hdr:
 		TOK_FUNCTION global_id func_params
 			{

src/portmap-analyzer.pac

@@ -79,6 +79,10 @@ function PortmapBuildDumpVal(params: PortmapDumpResults): BroVal
 
 	for ( int i = 0; i < params->size(); ++i )
 		{
+		// The last element has cont()!=1 and this element doesn't contain a
+		// mapping.
+		if ((*params)[i]->cont() != 1)
+			continue;
 		Val* m = PortmapBuildMappingVal((*params)[i]->mapping());
 		Val* index = new Val(i + 1, TYPE_COUNT);
 		mappings->Assign(index, m);

src/portmap-protocol.pac

@@ -68,6 +68,7 @@ type PortmapDumpEntry = record {
 	};
 };
 
+# The final element that has cont!=1 will be included in the array.
 type PortmapDumpResults = PortmapDumpEntry[] &until($element.cont != 1);
 
 type PortmapCallItResults = record {

src/scan.l

@@ -151,9 +151,7 @@ file	return TOK_FILE;
 for	return TOK_FOR;
 function	return TOK_FUNCTION;
 global	return TOK_GLOBAL;
-global_attr	return TOK_GLOBAL_ATTR;
 "?$"	return TOK_HAS_FIELD;
-"?$$"	return TOK_HAS_ATTR;
 if	return TOK_IF;
 in	return TOK_IN;
 "!"{OWS}in/[^A-Za-z0-9]	return TOK_NOT_IN;	/* don't confuse w "! infoo"! */