Merge remote branch 'origin/master' into topic/policy-scripts-new

Makefile

@@ -5,15 +5,12 @@
 # to offer. For more, execute that one directly. 
 #
 
-BUILD=build
+SOURCE=$(PWD)
-BROCCOLI=aux/broccoli
+BUILD=$(SOURCE)/build
-BROCTL=aux/broctl
+TMP=/tmp/bro-dist.$(UID)
-
+BRO_V=`cat $(SOURCE)/VERSION`
-# CMake/CPack versions before 2.8.2 have bugs that can create bad packages
+BROCCOLI_V=`cat $(SOURCE)/aux/broccoli/VERSION`
-CMAKE_PACK_REQ=2.8.2
+BROCTL_V=`cat $(SOURCE)/aux/broctl/VERSION`
-CMAKE_VER=`cmake -version`
-
-OSX_VER_CMD=sw_vers | sed -n 's/ProductVersion://p' | cut -d . -f 2
 
 all: configured
 	( cd $(BUILD) && make )
@@ -31,39 +28,25 @@ doc: configured
 docclean: configured
 	( cd $(BUILD) && make docclean && make restclean )
 
-dist: cmake_version
+dist:
-	# Minimum Bro source package
+	@( mkdir -p $(BUILD) && rm -rf $(TMP) && mkdir $(TMP) )
-	( \
+	@cp -R $(SOURCE) $(TMP)/Bro-$(BRO_V)
-	./configure --ignore-dirs='aux/broctl;aux/broccoli' --pkg-name-prefix=Bro && \
+	@( cd $(TMP) && find . -name .git\* | xargs rm -rf )
-	cd $(BUILD) && \
+	@( cd $(TMP) && find . -name \*.swp | xargs rm -rf )
-	make package_source \
+	@( cd $(TMP) && find . -type d -name build | xargs rm -rf )
-	)
+	@( cd $(TMP) && tar -czf $(BUILD)/Bro-all-$(BRO_V).tar.gz Bro-$(BRO_V) )
-	# Full Bro source package
+	@( cd $(TMP)/Bro-$(BRO_V)/aux && mv broccoli Broccoli-$(BROCCOLI_V) && \
-	( \
+	    tar -czf $(BUILD)/Broccoli-$(BROCCOLI_V).tar.gz Broccoli-$(BROCCOLI_V) )
-	./configure --pkg-name-prefix=Bro-all && \
+	@( cd $(TMP)/Bro-$(BRO_V)/aux && mv broctl Broctl-$(BROCTL_V) && \
-	cd $(BUILD) && \
+	    tar -czf $(BUILD)/Broctl-$(BROCTL_V).tar.gz Broctl-$(BROCTL_V) )
-	make package_source \
+	@( cd $(TMP)/Bro-$(BRO_V)/aux && rm -rf Broctl* Broccoli* )
-	)
+	@( cd $(TMP) && tar -czf $(BUILD)/Bro-$(BRO_V).tar.gz Bro-$(BRO_V) )
-	# Broccoli source package
+	@rm -rf $(TMP)
-	( \
+	@echo "Distribution source tarballs have been compiled in $(BUILD)"
-	cd $(BROCCOLI) && \
+
-	./configure && \
+bindist:
-	cd $(BUILD) && \
+	@( cd pkg && ( ./make-deb-packages || ./make-mac-packages || \
-	make package_source && \
+	               ./make-rpm-packages ) )
-	mv Broccoli*.tar.gz ../../../$(BUILD)/ && \
-	cd .. && \
-	rm -r $(BUILD) \
-	)
-	# Broctl source package
-	( \
-	cd $(BROCTL) && \
-	./configure && \
-	cd $(BUILD) && \
-	make package_source && \
-	mv Broctl*.tar.gz ../../../$(BUILD)/ && \
-	cd .. && \
-	rm -r $(BUILD) \
-	)
 
 distclean:
 	rm -rf $(BUILD)
@@ -72,7 +55,4 @@ configured:
 	@test -d $(BUILD) || ( echo "Error: No build/ directory found. Did you run configure?" && exit 1 )
 	@test -e $(BUILD)/Makefile || ( echo "Error: No build/Makefile found. Did you run configure?" && exit 1 )
 
-cmake_version:
+.PHONY : all install clean doc docclean dist bindist distclean configured
-	@test "$(CMAKE_VER)" \> "cmake version $(CMAKE_PACK_REQ)" || ( echo "Error: please use a CMake version greater than $(CMAKE_PACK_REQ)" && exit 1 )
-
-.PHONY : all install clean distclean configured cmake_version

aux/broctl

@@ -1 +1 @@
-Subproject commit 1bf5407722ef5910bafd513bcec6a51b280eeb10
+Subproject commit ad9528f6795f104db8ec2f1425fc0b69d77ab92d

pkg/check-cmake

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+# CMake/CPack versions before 2.8.3 have bugs that can create bad packages
+# Since packages will be built on several different systems, a single
+# version of CMake is required to obtain consistency, but can be increased
+# as new versions of CMake come out that also produce working packages.
+
+CMAKE_PACK_REQ="cmake version 2.8.4"
+CMAKE_VER=`cmake -version`
+
+if [ "${CMAKE_VER}" != "${CMAKE_PACK_REQ}" ]; then
+    echo "Package creation requires ${CMAKE_PACK_REQ}" >&2
+    exit 1
+fi

pkg/make-deb-packages

@@ -1,35 +1,30 @@
 #!/bin/sh
 
 # This script generates binary DEB packages.
-# They can be found in build/ after running.
+# They can be found in ../build/ after running.
 
-prefix=/opt/bro
+./check-cmake || { exit 1; }
-
-# CMake/CPack versions before 2.8.2 have bugs that can create bad packages
-CMAKE_PACK_REQ=2.8.2
-CMAKE_VER=`cmake -version`
-
-if [ "${CMAKE_VER}" \< "${CMAKE_PACK_REQ}" ]; then
-    echo "Package creation requires CMake > 2.8.2" >&2
-    exit 1
-fi
 
 # The DEB CPack generator depends on `dpkg-shlibdeps` to automatically
 # determine what dependencies to set for the packages
 type dpkg-shlibdeps > /dev/null 2>&1 || {
     echo "\
-Creating DEB packages requires the `dpkg-shlibs` command, usually provided by
+Creating DEB packages requires the "dpkg-shlibs" command, usually provided by
 the 'dpkg-dev' package, please install it first.
 " >&2;
     exit 1;
 }
 
+prefix=/opt/bro
+
 # During the packaging process, `dpkg-shlibs` will fail if used on a library
 # that links to other internal/project libraries unless an RPATH is used or
 # we set LD_LIBRARY_PATH such that it can find the internal/project library
 # in the temporary packaging tree.
 export LD_LIBRARY_PATH=./${prefix}/lib
 
+cd ..
+
 # Minimum Bro
 ./configure --prefix=${prefix} --disable-broccoli --disable-broctl \
             --pkg-name-prefix=Bro --binary-package

pkg/make-mac-packages

@@ -1,18 +1,9 @@
 #!/bin/sh
 
 # This script creates binary packages for Mac OS X.
-# They can be found in build/ after running.
+# They can be found in ../build/ after running.
 
-prefix=/opt/bro
+./check-cmake || { exit 1; }
-
-# CMake/CPack versions before 2.8.2 have bugs that can create bad packages
-CMAKE_PACK_REQ=2.8.3
-CMAKE_VER=`cmake -version`
-
-if [ "${CMAKE_VER}" \< "${CMAKE_PACK_REQ}" ]; then
-    echo "Package creation requires CMake > 2.8.2" >&2
-    exit 1
-fi
 
 type sw_vers > /dev/null 2>&1 || {
     echo "Unable to get Mac OS X version" >&2;
@@ -38,6 +29,10 @@ else
     arch=x86_64
 fi
 
+prefix=/opt/bro
+
+cd ..
+
 # Minimum Bro
 CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
     --disable-broccoli --disable-broctl --pkg-name-prefix=Bro \

pkg/make-rpm-packages

@@ -1,18 +1,22 @@
 #!/bin/sh
 
 # This script generates binary RPM packages.
-# They can be found in build/ after running.
+# They can be found in ../build/ after running.
+
+./check-cmake || { exit 1; }
+
+# The RPM CPack generator depends on `rpmbuild` to create packages
+type rpmbuild > /dev/null 2>&1 || {
+    echo "\
+Creating RPM packages requires the "rpmbuild" command, usually provided by
+the 'rpm-build' package, please install it first.
+" >&2;
+    exit 1;
+}
 
 prefix=/opt/bro
 
-# CMake/CPack versions before 2.8.2 have bugs that can create bad packages
+cd ..
-CMAKE_PACK_REQ=2.8.2
-CMAKE_VER=`cmake -version`
-
-if [ "${CMAKE_VER}" \< "${CMAKE_PACK_REQ}" ]; then
-    echo "Package creation requires CMake > 2.8.2" >&2
-    exit 1
-fi
 
 # Minimum Bro
 ./configure --prefix=${prefix} --disable-broccoli --disable-broctl \

src/Debug.cc

@@ -343,7 +343,7 @@ vector<ParseLocationRec> parse_location_string(const string& s)
 				plr.type = plrUnknown;
 
 			FILE* throwaway = search_for_file(filename.c_str(), "bro",
-								&full_filename);
+								&full_filename, true);
 			if ( ! throwaway )
 				{
 				debug_msg("No such policy file: %s.\n", filename.c_str());

src/Desc.cc

@@ -207,41 +207,71 @@ void ODesc::Indent()
 		}
 	}
 
-static const char hex_chars[] = "0123456789ABCDEF";
+static const char hex_chars[] = "0123456789abcdef";
+
+static const char* find_first_unprintable(ODesc* d, const char* bytes, unsigned int n)
+	{
+	if ( d->IsBinary() )
+		return 0;
+
+	while ( n-- )
+		{
+		if ( ! isprint(*bytes) )
+			return bytes;
+		++bytes;
+		}
+
+	return 0;
+	}
 
 void ODesc::AddBytes(const void* bytes, unsigned int n)
 	{
-	if ( ! escape )
-		return AddBytesRaw(bytes, n);
-
 	const char* s = (const char*) bytes;
 	const char* e = (const char*) bytes + n;
 
 	while ( s < e )
 		{
-		const char* t = (const char*) memchr(s, escape[0], e - s);
+		const char* t1 = escape ? (const char*) memchr(s, escape[0], e - s) : e;
+		const char* t2 = find_first_unprintable(this, s, t1 ? e - t1 : e - s);
 
-		if ( ! t )
+		if ( t2 && (t2 < t1 || ! t1) )
+			{
+			AddBytesRaw(s, t2 - s);
+
+			char hex[6] = "\\x00";
+			hex[2] = hex_chars[((*t2) & 0xf0) >> 4];
+			hex[3] = hex_chars[(*t2) & 0x0f];
+			AddBytesRaw(hex, sizeof(hex));
+
+			s = t2 + 1;
+			continue;
+			}
+
+		if ( ! escape )
 			break;
 
-		if ( memcmp(t, escape, escape_len) != 0 )
+		if ( ! t1 )
 			break;
 
-		AddBytesRaw(s, t - s);
+		if ( memcmp(t1, escape, escape_len) != 0 )
+			break;
+
+		AddBytesRaw(s, t1 - s);
 
 		for ( int i = 0; i < escape_len; ++i )
 			{
 			char hex[5] = "\\x00";
-			hex[2] = hex_chars[(*t) >> 4];
+			hex[2] = hex_chars[((*t1) & 0xf0) >> 4];
-			hex[3] = hex_chars[(*t) & 0x0f];
+			hex[3] = hex_chars[(*t1) & 0x0f];
 			AddBytesRaw(hex, sizeof(hex));
-			++t;
+			++t1;
 			}
 
-		s = t;
+		s = t1;
 		}
 
-	AddBytesRaw(s, e - s);
+	if ( s < e )
+		AddBytesRaw(s, e - s);
 	}
 
 void ODesc::AddBytesRaw(const void* bytes, unsigned int n)

src/LogWriterAscii.cc

@@ -223,7 +223,7 @@ bool LogWriterAscii::DoWrite(int num_fields, const LogField* const * fields,
 			return false;
 		}
 
-	desc.Add("\n");
+	desc.AddRaw("\n", 1);
 
 	if ( fwrite(desc.Bytes(), desc.Len(), 1, file) != 1 )
 		{

src/OSFinger.cc

@@ -295,7 +295,7 @@ void OSFingerprint::load_config(const char* file)
   uint32 ln=0;
   char buf[MAXLINE];
   char* p;
-  FILE* c = search_for_file( file, "osf", 0);
+  FILE* c = search_for_file( file, "osf", 0, false);
 
   if (!c)
     {

src/RuleMatcher.cc

@@ -195,7 +195,7 @@ bool RuleMatcher::ReadFiles(const name_list& files)
 
 	for ( int i = 0; i < files.length(); ++i )
 		{
-		rules_in = search_for_file( files[i], "sig", 0);
+		rules_in = search_for_file( files[i], "sig", 0, false);
 		if ( ! rules_in )
 			{
 			error("Can't open signature file", files[i]);

src/scan.l

@@ -557,7 +557,7 @@ static int load_files_with_prefix(const char* orig_file)
 			else
 				strcpy(new_filename, file);
 
-			f = search_for_file(new_filename, "bro", &full_filename);
+			f = search_for_file(new_filename, "bro", &full_filename, true);
 			delete [] new_filename;
 			}
 

src/util.cc

@@ -868,21 +868,45 @@ const char* bro_prefixes()
 	return p;
 	}
 
-FILE* open_file(const char* filename, const char** full_filename)
+static const char* PACKAGE_LOADER = "__load__.bro";
+
+// If filename is pointing to a directory that contains a file called
+// PACKAGE_LOADER, returns the files path. Otherwise returns filename itself.
+// In both cases, the returned string is newly allocated.
+static const char* check_for_dir(const char* filename, bool load_pkgs)
 	{
+	if ( load_pkgs && is_dir(filename) )
+		{
+		char init_filename_buf[1024];
+		safe_snprintf(init_filename_buf, sizeof(init_filename_buf),
+			      "%s/%s", filename, PACKAGE_LOADER);
+
+		if ( access(init_filename_buf, R_OK) == 0 )
+			return copy_string(init_filename_buf);
+		}
+
+	return copy_string(filename);
+	}
+
+FILE* open_file(const char* filename, const char** full_filename, bool load_pkgs)
+	{
+	filename = check_for_dir(filename, load_pkgs);
+
 	if ( full_filename )
 		*full_filename = copy_string(filename);
 
 	FILE* f = fopen(filename, "r");
 
+	delete [] filename;
+
 	return f;
 	}
 
 FILE* search_for_file(const char* filename, const char* ext,
-			const char** full_filename)
+			const char** full_filename, bool load_pkgs)
 	{
 	if ( filename[0] == '/' || filename[0] == '.' )
-		return open_file(filename, full_filename);
+		return open_file(filename, full_filename, load_pkgs);
 
 	char path[1024], full_filename_buf[1024];
 	safe_strncpy(path, bro_path(), sizeof(path));
@@ -905,13 +929,12 @@ FILE* search_for_file(const char* filename, const char* ext,
 				"%s/%s.%s", dir_beginning, filename, ext);
 		if ( access(full_filename_buf, R_OK) == 0 &&
 		     ! is_dir(full_filename_buf) )
-			return open_file(full_filename_buf, full_filename);
+			return open_file(full_filename_buf, full_filename, load_pkgs);
 
 		safe_snprintf(full_filename_buf, sizeof(full_filename_buf),
 				"%s/%s", dir_beginning, filename);
-		if ( access(full_filename_buf, R_OK) == 0 &&
+		if ( access(full_filename_buf, R_OK) == 0 )
-		      ! is_dir(full_filename_buf) )
+			return open_file(full_filename_buf, full_filename, load_pkgs);
-			return open_file(full_filename_buf, full_filename);
 
 		dir_beginning = ++dir_ending;
 		}

src/util.h

@@ -190,7 +190,7 @@ extern int int_list_cmp(const void* v1, const void* v2);
 extern const char* bro_path();
 extern const char* bro_prefixes();
 extern FILE* search_for_file(const char* filename, const char* ext,
-	const char** full_filename);
+	const char** full_filename, bool load_pkgs);
 
 // Renames the given file to a new temporary name, and opens a new file with
 // the original name. Returns new file or NULL on error. Inits rotate_info if

testing/btest/Baseline/core.load-pkg/output

@@ -0,0 +1,14 @@
+loading /home/robin/bro/master/policy/bro.init
+   loading /home/robin/bro/master/build/src/const.bif.bro
+   loading /home/robin/bro/master/build/src/types.bif.bro
+   loading /home/robin/bro/master/build/src/strings.bif.bro
+   loading /home/robin/bro/master/build/src/bro.bif.bro
+   loading /home/robin/bro/master/policy/logging.bro
+      loading /home/robin/bro/master/build/src/logging.bif.bro
+   loading /home/robin/bro/master/policy/logging-ascii.bro
+   loading /home/robin/bro/master/build/src/event.bif.bro
+   loading /home/robin/bro/master/policy/pcap.bro
+   loading /home/robin/bro/master/policy/server-ports.bro
+loading ./foo/test.bro
+loading ./foo/__load__.bro
+Foo loaded

testing/btest/core/load-pkg.bro

@@ -0,0 +1,7 @@
+# @TEST-EXEC: mkdir foo
+# @TEST-EXEC: echo "@load foo/test.bro" >foo/__load__.bro
+# @TEST-EXEC: cp %INPUT foo/test.bro
+# @TEST-EXEC: bro -l foo >output 2>&1
+# @TEST-EXEC: btest-diff output
+
+print "Foo loaded";

testing/btest/logging/ascii-binary.bro

@@ -0,0 +1,25 @@
+#
+# @TEST-EXEC: bro %INPUT
+# @TEST-EXEC: btest-diff ssh.log
+
+module SSH;
+
+export {
+	redef enum Log::ID += { SSH };
+
+	type Log: record {
+		data: string;
+		data2: string;
+	} &log;
+}
+
+redef LogAscii::separator = "|";
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Log]);
+	Log::write(SSH, [$data="abc\n\xffdef", $data2="DATA2"]);
+	Log::write(SSH, [$data="abc|\xffdef", $data2="DATA2"]);
+	Log::write(SSH, [$data="abc\xff|def", $data2="DATA2"]);
+}
+