diff --git a/Makefile b/Makefile index 77f0626cdc..863440661e 100644 --- a/Makefile +++ b/Makefile @@ -5,15 +5,12 @@ # to offer. For more, execute that one directly. # -BUILD=build -BROCCOLI=aux/broccoli -BROCTL=aux/broctl - -# CMake/CPack versions before 2.8.2 have bugs that can create bad packages -CMAKE_PACK_REQ=2.8.2 -CMAKE_VER=`cmake -version` - -OSX_VER_CMD=sw_vers | sed -n 's/ProductVersion://p' | cut -d . -f 2 +SOURCE=$(PWD) +BUILD=$(SOURCE)/build +TMP=/tmp/bro-dist.$(UID) +BRO_V=`cat $(SOURCE)/VERSION` +BROCCOLI_V=`cat $(SOURCE)/aux/broccoli/VERSION` +BROCTL_V=`cat $(SOURCE)/aux/broctl/VERSION` all: configured ( cd $(BUILD) && make ) @@ -31,39 +28,25 @@ doc: configured docclean: configured ( cd $(BUILD) && make docclean && make restclean ) -dist: cmake_version - # Minimum Bro source package - ( \ - ./configure --ignore-dirs='aux/broctl;aux/broccoli' --pkg-name-prefix=Bro && \ - cd $(BUILD) && \ - make package_source \ - ) - # Full Bro source package - ( \ - ./configure --pkg-name-prefix=Bro-all && \ - cd $(BUILD) && \ - make package_source \ - ) - # Broccoli source package - ( \ - cd $(BROCCOLI) && \ - ./configure && \ - cd $(BUILD) && \ - make package_source && \ - mv Broccoli*.tar.gz ../../../$(BUILD)/ && \ - cd .. && \ - rm -r $(BUILD) \ - ) - # Broctl source package - ( \ - cd $(BROCTL) && \ - ./configure && \ - cd $(BUILD) && \ - make package_source && \ - mv Broctl*.tar.gz ../../../$(BUILD)/ && \ - cd .. && \ - rm -r $(BUILD) \ - ) +dist: + @( mkdir -p $(BUILD) && rm -rf $(TMP) && mkdir $(TMP) ) + @cp -R $(SOURCE) $(TMP)/Bro-$(BRO_V) + @( cd $(TMP) && find . -name .git\* | xargs rm -rf ) + @( cd $(TMP) && find . -name \*.swp | xargs rm -rf ) + @( cd $(TMP) && find . -type d -name build | xargs rm -rf ) + @( cd $(TMP) && tar -czf $(BUILD)/Bro-all-$(BRO_V).tar.gz Bro-$(BRO_V) ) + @( cd $(TMP)/Bro-$(BRO_V)/aux && mv broccoli Broccoli-$(BROCCOLI_V) && \ + tar -czf $(BUILD)/Broccoli-$(BROCCOLI_V).tar.gz Broccoli-$(BROCCOLI_V) ) + @( cd $(TMP)/Bro-$(BRO_V)/aux && mv broctl Broctl-$(BROCTL_V) && \ + tar -czf $(BUILD)/Broctl-$(BROCTL_V).tar.gz Broctl-$(BROCTL_V) ) + @( cd $(TMP)/Bro-$(BRO_V)/aux && rm -rf Broctl* Broccoli* ) + @( cd $(TMP) && tar -czf $(BUILD)/Bro-$(BRO_V).tar.gz Bro-$(BRO_V) ) + @rm -rf $(TMP) + @echo "Distribution source tarballs have been compiled in $(BUILD)" + +bindist: + @( cd pkg && ( ./make-deb-packages || ./make-mac-packages || \ + ./make-rpm-packages ) ) distclean: rm -rf $(BUILD) @@ -72,7 +55,4 @@ configured: @test -d $(BUILD) || ( echo "Error: No build/ directory found. Did you run configure?" && exit 1 ) @test -e $(BUILD)/Makefile || ( echo "Error: No build/Makefile found. Did you run configure?" && exit 1 ) -cmake_version: - @test "$(CMAKE_VER)" \> "cmake version $(CMAKE_PACK_REQ)" || ( echo "Error: please use a CMake version greater than $(CMAKE_PACK_REQ)" && exit 1 ) - -.PHONY : all install clean distclean configured cmake_version +.PHONY : all install clean doc docclean dist bindist distclean configured diff --git a/aux/broctl b/aux/broctl index 1bf5407722..ad9528f679 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 1bf5407722ef5910bafd513bcec6a51b280eeb10 +Subproject commit ad9528f6795f104db8ec2f1425fc0b69d77ab92d diff --git a/pkg/check-cmake b/pkg/check-cmake new file mode 100755 index 0000000000..2c3ed765a6 --- /dev/null +++ b/pkg/check-cmake @@ -0,0 +1,14 @@ +#!/bin/sh + +# CMake/CPack versions before 2.8.3 have bugs that can create bad packages +# Since packages will be built on several different systems, a single +# version of CMake is required to obtain consistency, but can be increased +# as new versions of CMake come out that also produce working packages. + +CMAKE_PACK_REQ="cmake version 2.8.4" +CMAKE_VER=`cmake -version` + +if [ "${CMAKE_VER}" != "${CMAKE_PACK_REQ}" ]; then + echo "Package creation requires ${CMAKE_PACK_REQ}" >&2 + exit 1 +fi diff --git a/make-deb-packages b/pkg/make-deb-packages similarity index 77% rename from make-deb-packages rename to pkg/make-deb-packages index c8d501198b..a9de210e52 100755 --- a/make-deb-packages +++ b/pkg/make-deb-packages @@ -1,35 +1,30 @@ #!/bin/sh # This script generates binary DEB packages. -# They can be found in build/ after running. +# They can be found in ../build/ after running. -prefix=/opt/bro - -# CMake/CPack versions before 2.8.2 have bugs that can create bad packages -CMAKE_PACK_REQ=2.8.2 -CMAKE_VER=`cmake -version` - -if [ "${CMAKE_VER}" \< "${CMAKE_PACK_REQ}" ]; then - echo "Package creation requires CMake > 2.8.2" >&2 - exit 1 -fi +./check-cmake || { exit 1; } # The DEB CPack generator depends on `dpkg-shlibdeps` to automatically # determine what dependencies to set for the packages type dpkg-shlibdeps > /dev/null 2>&1 || { echo "\ -Creating DEB packages requires the `dpkg-shlibs` command, usually provided by +Creating DEB packages requires the "dpkg-shlibs" command, usually provided by the 'dpkg-dev' package, please install it first. " >&2; exit 1; } +prefix=/opt/bro + # During the packaging process, `dpkg-shlibs` will fail if used on a library # that links to other internal/project libraries unless an RPATH is used or # we set LD_LIBRARY_PATH such that it can find the internal/project library # in the temporary packaging tree. export LD_LIBRARY_PATH=./${prefix}/lib +cd .. + # Minimum Bro ./configure --prefix=${prefix} --disable-broccoli --disable-broctl \ --pkg-name-prefix=Bro --binary-package diff --git a/make-mac-packages b/pkg/make-mac-packages similarity index 84% rename from make-mac-packages rename to pkg/make-mac-packages index c3b6736d20..a8f7f965c8 100755 --- a/make-mac-packages +++ b/pkg/make-mac-packages @@ -1,18 +1,9 @@ #!/bin/sh # This script creates binary packages for Mac OS X. -# They can be found in build/ after running. +# They can be found in ../build/ after running. -prefix=/opt/bro - -# CMake/CPack versions before 2.8.2 have bugs that can create bad packages -CMAKE_PACK_REQ=2.8.3 -CMAKE_VER=`cmake -version` - -if [ "${CMAKE_VER}" \< "${CMAKE_PACK_REQ}" ]; then - echo "Package creation requires CMake > 2.8.2" >&2 - exit 1 -fi +./check-cmake || { exit 1; } type sw_vers > /dev/null 2>&1 || { echo "Unable to get Mac OS X version" >&2; @@ -38,6 +29,10 @@ else arch=x86_64 fi +prefix=/opt/bro + +cd .. + # Minimum Bro CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \ --disable-broccoli --disable-broctl --pkg-name-prefix=Bro \ diff --git a/make-rpm-packages b/pkg/make-rpm-packages similarity index 65% rename from make-rpm-packages rename to pkg/make-rpm-packages index 503d80ef4e..ac8dfa97b4 100755 --- a/make-rpm-packages +++ b/pkg/make-rpm-packages @@ -1,18 +1,22 @@ #!/bin/sh # This script generates binary RPM packages. -# They can be found in build/ after running. +# They can be found in ../build/ after running. + +./check-cmake || { exit 1; } + +# The RPM CPack generator depends on `rpmbuild` to create packages +type rpmbuild > /dev/null 2>&1 || { + echo "\ +Creating RPM packages requires the "rpmbuild" command, usually provided by +the 'rpm-build' package, please install it first. +" >&2; + exit 1; +} prefix=/opt/bro -# CMake/CPack versions before 2.8.2 have bugs that can create bad packages -CMAKE_PACK_REQ=2.8.2 -CMAKE_VER=`cmake -version` - -if [ "${CMAKE_VER}" \< "${CMAKE_PACK_REQ}" ]; then - echo "Package creation requires CMake > 2.8.2" >&2 - exit 1 -fi +cd .. # Minimum Bro ./configure --prefix=${prefix} --disable-broccoli --disable-broctl \ diff --git a/src/Debug.cc b/src/Debug.cc index da67c941e4..272d6739ae 100644 --- a/src/Debug.cc +++ b/src/Debug.cc @@ -343,7 +343,7 @@ vector parse_location_string(const string& s) plr.type = plrUnknown; FILE* throwaway = search_for_file(filename.c_str(), "bro", - &full_filename); + &full_filename, true); if ( ! throwaway ) { debug_msg("No such policy file: %s.\n", filename.c_str()); diff --git a/src/Desc.cc b/src/Desc.cc index ce6083800c..c15a461bb2 100644 --- a/src/Desc.cc +++ b/src/Desc.cc @@ -207,41 +207,71 @@ void ODesc::Indent() } } -static const char hex_chars[] = "0123456789ABCDEF"; +static const char hex_chars[] = "0123456789abcdef"; + +static const char* find_first_unprintable(ODesc* d, const char* bytes, unsigned int n) + { + if ( d->IsBinary() ) + return 0; + + while ( n-- ) + { + if ( ! isprint(*bytes) ) + return bytes; + ++bytes; + } + + return 0; + } void ODesc::AddBytes(const void* bytes, unsigned int n) { - if ( ! escape ) - return AddBytesRaw(bytes, n); - const char* s = (const char*) bytes; const char* e = (const char*) bytes + n; while ( s < e ) { - const char* t = (const char*) memchr(s, escape[0], e - s); + const char* t1 = escape ? (const char*) memchr(s, escape[0], e - s) : e; + const char* t2 = find_first_unprintable(this, s, t1 ? e - t1 : e - s); - if ( ! t ) + if ( t2 && (t2 < t1 || ! t1) ) + { + AddBytesRaw(s, t2 - s); + + char hex[6] = "\\x00"; + hex[2] = hex_chars[((*t2) & 0xf0) >> 4]; + hex[3] = hex_chars[(*t2) & 0x0f]; + AddBytesRaw(hex, sizeof(hex)); + + s = t2 + 1; + continue; + } + + if ( ! escape ) break; - if ( memcmp(t, escape, escape_len) != 0 ) + if ( ! t1 ) break; - AddBytesRaw(s, t - s); + if ( memcmp(t1, escape, escape_len) != 0 ) + break; + + AddBytesRaw(s, t1 - s); for ( int i = 0; i < escape_len; ++i ) { char hex[5] = "\\x00"; - hex[2] = hex_chars[(*t) >> 4]; - hex[3] = hex_chars[(*t) & 0x0f]; + hex[2] = hex_chars[((*t1) & 0xf0) >> 4]; + hex[3] = hex_chars[(*t1) & 0x0f]; AddBytesRaw(hex, sizeof(hex)); - ++t; + ++t1; } - s = t; + s = t1; } - AddBytesRaw(s, e - s); + if ( s < e ) + AddBytesRaw(s, e - s); } void ODesc::AddBytesRaw(const void* bytes, unsigned int n) diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index 4c54e76715..d831960a3c 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -223,7 +223,7 @@ bool LogWriterAscii::DoWrite(int num_fields, const LogField* const * fields, return false; } - desc.Add("\n"); + desc.AddRaw("\n", 1); if ( fwrite(desc.Bytes(), desc.Len(), 1, file) != 1 ) { diff --git a/src/OSFinger.cc b/src/OSFinger.cc index 8d3d2057bf..f7b4903700 100644 --- a/src/OSFinger.cc +++ b/src/OSFinger.cc @@ -295,7 +295,7 @@ void OSFingerprint::load_config(const char* file) uint32 ln=0; char buf[MAXLINE]; char* p; - FILE* c = search_for_file( file, "osf", 0); + FILE* c = search_for_file( file, "osf", 0, false); if (!c) { diff --git a/src/RuleMatcher.cc b/src/RuleMatcher.cc index 02eef0aad9..2a0246d121 100644 --- a/src/RuleMatcher.cc +++ b/src/RuleMatcher.cc @@ -195,7 +195,7 @@ bool RuleMatcher::ReadFiles(const name_list& files) for ( int i = 0; i < files.length(); ++i ) { - rules_in = search_for_file( files[i], "sig", 0); + rules_in = search_for_file( files[i], "sig", 0, false); if ( ! rules_in ) { error("Can't open signature file", files[i]); diff --git a/src/scan.l b/src/scan.l index 601962a37c..d58b1bcc9d 100644 --- a/src/scan.l +++ b/src/scan.l @@ -557,7 +557,7 @@ static int load_files_with_prefix(const char* orig_file) else strcpy(new_filename, file); - f = search_for_file(new_filename, "bro", &full_filename); + f = search_for_file(new_filename, "bro", &full_filename, true); delete [] new_filename; } diff --git a/src/util.cc b/src/util.cc index 5684fe20c1..3d69c981be 100644 --- a/src/util.cc +++ b/src/util.cc @@ -868,21 +868,45 @@ const char* bro_prefixes() return p; } -FILE* open_file(const char* filename, const char** full_filename) +static const char* PACKAGE_LOADER = "__load__.bro"; + +// If filename is pointing to a directory that contains a file called +// PACKAGE_LOADER, returns the files path. Otherwise returns filename itself. +// In both cases, the returned string is newly allocated. +static const char* check_for_dir(const char* filename, bool load_pkgs) { + if ( load_pkgs && is_dir(filename) ) + { + char init_filename_buf[1024]; + safe_snprintf(init_filename_buf, sizeof(init_filename_buf), + "%s/%s", filename, PACKAGE_LOADER); + + if ( access(init_filename_buf, R_OK) == 0 ) + return copy_string(init_filename_buf); + } + + return copy_string(filename); + } + +FILE* open_file(const char* filename, const char** full_filename, bool load_pkgs) + { + filename = check_for_dir(filename, load_pkgs); + if ( full_filename ) *full_filename = copy_string(filename); FILE* f = fopen(filename, "r"); + delete [] filename; + return f; } FILE* search_for_file(const char* filename, const char* ext, - const char** full_filename) + const char** full_filename, bool load_pkgs) { if ( filename[0] == '/' || filename[0] == '.' ) - return open_file(filename, full_filename); + return open_file(filename, full_filename, load_pkgs); char path[1024], full_filename_buf[1024]; safe_strncpy(path, bro_path(), sizeof(path)); @@ -905,13 +929,12 @@ FILE* search_for_file(const char* filename, const char* ext, "%s/%s.%s", dir_beginning, filename, ext); if ( access(full_filename_buf, R_OK) == 0 && ! is_dir(full_filename_buf) ) - return open_file(full_filename_buf, full_filename); + return open_file(full_filename_buf, full_filename, load_pkgs); safe_snprintf(full_filename_buf, sizeof(full_filename_buf), "%s/%s", dir_beginning, filename); - if ( access(full_filename_buf, R_OK) == 0 && - ! is_dir(full_filename_buf) ) - return open_file(full_filename_buf, full_filename); + if ( access(full_filename_buf, R_OK) == 0 ) + return open_file(full_filename_buf, full_filename, load_pkgs); dir_beginning = ++dir_ending; } diff --git a/src/util.h b/src/util.h index 82bc0adaf3..a288ed30c8 100644 --- a/src/util.h +++ b/src/util.h @@ -190,7 +190,7 @@ extern int int_list_cmp(const void* v1, const void* v2); extern const char* bro_path(); extern const char* bro_prefixes(); extern FILE* search_for_file(const char* filename, const char* ext, - const char** full_filename); + const char** full_filename, bool load_pkgs); // Renames the given file to a new temporary name, and opens a new file with // the original name. Returns new file or NULL on error. Inits rotate_info if diff --git a/testing/btest/Baseline/core.load-pkg/output b/testing/btest/Baseline/core.load-pkg/output new file mode 100644 index 0000000000..01c77289d2 --- /dev/null +++ b/testing/btest/Baseline/core.load-pkg/output @@ -0,0 +1,14 @@ +loading /home/robin/bro/master/policy/bro.init + loading /home/robin/bro/master/build/src/const.bif.bro + loading /home/robin/bro/master/build/src/types.bif.bro + loading /home/robin/bro/master/build/src/strings.bif.bro + loading /home/robin/bro/master/build/src/bro.bif.bro + loading /home/robin/bro/master/policy/logging.bro + loading /home/robin/bro/master/build/src/logging.bif.bro + loading /home/robin/bro/master/policy/logging-ascii.bro + loading /home/robin/bro/master/build/src/event.bif.bro + loading /home/robin/bro/master/policy/pcap.bro + loading /home/robin/bro/master/policy/server-ports.bro +loading ./foo/test.bro +loading ./foo/__load__.bro +Foo loaded diff --git a/testing/btest/Baseline/logging.ascii-binary/ssh.log b/testing/btest/Baseline/logging.ascii-binary/ssh.log new file mode 100644 index 0000000000..84a2cc609e Binary files /dev/null and b/testing/btest/Baseline/logging.ascii-binary/ssh.log differ diff --git a/testing/btest/Baseline/logging.ascii-escape/ssh.log b/testing/btest/Baseline/logging.ascii-escape/ssh.log index 6797e1e302..aa08625281 100644 Binary files a/testing/btest/Baseline/logging.ascii-escape/ssh.log and b/testing/btest/Baseline/logging.ascii-escape/ssh.log differ diff --git a/testing/btest/core/load-pkg.bro b/testing/btest/core/load-pkg.bro new file mode 100644 index 0000000000..907dde78ce --- /dev/null +++ b/testing/btest/core/load-pkg.bro @@ -0,0 +1,7 @@ +# @TEST-EXEC: mkdir foo +# @TEST-EXEC: echo "@load foo/test.bro" >foo/__load__.bro +# @TEST-EXEC: cp %INPUT foo/test.bro +# @TEST-EXEC: bro -l foo >output 2>&1 +# @TEST-EXEC: btest-diff output + +print "Foo loaded"; diff --git a/testing/btest/logging/ascii-binary.bro b/testing/btest/logging/ascii-binary.bro new file mode 100644 index 0000000000..6f095db0c7 --- /dev/null +++ b/testing/btest/logging/ascii-binary.bro @@ -0,0 +1,25 @@ +# +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: btest-diff ssh.log + +module SSH; + +export { + redef enum Log::ID += { SSH }; + + type Log: record { + data: string; + data2: string; + } &log; +} + +redef LogAscii::separator = "|"; + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + Log::write(SSH, [$data="abc\n\xffdef", $data2="DATA2"]); + Log::write(SSH, [$data="abc|\xffdef", $data2="DATA2"]); + Log::write(SSH, [$data="abc\xff|def", $data2="DATA2"]); +} +