FileAnalysis: add custom libmagic database.

- It's derived from the magic database of libmagic 5.14, but with most
  everything not related to mime types removed.

- The custom database is always used by default for mime detection, but
  the more verbose file type detection will fall back on the default
  libmagic installation's database.  The result is: mime type strings
  are now guaranteed to be consistent across platforms, but the verbose
  file type descriptions are not.

- The custom database gets installed in $prefix/share/bro/magic, and
  should even be extensible if files with new patterns are added inside
  the directory.

- The search path for the mime magic database can be controlled via
  BROMAGIC environment variable.

- Remove mime_desc field from ftp.log.

- Stop using the mime/file type canonifier with unit tests.

- libmagic >= 5.04 is now a requirement.

src/main.cc

@@ -206,6 +206,7 @@ void usage()
 	fprintf(stderr, "    --use-binpac                   | use new-style BinPAC parsers when available\n");
 
 	fprintf(stderr, "    $BROPATH                       | file search path (%s)\n", bro_path());
+	fprintf(stderr, "    $BROMAGIC                      | libmagic mime magic database search path (%s)\n", bro_magic_path());
 	fprintf(stderr, "    $BRO_PREFIXES                  | prefix list (%s)\n", bro_prefixes());
 	fprintf(stderr, "    $BRO_DNS_FAKE                  | disable DNS lookups (%s)\n", bro_dns_fake());
 	fprintf(stderr, "    $BRO_SEED_FILE                 | file to load seeds from (not set)\n");

src/util-config.h.in

@@ -1,3 +1,4 @@
 #define BRO_SCRIPT_INSTALL_PATH "@BRO_SCRIPT_INSTALL_PATH@"
 #define BRO_SCRIPT_SOURCE_PATH "@BRO_SCRIPT_SOURCE_PATH@"
 #define BRO_BUILD_PATH "@CMAKE_CURRENT_BINARY_DIR@"
+#define BRO_MAGIC_INSTALL_PATH "@BRO_MAGIC_INSTALL_PATH@"

src/util.cc

@@ -864,6 +864,16 @@ const char* bro_path()
 	return path;
 	}
 
+const char* bro_magic_path()
+	{
+	const char* path = getenv("BROMAGIC");
+
+	if ( ! path )
+		path = BRO_MAGIC_INSTALL_PATH;
+
+	return path;
+	}
+
 const char* bro_prefixes()
 	{
 	int len = 1;	// room for \0
@@ -1560,16 +1570,22 @@ void bro_init_magic(magic_t* cookie_ptr, int flags)
 
 	*cookie_ptr = magic_open(flags);
 
+	// Use our custom database for mime types, but the default database
+	// from libmagic for the verbose file type.
+	const char* database = (flags & MAGIC_MIME) ? bro_magic_path() : 0;
+
 	if ( ! *cookie_ptr )
 		{
 		const char* err = magic_error(*cookie_ptr);
-		reporter->Error("can't init libmagic: %s", err ? err : "unknown");
+		if ( ! err ) err = "unknown";
+		reporter->InternalError("can't init libmagic: %s", err);
 		}
 
-	else if ( magic_load(*cookie_ptr, 0) < 0 )
+	else if ( magic_load(*cookie_ptr, database) < 0 )
 		{
 		const char* err = magic_error(*cookie_ptr);
-		reporter->Error("can't load magic file: %s", err ? err : "unknown");
+		if ( ! err ) err = "unknown";
+		reporter->InternalError("can't load magic file: %s", err);
 		magic_close(*cookie_ptr);
 		*cookie_ptr = 0;
 		}

src/util.h

@@ -188,6 +188,7 @@ extern void pinpoint();
 extern int int_list_cmp(const void* v1, const void* v2);
 
 extern const char* bro_path();
+extern const char* bro_magic_path();
 extern const char* bro_prefixes();
 std::string dot_canon(std::string path, std::string file, std::string prefix = "");
 const char* normalize_path(const char* path);