Merge remote-tracking branch 'origin/topic/jsiwek/file-caching-serialization'

* origin/topic/jsiwek/file-caching-serialization:
  Changes to open-file caching limits and uncached file unserialization.

Closes #780.

scripts/base/init-bare.bro

@@ -2329,6 +2329,11 @@ type bt_tracker_headers: table[string] of string;
 ## BPF filter the user has set via the -f command line options. Empty if none.
 const cmd_line_bpf_filter = "" &redef;
 
+## The maximum number of open files to keep cached at a given time.
+## If set to zero, this is automatically determined by inspecting
+## the current/maximum limit on open files for the process.
+const max_files_in_cache = 0 &redef;
+
 ## Deprecated.
 const log_rotate_interval = 0 sec &redef;
 

src/File.cc

@@ -74,9 +74,8 @@ void RotateTimer::Dispatch(double t, int is_expire)
 
 // The following could in principle be part of a "file manager" object.
 
-#define MAX_FILE_CACHE_SIZE 32
+#define MAX_FILE_CACHE_SIZE 512
 static int num_files_in_cache = 0;
-static int max_files_in_cache = 0;
 static BroFile* head = 0;
 static BroFile* tail = 0;
 
@@ -87,12 +86,9 @@ double BroFile::default_rotation_size = 0;
 // that we should use for the cache.
 static int maximize_num_fds()
 	{
-#ifdef NO_HAVE_SETRLIMIT
-	return MAX_FILE_CACHE_SIZE;
-#else
 	struct rlimit rl;
 	if ( getrlimit(RLIMIT_NOFILE, &rl) < 0 )
-		reporter->InternalError("maximize_num_fds(): getrlimit failed");
+		reporter->FatalError("maximize_num_fds(): getrlimit failed");
 
 	if ( rl.rlim_max == RLIM_INFINITY )
 		{
@@ -108,10 +104,9 @@ static int maximize_num_fds()
 	rl.rlim_cur = rl.rlim_max;
 
 	if ( setrlimit(RLIMIT_NOFILE, &rl) < 0 )
-		reporter->InternalError("maximize_num_fds(): setrlimit failed");
+		reporter->FatalError("maximize_num_fds(): setrlimit failed");
 
 	return rl.rlim_cur / 2;
-#endif
 	}
 
 
@@ -172,7 +167,7 @@ const char* BroFile::Name() const
 	return 0;
 	}
 
-bool BroFile::Open(FILE* file)
+bool BroFile::Open(FILE* file, const char* mode)
 	{
 	open_time = network_time ? network_time : current_time();
 
@@ -196,7 +191,12 @@ bool BroFile::Open(FILE* file)
 	InstallRotateTimer();
 
 	if ( ! f )
+		{
+		if ( ! mode )
 			f = fopen(name, access);
+		else
+			f = fopen(name, mode);
+		}
 
 	SetBuf(buffered);
 
@@ -846,8 +846,8 @@ BroFile* BroFile::Unserialize(UnserialInfo* info)
 			}
 		}
 
-	// Otherwise, open.
-	if ( ! file->Open() )
+	// Otherwise, open, but don't clobber.
+	if ( ! file->Open(0, "a") )
 		{
 		info->s->Error(fmt("cannot open %s: %s",
 					file->name, strerror(errno)));

src/File.h

@@ -87,7 +87,13 @@ protected:
 
 	BroFile()	{ Init(); }
 	void Init();
-	bool Open(FILE* f = 0);	// if file is given, it's an open file to use
+
+	/**
+	 * If file is given, it's an open file to use already.
+	 * If file is not given and mode is, the filename will be opened with that
+	 * access mode.
+	 */
+	bool Open(FILE* f = 0, const char* mode = 0);
 
 	BroFile* Prev()	{ return prev; }
 	BroFile* Next()	{ return next; }

src/NetVar.cc

@@ -167,6 +167,7 @@ TableVal* preserve_orig_addr;
 TableVal* preserve_resp_addr;
 TableVal* preserve_other_addr;
 
+int max_files_in_cache;
 double log_rotate_interval;
 double log_max_size;
 RecordType* rotate_info;
@@ -257,6 +258,7 @@ void init_general_global_var()
 	state_dir = internal_val("state_dir")->AsStringVal();
 	state_write_delay = opt_internal_double("state_write_delay");
 
+	max_files_in_cache = opt_internal_int("max_files_in_cache");
 	log_rotate_interval = opt_internal_double("log_rotate_interval");
 	log_max_size = opt_internal_double("log_max_size");
 	rotate_info = internal_type("rotate_info")->AsRecordType();

src/NetVar.h

@@ -170,6 +170,7 @@ extern double connection_status_update_interval;
 extern StringVal* state_dir;
 extern double state_write_delay;
 
+extern int max_files_in_cache;
 extern double log_rotate_interval;
 extern double log_max_size;
 extern RecordType* rotate_info;

testing/btest/Baseline/core.file-caching-serialization/one0

@@ -0,0 +1,4 @@
+opened
+write 0
+write 3
+write 6

testing/btest/Baseline/core.file-caching-serialization/one1

@@ -0,0 +1,4 @@
+opened
+write 1
+write 4
+write 7

testing/btest/Baseline/core.file-caching-serialization/one2

@@ -0,0 +1,4 @@
+opened
+write 2
+write 5
+write 8

testing/btest/Baseline/core.file-caching-serialization/two0

@@ -0,0 +1,6 @@
+opened
+write 0
+opened
+write 3
+opened
+write 6

testing/btest/Baseline/core.file-caching-serialization/two1

@@ -0,0 +1,6 @@
+opened
+write 1
+opened
+write 4
+opened
+write 7

testing/btest/Baseline/core.file-caching-serialization/two2

@@ -0,0 +1,6 @@
+opened
+write 2
+opened
+write 5
+opened
+write 8

testing/btest/core/file-caching-serialization.test

@@ -0,0 +1,49 @@
+# This checks that the interactions between open-file caching and
+# serialization works ok.  In the first case, all files can fit
+# in the cache, but get serialized before every write.  In the
+# second case, files are eventually forced out of the cache and
+# undergo serialization, which requires re-opening.
+
+# @TEST-EXEC: bro -b %INPUT "test_file_prefix=one"
+# @TEST-EXEC: btest-diff one0
+# @TEST-EXEC: btest-diff one1
+# @TEST-EXEC: btest-diff one2
+# @TEST-EXEC: bro -b %INPUT "test_file_prefix=two" "max_files_in_cache=2"
+# @TEST-EXEC: btest-diff two0
+# @TEST-EXEC: btest-diff two1
+# @TEST-EXEC: btest-diff two2
+
+const test_file_prefix = "" &redef;
+global file_table: table[string] of file;
+global iterations: vector of count = vector(0,1,2,3,4,5,6,7,8);
+
+function write_to_file(c: count)
+	{
+	local f: file;
+	# Take turns writing across three output files.
+	local filename = fmt("%s%s", test_file_prefix, c % 3 );
+
+	if ( filename in file_table )
+		f = file_table[filename];
+	else
+		{
+		f = open(filename);
+		file_table[filename] = f;
+		}
+
+	# This when block is a trick to get the frame cloned
+	# and thus serialize the local file value
+	when ( local s = fmt("write %d", c) )
+		print f, s;
+	}
+
+event file_opened(f: file)
+	{
+	print f, "opened";
+	}
+
+event bro_init()
+	{
+	for ( i in iterations )
+		write_to_file(iterations[i]);
+	}