Ascii reader error changes - fix small bugs

The changes are now a bit more succinct with less code changes required.
Behavior is tested a little bit more thoroughly and a memory problem
when reading incomplete lines was fixed. ReadHeader also always directly
returns if header reading failed.

Error messages now are back to what they were before the change, if the
new behavior is not used.

I also tweaked the documentation text a bit.

testing/btest/scripts/base/frameworks/input/invalid-lines.bro

@@ -0,0 +1,67 @@
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait 10
+# @TEST-EXEC: btest-diff out
+
+redef exit_only_after_terminate = T;
+redef InputAscii::fail_on_invalid_lines = F;
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	ns
+#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	string
+T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30
+T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	4242
+T	-43	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	4242 HOHOHO
+T	-41
+@TEST-END-FILE
+
+@load base/protocols/ssh
+
+global outfile: file;
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	b: bool;
+	e: Log::ID;
+	c: count;
+	p: port;
+	sn: subnet;
+	a: addr;
+	d: double;
+	t: time;
+	iv: interval;
+	s: string;
+	ns: string;
+	sc: set[count];
+	ss: set[string];
+	se: set[string];
+	vc: vector of int;
+	ve: vector of int;
+};
+
+global servers: table[int] of Val = table();
+global servers2: table[int] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
+	Input::add_table([$source="../input.log", $name="ssh2", $idx=Idx, $val=Val, $destination=servers2, $config=table(["fail_on_invalid_lines"] = "T")]);
+	}
+
+event Input::end_of_data(name: string, source:string)
+	{
+	print outfile, servers;
+	Input::remove("ssh");
+	close(outfile);
+	terminate();
+	}

testing/btest/scripts/base/frameworks/input/invalidtext.bro

@@ -13,6 +13,7 @@
 @TEST-END-FILE
 
 redef exit_only_after_terminate = T;
+redef InputAscii::fail_on_invalid_lines = T;
 
 global outfile: file;
 

testing/btest/scripts/base/frameworks/input/missing-file-initially.bro

@@ -1,10 +1,12 @@
 # This tests files that don't exist initially and then do later during
 # runtime to make sure the ascii reader is resilient to files missing.
-# It does a second test at the same time which configures the old 
+# It does a second test at the same time which configures the old
 # failing behavior.
 
 # @TEST-EXEC: btest-bg-run bro bro %INPUT
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: sleep 2; cp does-exist.dat does-not-exist.dat
+# @TEST-EXEC: sleep 2; mv does-not-exist.dat does-not-exist-again.dat; echo "Streaming still works" >> does-not-exist-again.dat
+# @TEST-EXEC: btest-bg-wait -k 3
 # @TEST-EXEC: btest-diff bro/.stdout
 # @TEST-EXEC: btest-diff bro/.stderr
 
@@ -40,8 +42,8 @@ event line2(description: Input::EventDescription, tpe: Input::Event, v: Val)
 event bro_init()
 	{
 	Input::add_event([$source="../does-not-exist.dat", $name="input", $reader=Input::READER_ASCII, $mode=Input::REREAD, $fields=Val, $ev=line, $want_record=T]);
-	Input::add_event([$source="../does-not-exist.dat", $name="input2", $reader=Input::READER_ASCII, $mode=Input::REREAD, $fields=Val, $ev=line2, $want_record=T, 
+	Input::add_event([$source="../does-not-exist.dat", $name="inputstream", $reader=Input::READER_ASCII, $mode=Input::STREAM, $fields=Val, $ev=line, $want_record=T]);
+	Input::add_event([$source="../does-not-exist.dat", $name="inputmanual", $reader=Input::READER_ASCII, $mode=Input::MANUAL, $fields=Val, $ev=line, $want_record=T]);
+	Input::add_event([$source="../does-not-exist.dat", $name="input2", $reader=Input::READER_ASCII, $mode=Input::REREAD, $fields=Val, $ev=line2, $want_record=T,
 	                  $config=table(["fail_on_file_problem"] = "T")]);
-
-	system("sleep 2; mv ../does-exist.dat ../does-not-exist.dat;");
 	}

testing/btest/scripts/base/frameworks/input/missing-file.bro

@@ -3,6 +3,7 @@
 # @TEST-EXEC: btest-diff bro/.stderr
 
 redef exit_only_after_terminate = T;
+redef InputAscii::fail_on_file_problem = T;
 
 global outfile: file;
 global try: count;