Merge remote-tracking branch 'origin/topic/bernhard/input-update'

Closes #1021.

* origin/topic/bernhard/input-update:
  this event handler fails the unused-event-handlers test because it is a bit of a special case.
  ...and fix the event ordering issue. Dispatch != QueueEvent
  add Terminate to input framework to prevent potential shutdown race-conditions.
  fix warning.
  fix stderr test. ls behaves differently on errors on linux...
  small fixes.
  linux does not have strnstr
  and close only fds that are currently open (the logging framework really did not like that :) )
  A bunch of more changes for the raw reader
  make reading from stdout and stderr simultaneously work.
  allow sending data to stdin of child process
  Streaming reads from external commands work without blocking anything.
  replace popen with fork and exec.
  change raw reader to use basic c io instead of fdstream encapsulation class.

testing/btest/scripts/base/frameworks/input/raw/executestdin.bro

@@ -0,0 +1,44 @@
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff test.txt
+# @TEST-EXEC: btest-diff out
+
+redef exit_only_after_terminate = T;
+@load base/frameworks/communication  # let network-time run. otherwise there are no heartbeats...
+
+global outfile: file;
+global try: count;
+
+module A;
+
+type Val: record {
+	s: string;
+};
+
+event line(description: Input::EventDescription, tpe: Input::Event, s: string)
+	{
+	print outfile, description;
+	print outfile, tpe;
+	print outfile, s;
+	try = try + 1;
+	if ( try == 2 )
+		{
+		Input::remove("input2");
+		close(outfile);
+		terminate();
+		}
+	}
+
+event bro_init()
+	{
+	local config_strings: table[string] of string = {
+		["stdin"] = "hello\nthere\1\2\3\4\5\1\2\3yay"
+		#["stdin"] = "yay"
+	};
+
+	try = 0;
+	outfile = open("../out");
+	Input::add_event([$source="cat > ../test.txt |", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line, $want_record=F, $config=config_strings]);
+	Input::remove("input");
+	Input::add_event([$source="cat |", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input2", $fields=Val, $ev=line, $want_record=F, $config=config_strings]);
+	}

testing/btest/scripts/base/frameworks/input/raw/executestream.bro

@@ -0,0 +1,61 @@
+# @TEST-EXEC: cp input1.log input.log
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT 
+# @TEST-EXEC: sleep 3
+# @TEST-EXEC: cat input2.log >> input.log
+# @TEST-EXEC: sleep 3
+# @TEST-EXEC: cat input3.log >> input.log
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+redef exit_only_after_terminate = T;
+
+@TEST-START-FILE input1.log
+sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF
+@TEST-END-FILE
+
+@TEST-START-FILE input2.log
+DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF
+q3r3057fdf
+@TEST-END-FILE
+
+@TEST-START-FILE input3.log
+sdfs\d
+
+dfsdf
+sdf
+3rw43wRRERLlL#RWERERERE.
+@TEST-END-FILE
+
+@load base/frameworks/communication  # let network-time run
+
+module A;
+
+type Val: record {
+	s: string;
+};
+
+global try: count;
+global outfile: file;
+
+event line(description: Input::EventDescription, tpe: Input::Event, s: string)
+	{
+	print outfile, description;
+	print outfile, tpe;
+	print outfile, s;
+
+	try = try + 1;
+	if ( try == 8 )
+		{
+		print outfile, "done";
+		close(outfile);
+		Input::remove("input");
+		terminate();
+		}
+	}
+
+event bro_init()
+	{
+	outfile = open("../out");
+	try = 0;
+	Input::add_event([$source="tail -f ../input.log |", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line, $want_record=F]);
+	}

testing/btest/scripts/base/frameworks/input/raw/long.bro

@@ -0,0 +1,37 @@
+# @TEST-EXEC: dd if=/dev/zero of=input.log bs=8193 count=1
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+#
+# this test should be longer than one block-size. to test behavior of input-reader if it has to re-allocate stuff.
+
+redef exit_only_after_terminate = T;
+
+global outfile: file;
+global try: count;
+
+module A;
+
+type Val: record {
+	s: string;
+};
+
+event line(description: Input::EventDescription, tpe: Input::Event, s: string)
+	{
+	print outfile, tpe;
+	print outfile, |s|;
+	try = try + 1;
+	if ( try == 1 )
+		{
+		close(outfile);
+		terminate();
+		}
+	}
+
+event bro_init()
+	{
+	try = 0;
+	outfile = open("../out");
+	Input::add_event([$source="../input.log", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line, $want_record=F]);
+	Input::remove("input");
+	}

testing/btest/scripts/base/frameworks/input/raw/stderr.bro

@@ -0,0 +1,66 @@
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT 
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+redef exit_only_after_terminate = T;
+
+type Val: record {
+	s: string;
+	is_stderr: bool;
+};
+
+global try: count;
+global outfile: file;
+
+event line(description: Input::EventDescription, tpe: Input::Event, s: string, is_stderr: bool)
+	{
+	print outfile, tpe;
+	if ( is_stderr ) 
+		{
+		# work around localized error messages. and if some localization does not include the filename... well... that would be bad :)
+		if ( strstr(s, "nonexistant") > 0 ) 
+			{
+			print outfile, "stderr output contained nonexistant";
+			}
+		}
+	else
+		{
+		print outfile, s;
+		}
+	print outfile, is_stderr;
+
+	try = try + 1;
+	if ( try == 7 )
+		{
+		print outfile, "done";
+		Input::remove("input");
+		}
+	}
+
+event Input::end_of_data(name: string, source:string)
+	{
+	print outfile, "End of Data event";
+	print outfile, name;
+	terminate(); # due to the current design, end_of_data will be called after process_finshed and all line events.
+	# this could potentially change
+	}
+
+event InputRaw::process_finished(name: string, source:string, exit_code:count, signal_exit:bool)
+	{
+	print outfile, "Process finished event";
+	print outfile, name;
+	if ( exit_code != 0 ) 
+		print outfile, "Exit code != 0";
+	}
+
+event bro_init()
+	{
+
+	local config_strings: table[string] of string = {
+		["read_stderr"] = "1"
+	};
+
+	outfile = open("../out");
+	try = 0;
+	Input::add_event([$source="ls .. ../nonexistant ../nonexistant2 ../nonexistant3 |", $reader=Input::READER_RAW, $name="input", $fields=Val, $ev=line, $want_record=F, $config=config_strings]);
+	}