Merge remote-tracking branch 'origin/topic/johanna/config-framework-fixes'

* origin/topic/johanna/config-framework-fixes:
  Fix test that fails now that options are automatically redefable.
  Make options redef-able by default.
  Ascii formatter: do not complain about port text.
  Make parsing of booleans a little bit more lenient.

CHANGES

@@ -1,4 +1,18 @@
 
+2.5-839 | 2018-08-13 10:51:43 -0500
+
+  * Make options redef-able by default. (Johanna Amann, Corelight)
+
+  * Fix incorrect input framework warnings when parsing ports.
+    (Johanna Amann, Corelight)
+
+  * Allow input framework to accept 0 and 1 as valid boolean values.
+    (Johanna Amann, Corelight)
+
+  * Improve the travis-job script to work outside of Travis (Daniel Thayer)
+
+  * Fix validate-certs.bro comments (Jon Siwek, Corelight)
+
 2.5-831 | 2018-08-10 17:12:53 -0500
 
   * Immediately apply broker subscriptions made during bro_init()

VERSION

@@ -1 +1 @@
-2.5-831
+2.5-839

src/ID.cc

@@ -294,6 +294,22 @@ void ID::RemoveAttr(attr_tag a)
 		}
 	}
 
+void ID::SetOption()
+	{
+	if ( is_option )
+		return;
+
+	is_option = true;
+
+	// option implied redefinable
+	if ( ! IsRedefinable() )
+		{
+		attr_list* attr = new attr_list;
+		attr->append(new Attr(ATTR_REDEF));
+		AddAttrs(new Attributes(attr, Type(), false));
+		}
+	}
+
 void ID::EvalFunc(Expr* ef, Expr* ev)
 	{
 	Expr* arg1 = new ConstExpr(val->Ref());

src/ID.h

@@ -60,7 +60,7 @@ public:
 	void SetConst()			{ is_const = true; }
 	bool IsConst() const		{ return is_const; }
 
-	void SetOption()			{ is_option = true; }
+	void SetOption();
 	bool IsOption() const		{ return is_option; }
 
 	void SetEnumConst()		{ is_enum_const = true; }

src/threading/formatters/Ascii.cc

@@ -227,9 +227,9 @@ threading::Value* Ascii::ParseValue(const string& s, const string& name, TypeTag
 		}
 
 	case TYPE_BOOL:
-		if ( s == "T" )
+		if ( s == "T" || s == "1" )
 			val->val.int_val = 1;
-		else if ( s == "F" )
+		else if ( s == "F" || s == "0" )
 			val->val.int_val = 0;
 		else
 			{
@@ -261,8 +261,10 @@ threading::Value* Ascii::ParseValue(const string& s, const string& name, TypeTag
 		break;
 
 	case TYPE_PORT:
+		{
 		val->val.port_val.proto = TRANSPORT_UNKNOWN;
 		pos = s.find('/');
+		string numberpart;
 		if ( pos != std::string::npos && s.length() > pos + 1 )
 			{
 			auto proto = s.substr(pos+1);
@@ -272,10 +274,21 @@ threading::Value* Ascii::ParseValue(const string& s, const string& name, TypeTag
 				val->val.port_val.proto = TRANSPORT_UDP;
 			else if ( strtolower(proto) == "icmp" )
 				val->val.port_val.proto = TRANSPORT_ICMP;
+			else if ( strtolower(proto) == "unknown" )
+				val->val.port_val.proto = TRANSPORT_UNKNOWN;
+			else
+				GetThread()->Warning(GetThread()->Fmt("Port '%s' contained unknown protocol '%s'", s.c_str(), proto.c_str()));
+			}
+
+		if ( pos != std::string::npos && pos > 0 )
+			{
+			numberpart = s.substr(0, pos);
+			start = numberpart.c_str();
 			}
 		val->val.port_val.port = strtoull(start, &end, 10);
 		if ( CheckNumberError(start, end) )
 			goto parse_error;
+		}
 		break;
 
 	case TYPE_SUBNET:

testing/btest/Baseline/core.option-errors-4/.stderr

@@ -1 +0,0 @@
-error in /Users/johanna/corelight/bro/testing/btest/.tmp/core.option-errors-4/option-errors.bro, line 2 and /Users/johanna/corelight/bro/testing/btest/.tmp/core.option-errors-4/option-errors.bro, line 3: already defined (testopt)

testing/btest/Baseline/core.option-redef/.stdout

@@ -1 +1,2 @@
 6
+7

testing/btest/Baseline/scripts.base.frameworks.config.basic/bro..stderr

@@ -0,0 +1 @@
+received termination signal

testing/btest/Baseline/scripts.base.frameworks.config.basic/bro.config.log

@@ -3,21 +3,23 @@
 #empty_field	(empty)
 #unset_field	-
 #path	config
-#open	2017-10-11-20-23-11
+#open	2018-08-10-18-16-52
 #fields	ts	id	old_value	new_value	location
 #types	time	string	string	string	string
-1507753391.587107	testbool	T	F	../configfile
+1533925012.140634	testbool	T	F	../configfile
-1507753391.587107	testcount	0	1	../configfile
+1533925012.140634	testcount	0	1	../configfile
-1507753391.587107	testcount	1	2	../configfile
+1533925012.140634	testcount	1	2	../configfile
-1507753391.587107	testint	0	-1	../configfile
+1533925012.140634	testint	0	-1	../configfile
-1507753391.587107	testenum	SSH::LOG	Conn::LOG	../configfile
+1533925012.140634	testenum	SSH::LOG	Conn::LOG	../configfile
-1507753391.587107	testport	42/tcp	45/unknown	../configfile
+1533925012.140634	testport	42/tcp	45/unknown	../configfile
-1507753391.587107	testaddr	127.0.0.1	127.0.0.1	../configfile
+1533925012.140634	testporttcp	40/udp	42/tcp	../configfile
-1507753391.587107	testaddr	127.0.0.1	2607:f8b0:4005:801::200e	../configfile
+1533925012.140634	testportudp	40/tcp	42/udp	../configfile
-1507753391.587107	testinterval	1.0 sec	60.0	../configfile
+1533925012.140634	testaddr	127.0.0.1	127.0.0.1	../configfile
-1507753391.587107	testtime	0.0	1507321987.0	../configfile
+1533925012.140634	testaddr	127.0.0.1	2607:f8b0:4005:801::200e	../configfile
-1507753391.587107	test_set	(empty)	b,c,a,d,erdbeerschnitzel	../configfile
+1533925012.140634	testinterval	1.0 sec	60.0	../configfile
-1507753391.587107	test_vector	(empty)	1,2,3,4,5,6	../configfile
+1533925012.140634	testtime	0.0	1507321987.0	../configfile
-1507753391.587107	test_set	b,c,a,d,erdbeerschnitzel	(empty)	../configfile
+1533925012.140634	test_set	(empty)	b,c,a,d,erdbeerschnitzel	../configfile
-1507753391.587107	test_set	(empty)	\x2d	../configfile
+1533925012.140634	test_vector	(empty)	1,2,3,4,5,6	../configfile
-#close	2017-10-11-20-23-11
+1533925012.140634	test_set	b,c,a,d,erdbeerschnitzel	(empty)	../configfile
+1533925012.140634	test_set	(empty)	\x2d	../configfile
+#close	2018-08-10-18-16-52

testing/btest/Baseline/scripts.base.frameworks.input.basic/out

@@ -1,5 +1,5 @@
 {
-[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, pp=5/icmp, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, ns=4242, sc={
+[-42] = [b=T, bt=T, e=SSH::LOG, c=21, p=123/unknown, pp=5/icmp, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, ns=4242, sc={
 2,
 4,
 1,

testing/btest/Baseline/scripts.base.frameworks.input.port-embedded/bro..stderr

@@ -0,0 +1,2 @@
+warning: ../input.log/Input::READER_ASCII: Port '50/trash' contained unknown protocol 'trash'
+received termination signal

testing/btest/Baseline/scripts.base.frameworks.input.port-embedded/bro..stdout

@@ -0,0 +1,4 @@
+[i=1.2.3.4], [p=80/tcp]
+[i=1.2.3.5], [p=52/udp]
+[i=1.2.3.6], [p=30/unknown]
+[i=1.2.3.7], [p=50/unknown]

testing/btest/core/option-errors.bro

@@ -11,8 +11,3 @@ option testbool : bool;
 
 option testopt = 5;
 testopt = 6;
-
-@TEST-START-NEXT
-
-option testopt = 5;
-redef testopt = 6;

testing/btest/core/option-redef.bro

@@ -2,11 +2,15 @@
 # @TEST-EXEC: btest-diff .stdout
 
 # options are allowed to be redef-able.
+# And they are even redef-able by default.
 
 option testopt = 5 &redef;
 redef testopt = 6;
+option anotheropt = 6;
+redef anotheropt = 7;
 
 event bro_init() {
 	print testopt;
+	print anotheropt;
 }
 

testing/btest/scripts/base/frameworks/config/basic.bro

@@ -1,6 +1,7 @@
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
 # @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff bro/config.log
+# @TEST-EXEC: btest-diff bro/.stderr
 
 @load base/frameworks/config
 @load base/protocols/conn
@@ -16,6 +17,8 @@ testcount 2
 testint		-1
 testenum Conn::LOG
 testport 45
+testporttcp 42/tcp
+testportudp 42/udp
 testaddr 127.0.0.1
 testaddr 2607:f8b0:4005:801::200e
 testinterval 60
@@ -35,6 +38,8 @@ export {
 	option testint: int = 0;
 	option testenum = SSH::LOG;
 	option testport = 42/tcp;
+	option testporttcp = 40/udp;
+	option testportudp = 40/tcp;
 	option testaddr = 127.0.0.1;
 	option testtime = network_time();
 	option testinterval = 1sec;

testing/btest/scripts/base/frameworks/input/basic.bro

@@ -7,9 +7,9 @@ redef exit_only_after_terminate = T;
 @TEST-START-FILE input.log
 #separator \x09
 #path	ssh
-#fields	b	i	e	c	p	pp	sn	a	d	t	iv	s	sc	ss	se	vc	ve	ns
+#fields	b	bt	i	e	c	p	pp	sn	a	d	t	iv	s	sc	ss	se	vc	ve	ns
 #types	bool	int	enum	count	port	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	string
-T	-42	SSH::LOG	21	123	5/icmp	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	4242
+T	1	-42	SSH::LOG	21	123	5/icmp	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	4242
 @TEST-END-FILE
 
 @load base/protocols/ssh
@@ -26,6 +26,7 @@ type Idx: record {
 
 type Val: record {
 	b: bool;
+	bt: bool;
 	e: Log::ID;
 	c: count;
 	p: port;

testing/btest/scripts/base/frameworks/input/port-embedded.bro

@@ -0,0 +1,44 @@
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait 10
+# @TEST-EXEC: btest-diff bro/.stdout
+# @TEST-EXEC: btest-diff bro/.stderr
+
+@TEST-START-FILE input.log
+#fields	i	p
+1.2.3.4	80/tcp
+1.2.3.5	52/udp
+1.2.3.6	30/unknown
+1.2.3.7	50/trash
+@TEST-END-FILE
+
+redef exit_only_after_terminate = T;
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: addr;
+};
+
+type Val: record {
+	p: port;
+};
+
+global servers: table[addr] of Val = table();
+
+event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val)
+	{
+	print left, right;
+	}
+
+event bro_init()
+	{
+	Input::add_table([$source="../input.log", $name="input", $idx=Idx, $val=Val, $ev=line, $destination=servers]);
+	}
+
+event Input::end_of_data(name: string, source: string)
+	{
+	Input::remove("input");
+	terminate();
+	}

testing/btest/scripts/base/frameworks/input/reread.bro

@@ -43,7 +43,7 @@ T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz
 F	-43	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
 F	-44	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
 F	-45	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
-F	-46	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+0	-46	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
 F	-47	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
 F	-48	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
 @TEST-END-FILE