Merge remote-tracking branch 'origin/topic/seth/bittorrent-fix-and-dpd-sig-breakout'

Closes #1035.

* origin/topic/seth/bittorrent-fix-and-dpd-sig-breakout:
  Small test fixes.
  Added a missing curly brace in smtp/dpd.sig
  Fix a bug where the same analyzer tag was reused for two different analyzers.
  Moved DPD signatures into script specific directories.

scripts/base/frameworks/dpd/dpd.sig

@@ -1,212 +0,0 @@
-# Signatures to initiate dynamic protocol detection.
-
-signature dpd_ftp_client {
-  ip-proto == tcp
-  payload /(|.*[\n\r]) *[uU][sS][eE][rR] /
-  tcp-state originator
-}
-
-# Match for server greeting (220, 120) and for login or passwd
-# required (230, 331).
-signature dpd_ftp_server {
-  ip-proto == tcp
-  payload /[\n\r ]*(120|220)[^0-9].*[\n\r] *(230|331)[^0-9]/
-  tcp-state responder
-  requires-reverse-signature dpd_ftp_client
-  enable "ftp"
-}
-
-signature dpd_http_client {
-  ip-proto == tcp
-  payload /^[[:space:]]*(GET|HEAD|POST)[[:space:]]*/
-  tcp-state originator
-}
-
-signature dpd_http_server {
-  ip-proto == tcp
-  payload /^HTTP\/[0-9]/
-  tcp-state responder
-  requires-reverse-signature dpd_http_client
-  enable "http"
-}
-
-signature dpd_bittorrenttracker_client {
-  ip-proto == tcp
-  payload /^.*\/announce\?.*info_hash/
-  tcp-state originator
-}
-
-signature dpd_bittorrenttracker_server {
-  ip-proto == tcp
-  payload /^HTTP\/[0-9]/
-  tcp-state responder
-  requires-reverse-signature dpd_bittorrenttracker_client
-  enable "bittorrenttracker"
-}
-
-signature dpd_bittorrent_peer1 {
-  ip-proto == tcp
-  payload /^\x13BitTorrent protocol/
-  tcp-state originator
-}
-
-signature dpd_bittorrent_peer2 {
-  ip-proto == tcp
-  payload /^\x13BitTorrent protocol/
-  tcp-state responder
-  requires-reverse-signature dpd_bittorrent_peer1
-  enable "bittorrent"
-}
-
-signature irc_client1 {
-  ip-proto == tcp
-  payload /(|.*[\r\n]) *[Uu][Ss][Ee][Rr] +.+[\n\r]+ *[Nn][Ii][Cc][Kk] +.*[\r\n]/
-  requires-reverse-signature irc_server_reply
-  tcp-state originator
-  enable "irc"
-}
-
-signature irc_client2 {
-  ip-proto == tcp
-  payload /(|.*[\r\n]) *[Nn][Ii][Cc][Kk] +.+[\r\n]+ *[Uu][Ss][Ee][Rr] +.+[\r\n]/
-  requires-reverse-signature irc_server_reply
-  tcp-state originator
-  enable "irc"
-}
-
-signature irc_server_reply {
-  ip-proto == tcp
-  payload /^(|.*[\n\r])(:[^ \n\r]+ )?[0-9][0-9][0-9] /
-  tcp-state responder
-}
-
-signature irc_server_to_server1 {
-  ip-proto == tcp
-  payload /(|.*[\r\n]) *[Ss][Ee][Rr][Vv][Ee][Rr] +[^ ]+ +[0-9]+ +:.+[\r\n]/
-}
-
-signature irc_server_to_server2 {
-  ip-proto == tcp
-  payload /(|.*[\r\n]) *[Ss][Ee][Rr][Vv][Ee][Rr] +[^ ]+ +[0-9]+ +:.+[\r\n]/
-  requires-reverse-signature irc_server_to_server1
-  enable "irc"
-}
-
-signature dpd_smtp_client {
-  ip-proto == tcp
-  payload /(|.*[\n\r])[[:space:]]*([hH][eE][lL][oO]|[eE][hH][lL][oO])/
-  requires-reverse-signature dpd_smtp_server
-  enable "smtp"
-  tcp-state originator
-}
-
-signature dpd_smtp_server {
-  ip-proto == tcp
-  payload /^[[:space:]]*220[[:space:]-]/
-  tcp-state responder
-}
-
-signature dpd_ssh_client {
-  ip-proto == tcp
-  payload /^[sS][sS][hH]-/
-  requires-reverse-signature dpd_ssh_server
-  enable "ssh"
-  tcp-state originator
-}
-
-signature dpd_ssh_server {
-  ip-proto == tcp
-  payload /^[sS][sS][hH]-/
-  tcp-state responder
-}
-
-signature dpd_pop3_server {
-  ip-proto == tcp
-  payload /^\+OK/
-  requires-reverse-signature dpd_pop3_client
-  enable "pop3"
-  tcp-state responder
-}
-
-signature dpd_pop3_client {
-  ip-proto == tcp
-  payload /(|.*[\r\n])[[:space:]]*([uU][sS][eE][rR][[:space:]]|[aA][pP][oO][pP][[:space:]]|[cC][aA][pP][aA]|[aA][uU][tT][hH])/
-  tcp-state originator
-}
-
-signature dpd_ssl_server {
-  ip-proto == tcp
-  # Server hello.
-  payload /^(\x16\x03[\x00\x01\x02]..\x02...\x03[\x00\x01\x02]|...?\x04..\x00\x02).*/
-  requires-reverse-signature dpd_ssl_client
-  enable "ssl"
-  tcp-state responder
-}
-
-signature dpd_ssl_client {
-  ip-proto == tcp
-  # Client hello.
-  payload /^(\x16\x03[\x00\x01\x02]..\x01...\x03[\x00\x01\x02]|...?\x01[\x00\x01\x02][\x02\x03]).*/
-  tcp-state originator
-}
-
-signature dpd_ayiya {
-  ip-proto = udp
-  payload /^..\x11\x29/
-  enable "ayiya"
-}
-
-signature dpd_teredo {
-  ip-proto = udp
-  payload /^(\x00\x00)|(\x00\x01)|([\x60-\x6f])/
-  enable "teredo"
-}
-
-signature dpd_socks4_client {
-	ip-proto == tcp
-	# '32' is a rather arbitrary max length for the user name.
-	payload /^\x04[\x01\x02].{0,32}\x00/
-	tcp-state originator
-}
-
-signature dpd_socks4_server {
-	ip-proto == tcp
-	requires-reverse-signature dpd_socks4_client
-	payload /^\x00[\x5a\x5b\x5c\x5d]/
-	tcp-state responder
-	enable "socks"
-}
-
-signature dpd_socks4_reverse_client {
-	ip-proto == tcp
-	# '32' is a rather arbitrary max length for the user name.
-	payload /^\x04[\x01\x02].{0,32}\x00/
-	tcp-state responder
-}
-
-signature dpd_socks4_reverse_server {
-	ip-proto == tcp
-	requires-reverse-signature dpd_socks4_reverse_client
-	payload /^\x00[\x5a\x5b\x5c\x5d]/
-	tcp-state originator
-	enable "socks"
-}
-
-signature dpd_socks5_client {
-	ip-proto == tcp
-	# Watch for a few authentication methods to reduce false positives.
-	payload /^\x05.[\x00\x01\x02]/
-	tcp-state originator
-}
-
-signature dpd_socks5_server {
-	ip-proto == tcp
-	requires-reverse-signature dpd_socks5_client
-	# Watch for a single authentication method to be chosen by the server or
-	# the server to indicate the no authentication is required.
-	payload /^\x05(\x00|\x01[\x00\x01\x02])/
-	tcp-state responder
-	enable "socks"
-}
-
-

scripts/base/frameworks/dpd/main.bro

@@ -3,8 +3,6 @@
 
 module DPD;
 
-@load-sigs ./dpd.sig
-
 export {
 	## Add the DPD logging stream identifier.
 	redef enum Log::ID += { LOG };

scripts/base/init-default.bro

@@ -46,5 +46,6 @@
 @load base/protocols/ssh
 @load base/protocols/ssl
 @load base/protocols/syslog
+@load base/protocols/tunnels
 
 @load base/misc/find-checksum-offloading

scripts/base/protocols/ftp/__load__.bro

@@ -3,3 +3,5 @@
 @load ./file-analysis
 @load ./file-extract
 @load ./gridftp
+
+@load-sigs ./dpd.sig

scripts/base/protocols/ftp/dpd.sig

@@ -0,0 +1,15 @@
+signature dpd_ftp_client {
+  ip-proto == tcp
+  payload /(|.*[\n\r]) *[uU][sS][eE][rR] /
+  tcp-state originator
+}
+
+# Match for server greeting (220, 120) and for login or passwd
+# required (230, 331).
+signature dpd_ftp_server {
+  ip-proto == tcp
+  payload /[\n\r ]*(120|220)[^0-9].*[\n\r] *(230|331)[^0-9]/
+  tcp-state responder
+  requires-reverse-signature dpd_ftp_client
+  enable "ftp"
+}

scripts/base/protocols/http/__load__.bro

@@ -4,3 +4,5 @@
 @load ./file-ident
 @load ./file-hash
 @load ./file-extract
+
+@load-sigs ./dpd.sig

scripts/base/protocols/http/dpd.sig

@@ -0,0 +1,13 @@
+signature dpd_http_client {
+  ip-proto == tcp
+  payload /^[[:space:]]*(GET|HEAD|POST)[[:space:]]*/
+  tcp-state originator
+}
+
+signature dpd_http_server {
+  ip-proto == tcp
+  payload /^HTTP\/[0-9]/
+  tcp-state responder
+  requires-reverse-signature dpd_http_client
+  enable "http"
+}

scripts/base/protocols/irc/__load__.bro

@@ -1,3 +1,5 @@
 @load ./main
 @load ./dcc-send
 @load ./file-analysis
+
+@load-sigs ./dpd.sig

scripts/base/protocols/irc/dpd.sig

@@ -0,0 +1,33 @@
+signature irc_client1 {
+  ip-proto == tcp
+  payload /(|.*[\r\n]) *[Uu][Ss][Ee][Rr] +.+[\n\r]+ *[Nn][Ii][Cc][Kk] +.*[\r\n]/
+  requires-reverse-signature irc_server_reply
+  tcp-state originator
+  enable "irc"
+}
+
+signature irc_client2 {
+  ip-proto == tcp
+  payload /(|.*[\r\n]) *[Nn][Ii][Cc][Kk] +.+[\r\n]+ *[Uu][Ss][Ee][Rr] +.+[\r\n]/
+  requires-reverse-signature irc_server_reply
+  tcp-state originator
+  enable "irc"
+}
+
+signature irc_server_reply {
+  ip-proto == tcp
+  payload /^(|.*[\n\r])(:[^ \n\r]+ )?[0-9][0-9][0-9] /
+  tcp-state responder
+}
+
+signature irc_server_to_server1 {
+  ip-proto == tcp
+  payload /(|.*[\r\n]) *[Ss][Ee][Rr][Vv][Ee][Rr] +[^ ]+ +[0-9]+ +:.+[\r\n]/
+}
+
+signature irc_server_to_server2 {
+  ip-proto == tcp
+  payload /(|.*[\r\n]) *[Ss][Ee][Rr][Vv][Ee][Rr] +[^ ]+ +[0-9]+ +:.+[\r\n]/
+  requires-reverse-signature irc_server_to_server1
+  enable "irc"
+}

scripts/base/protocols/smtp/__load__.bro

@@ -2,3 +2,5 @@
 @load ./entities
 @load ./entities-excerpt
 @load ./file-analysis
+
+@load-sigs ./dpd.sig

scripts/base/protocols/smtp/dpd.sig

@@ -0,0 +1,13 @@
+signature dpd_smtp_client {
+  ip-proto == tcp
+  payload /(|.*[\n\r])[[:space:]]*([hH][eE][lL][oO]|[eE][hH][lL][oO])/
+  requires-reverse-signature dpd_smtp_server
+  enable "smtp"
+  tcp-state originator
+}
+
+signature dpd_smtp_server {
+  ip-proto == tcp
+  payload /^[[:space:]]*220[[:space:]-]/
+  tcp-state responder
+}

scripts/base/protocols/socks/__load__.bro

@@ -1,2 +1,4 @@
 @load ./consts
 @load ./main
+
+@load-sigs ./dpd.sig

scripts/base/protocols/socks/dpd.sig

@@ -0,0 +1,48 @@
+signature dpd_socks4_client {
+	ip-proto == tcp
+	# '32' is a rather arbitrary max length for the user name.
+	payload /^\x04[\x01\x02].{0,32}\x00/
+	tcp-state originator
+}
+
+signature dpd_socks4_server {
+	ip-proto == tcp
+	requires-reverse-signature dpd_socks4_client
+	payload /^\x00[\x5a\x5b\x5c\x5d]/
+	tcp-state responder
+	enable "socks"
+}
+
+signature dpd_socks4_reverse_client {
+	ip-proto == tcp
+	# '32' is a rather arbitrary max length for the user name.
+	payload /^\x04[\x01\x02].{0,32}\x00/
+	tcp-state responder
+}
+
+signature dpd_socks4_reverse_server {
+	ip-proto == tcp
+	requires-reverse-signature dpd_socks4_reverse_client
+	payload /^\x00[\x5a\x5b\x5c\x5d]/
+	tcp-state originator
+	enable "socks"
+}
+
+signature dpd_socks5_client {
+	ip-proto == tcp
+	# Watch for a few authentication methods to reduce false positives.
+	payload /^\x05.[\x00\x01\x02]/
+	tcp-state originator
+}
+
+signature dpd_socks5_server {
+	ip-proto == tcp
+	requires-reverse-signature dpd_socks5_client
+	# Watch for a single authentication method to be chosen by the server or
+	# the server to indicate the no authentication is required.
+	payload /^\x05(\x00|\x01[\x00\x01\x02])/
+	tcp-state responder
+	enable "socks"
+}
+
+

scripts/base/protocols/ssh/__load__.bro

@@ -1 +1,3 @@
 @load ./main
+
+@load-sigs ./dpd.sig

scripts/base/protocols/ssh/dpd.sig

@@ -0,0 +1,13 @@
+signature dpd_ssh_client {
+  ip-proto == tcp
+  payload /^[sS][sS][hH]-/
+  requires-reverse-signature dpd_ssh_server
+  enable "ssh"
+  tcp-state originator
+}
+
+signature dpd_ssh_server {
+  ip-proto == tcp
+  payload /^[sS][sS][hH]-/
+  tcp-state responder
+}

scripts/base/protocols/ssl/__load__.bro

@@ -1,3 +1,5 @@
 @load ./consts
 @load ./main
 @load ./mozilla-ca-list
+
+@load-sigs ./dpd.sig

scripts/base/protocols/ssl/dpd.sig

@@ -0,0 +1,15 @@
+signature dpd_ssl_server {
+  ip-proto == tcp
+  # Server hello.
+  payload /^(\x16\x03[\x00\x01\x02]..\x02...\x03[\x00\x01\x02]|...?\x04..\x00\x02).*/
+  requires-reverse-signature dpd_ssl_client
+  enable "ssl"
+  tcp-state responder
+}
+
+signature dpd_ssl_client {
+  ip-proto == tcp
+  # Client hello.
+  payload /^(\x16\x03[\x00\x01\x02]..\x01...\x03[\x00\x01\x02]|...?\x01[\x00\x01\x02][\x02\x03]).*/
+  tcp-state originator
+}

scripts/base/protocols/tunnels/__load__.bro

@@ -0,0 +1 @@
+@load-sigs ./dpd.sig

scripts/base/protocols/tunnels/dpd.sig

@@ -0,0 +1,14 @@
+# Provide DPD signatures for tunneling protocols that otherwise
+# wouldn't be detected at all.
+
+signature dpd_ayiya {
+  ip-proto = udp
+  payload /^..\x11\x29/
+  enable "ayiya"
+}
+
+signature dpd_teredo {
+  ip-proto = udp
+  payload /^(\x00\x00)|(\x00\x01)|([\x60-\x6f])/
+  enable "teredo"
+}

src/analyzer/protocol/bittorrent/BitTorrentTracker.cc

@@ -22,7 +22,7 @@ static RecordType* bittorrent_benc_value;
 static TableType* bittorrent_benc_dir;
 
 BitTorrentTracker_Analyzer::BitTorrentTracker_Analyzer(Connection* c)
-: tcp::TCP_ApplicationAnalyzer("BITTORRENT", c)
+: tcp::TCP_ApplicationAnalyzer("BITTORRENTTRACKER", c)
 	{
 	if ( ! bt_tracker_headers )
 		{

src/analyzer/protocol/bittorrent/Plugin.cc

@@ -7,6 +7,6 @@
 BRO_PLUGIN_BEGIN(Bro, BitTorrent)
 	BRO_PLUGIN_DESCRIPTION("BitTorrent Analyzer");
 	BRO_PLUGIN_ANALYZER("BitTorrent", bittorrent::BitTorrent_Analyzer);
-	BRO_PLUGIN_ANALYZER("BitTorrentTracker", bittorrent::BitTorrent_Analyzer);
+	BRO_PLUGIN_ANALYZER("BitTorrentTracker", bittorrent::BitTorrentTracker_Analyzer);
 	BRO_PLUGIN_BIF_FILE(events);
 BRO_PLUGIN_END

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2013-07-05-05-21-48
+#open	2013-07-10-03-19-58
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -191,6 +191,7 @@ scripts/base/init-default.bro
   scripts/base/protocols/syslog/__load__.bro
     scripts/base/protocols/syslog/consts.bro
     scripts/base/protocols/syslog/main.bro
+  scripts/base/protocols/tunnels/__load__.bro
   scripts/base/misc/find-checksum-offloading.bro
 scripts/policy/misc/loaded-scripts.bro
-#close	2013-07-05-05-21-48
+#close	2013-07-10-03-19-58

testing/btest/core/tunnels/teredo-known-services.test

@@ -1,6 +1,6 @@
-# @TEST-EXEC: bro -r $TRACES/tunnels/false-teredo.pcap base/frameworks/dpd protocols/conn/known-services Tunnel::delay_teredo_confirmation=T "Site::local_nets+={192.168.1.0/24}"
+# @TEST-EXEC: bro -r $TRACES/tunnels/false-teredo.pcap base/frameworks/dpd base/protocols/tunnels protocols/conn/known-services Tunnel::delay_teredo_confirmation=T "Site::local_nets+={192.168.1.0/24}"
 # @TEST-EXEC: test ! -e known_services.log
-# @TEST-EXEC: bro -b -r $TRACES/tunnels/false-teredo.pcap base/frameworks/dpd protocols/conn/known-services Tunnel::delay_teredo_confirmation=F "Site::local_nets+={192.168.1.0/24}"
+# @TEST-EXEC: bro -b -r $TRACES/tunnels/false-teredo.pcap base/frameworks/dpd base/protocols/tunnels protocols/conn/known-services Tunnel::delay_teredo_confirmation=F "Site::local_nets+={192.168.1.0/24}"
 # @TEST-EXEC: btest-diff known_services.log
 
 # The first case using Tunnel::delay_teredo_confirmation=T doesn't produce