include cleanup

The Zeek code base has very inconsistent #includes.  Many sources
included a few headers, and those headers included other headers, and
in the end, nearly everything is included everywhere, so missing
#includes were never noticed.  Another side effect was a lot of header
bloat which slows down the build.

First step to fix it: in each source file, its own header should be
included first to verify that each header's includes are correct, and
none is missing.

After adding the missing #includes, I replaced lots of #includes
inside headers with class forward declarations.  In most headers,
object pointers are never referenced, so declaring the function
prototypes with forward-declared classes is just fine.

This patch speeds up the build by 19%, because each compilation unit
gets smaller.  Here are the "time" numbers for a fresh build (with a
warm page cache but without ccache):

Before this patch:

 3144.94user 161.63system 3:02.87elapsed 1808%CPU (0avgtext+0avgdata 2168608maxresident)k
 760inputs+12008400outputs (1511major+57747204minor)pagefaults 0swaps

After this patch:

 2565.17user 141.83system 2:25.46elapsed 1860%CPU (0avgtext+0avgdata 1489076maxresident)k
 72576inputs+9130920outputs (1667major+49400430minor)pagefaults 0swaps

src/file_analysis/Component.h

@@ -6,10 +6,9 @@
 #include "plugin/Component.h"
 #include "plugin/TaggedComponent.h"
 
-#include "Val.h"
-
 #include "../zeek-config.h"
-#include "../util.h"
+
+class RecordVal;
 
 namespace file_analysis {
 

src/file_analysis/File.cc

@@ -1,9 +1,10 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 
-#include <string>
-#include <algorithm>
-
 #include "File.h"
+
+#include <utility>
+
+#include "FileReassembler.h"
 #include "FileTimer.h"
 #include "Analyzer.h"
 #include "Manager.h"

src/file_analysis/File.h

@@ -4,19 +4,20 @@
 
 #include <string>
 #include <utility>
-#include <vector>
 
-#include "FileReassembler.h"
-#include "Conn.h"
-#include "Val.h"
-#include "Tag.h"
 #include "AnalyzerSet.h"
 #include "BroString.h"
 #include "WeirdState.h"
 
+using std::string;
+
+class Connection;
+class RecordVal;
+
 namespace file_analysis {
 
 class FileReassembler;
+class Tag;
 
 /**
  * Wrapper class around \c fa_file record values from script layer.

src/file_analysis/FileReassembler.h

@@ -1,7 +1,6 @@
 #pragma once
 
 #include "Reassem.h"
-#include "File.h"
 
 class BroFile;
 class Connection;

src/file_analysis/FileTimer.cc

@@ -1,7 +1,8 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 
-#include "Manager.h"
+#include "FileTimer.h"
 #include "File.h"
+#include "Manager.h"
 
 using namespace file_analysis;
 

src/file_analysis/FileTimer.h

@@ -2,9 +2,12 @@
 
 #pragma once
 
-#include <string>
 #include "Timer.h"
 
+#include <string>
+
+using std::string;
+
 namespace file_analysis {
 
 /**

src/file_analysis/Manager.cc

@@ -1,9 +1,5 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 
-#include <vector>
-#include <string>
-#include <openssl/md5.h>
-
 #include "Manager.h"
 #include "File.h"
 #include "Analyzer.h"
@@ -15,6 +11,8 @@
 #include "plugin/Manager.h"
 #include "analyzer/Manager.h"
 
+#include <openssl/md5.h>
+
 using namespace file_analysis;
 
 TableVal* Manager::disabled = 0;

src/file_analysis/Manager.h

@@ -6,26 +6,30 @@
 #include <set>
 #include <map>
 
-#include "Dict.h"
+#include "Component.h"
 #include "Net.h"
-#include "Conn.h"
-#include "Val.h"
-#include "Analyzer.h"
-#include "Timer.h"
-#include "EventHandler.h"
 #include "RuleMatcher.h"
 
-#include "File.h"
-#include "FileTimer.h"
-#include "Component.h"
-#include "Tag.h"
 #include "plugin/ComponentManager.h"
-#include "analyzer/Tag.h"
 
 #include "file_analysis/file_analysis.bif.h"
 
+using std::map;
+using std::set;
+
+class TableVal;
+class VectorVal;
+
+namespace analyzer {
+class Analyzer;
+class Tag;
+}
+
 namespace file_analysis {
 
+class File;
+class Tag;
+
 /**
  * Main entry point for interacting with file analysis.
  */

src/file_analysis/Tag.h

@@ -3,13 +3,17 @@
 #pragma once
 
 #include "zeek-config.h"
-#include "util.h"
 #include "../Tag.h"
-#include "plugin/TaggedComponent.h"
-#include "plugin/ComponentManager.h"
 
 class EnumVal;
 
+namespace plugin {
+template <class T>
+class TaggedComponent;
+template <class T, class C>
+class ComponentManager;
+}
+
 namespace file_analysis {
 
 class Component;

src/file_analysis/analyzer/data_event/Plugin.cc

@@ -1,8 +1,8 @@
 // See the file in the main distribution directory for copyright.
 
-#include "plugin/Plugin.h"
-
 #include "DataEvent.h"
+#include "plugin/Plugin.h"
+#include "file_analysis/Component.h"
 
 namespace plugin {
 namespace Zeek_FileDataEvent {

src/file_analysis/analyzer/entropy/Plugin.cc

@@ -1,8 +1,8 @@
 // See the file  in the main distribution directory for copyright.
 
-#include "plugin/Plugin.h"
-
 #include "Entropy.h"
+#include "plugin/Plugin.h"
+#include "file_analysis/Component.h"
 
 namespace plugin {
 namespace Zeek_FileEntropy {

src/file_analysis/analyzer/extract/Plugin.cc

@@ -1,8 +1,8 @@
 // See the file  in the main distribution directory for copyright.
 
-#include "plugin/Plugin.h"
-
 #include "Extract.h"
+#include "plugin/Plugin.h"
+#include "file_analysis/Component.h"
 
 namespace plugin {
 namespace Zeek_FileExtract {

src/file_analysis/analyzer/hash/Plugin.cc

@@ -1,8 +1,8 @@
 // See the file  in the main distribution directory for copyright.
 
-#include "plugin/Plugin.h"
-
 #include "Hash.h"
+#include "plugin/Plugin.h"
+#include "file_analysis/Component.h"
 
 namespace plugin {
 namespace Zeek_FileHash {

src/file_analysis/analyzer/pe/Plugin.cc

@@ -1,8 +1,8 @@
 // See the file  in the main distribution directory for copyright.
 
-#include "plugin/Plugin.h"
-
 #include "PE.h"
+#include "plugin/Plugin.h"
+#include "file_analysis/Component.h"
 
 namespace plugin {
 namespace Zeek_PE {

src/file_analysis/analyzer/unified2/Plugin.cc

@@ -2,9 +2,9 @@
 
 // See the file "COPYING" in the main distribution directory for copyright.
 
-#include "plugin/Plugin.h"
-
 #include "Unified2.h"
+#include "plugin/Plugin.h"
+#include "file_analysis/Component.h"
 
 namespace plugin {
 namespace Zeek_Unified2 {

src/file_analysis/analyzer/x509/OCSP.cc

@@ -5,6 +5,7 @@
 #include "OCSP.h"
 #include "X509.h"
 #include "Event.h"
+#include "Reporter.h"
 
 #include "types.bif.h"
 #include "ocsp_events.bif.h"

src/file_analysis/analyzer/x509/Plugin.cc

@@ -1,10 +1,9 @@
 // See the file  in the main distribution directory for copyright.
 
-
-#include "plugin/Plugin.h"
-
 #include "X509.h"
 #include "OCSP.h"
+#include "plugin/Plugin.h"
+#include "file_analysis/Component.h"
 
 namespace plugin {
 namespace Zeek_X509 {

src/file_analysis/analyzer/x509/X509Common.cc

@@ -2,6 +2,7 @@
 
 #include "X509Common.h"
 #include "x509-extension_pac.h"
+#include "Reporter.h"
 
 #include "events.bif.h"
 #include "ocsp_events.bif.h"

src/file_analysis/analyzer/x509/X509Common.h

@@ -11,6 +11,8 @@
 #include <openssl/x509.h>
 #include <openssl/asn1.h>
 
+class Reporter;
+
 namespace file_analysis {
 
 class X509Common : public file_analysis::Analyzer {

src/file_analysis/file_analysis.bif

@@ -4,6 +4,8 @@ module Files;
 
 %%{
 #include "file_analysis/Manager.h"
+#include "file_analysis/File.h"
+#include "Reporter.h"
 %%}
 
 type AnalyzerArgs: record;