From 67835a6520e4d358efaa17ee91ed4e2edcb8eff0 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@corelight.com>
Date: Thu, 6 Aug 2020 17:35:42 -0700
Subject: [PATCH] Extend external test suite canonifier with set-sorting logic

Two new canonifiers: one to sort the contents of conn.log "service"
field and another to sort the contents of any field of type "set".
---
 testing/scripts/diff-canonifier-external |  2 +
 testing/scripts/diff-sort-conn-service   | 62 ++++++++++++++++++++++++
 testing/scripts/diff-sort-set-elements   | 60 +++++++++++++++++++++++
 3 files changed, 124 insertions(+)
 create mode 100755 testing/scripts/diff-sort-conn-service
 create mode 100755 testing/scripts/diff-sort-set-elements

diff --git a/testing/scripts/diff-canonifier-external b/testing/scripts/diff-canonifier-external
index bd91924d48..99520adbb4 100755
--- a/testing/scripts/diff-canonifier-external
+++ b/testing/scripts/diff-canonifier-external
@@ -23,6 +23,8 @@ fi
     | `dirname $0`/diff-remove-uids \
     | `dirname $0`/diff-remove-file-ids \
     | `dirname $0`/diff-remove-x509-names \
+    | `dirname $0`/diff-sort-conn-service \
+    | `dirname $0`/diff-sort-set-elements \
     | `dirname $0`/diff-sort \
     | eval $addl
 
diff --git a/testing/scripts/diff-sort-conn-service b/testing/scripts/diff-sort-conn-service
new file mode 100755
index 0000000000..b66ff61350
--- /dev/null
+++ b/testing/scripts/diff-sort-conn-service
@@ -0,0 +1,62 @@
+#! /usr/bin/env bash
+#
+# A diff canonifier that sorts conn.log service field (it's derived from a set
+# type and output may be unstable)
+
+awk '
+BEGIN {
+    FS="\t";
+    OFS="\t";
+    process = 0;
+    }
+
+function bubble_sort(arr, len,    keep_going, i, tmp)
+    {
+    keep_going = 1;
+
+    while ( keep_going == 1 )
+        {
+        keep_going = 0;
+
+        for ( i = 1; i <= len - 1; ++i )
+            {
+            if ( arr[i] > arr[i + 1] )
+                {
+                tmp = arr[i];
+                arr[i] = arr[i + 1];
+                arr[i + 1] = tmp;
+                keep_going = 1;
+                }
+            }
+        }
+
+    return len;
+    }
+
+$1 == "#path" && $2 == "conn" {
+    process = 1;
+    }
+
+/^[^#]/ {
+    if ( process && column1 > 0 && $column1 != "-" )
+        {
+        n = split($column1, set_contents, ",");
+        n = bubble_sort(set_contents, n);
+        sorted_field = set_contents[1];
+
+        for ( e = 2; e <= n; ++e )
+            sorted_field = sorted_field "," set_contents[e];
+
+        $column1 = sorted_field;
+        }
+    }
+
+/^#fields/ {
+    for ( i = 2; i <= NF; ++i ) {
+        if ( $i == "service" )
+            column1 = i - 1;
+        }
+    }
+
+{ print }
+'
diff --git a/testing/scripts/diff-sort-set-elements b/testing/scripts/diff-sort-set-elements
new file mode 100755
index 0000000000..319ebf5cde
--- /dev/null
+++ b/testing/scripts/diff-sort-set-elements
@@ -0,0 +1,60 @@
+#! /usr/bin/env bash
+#
+# A diff canonifier that sorts elements within fields of type set[T] for any T.
+
+awk '
+BEGIN { FS="\t"; OFS="\t"; }
+
+function bubble_sort(arr, len,    keep_going, i, tmp)
+    {
+    keep_going = 1;
+
+    while ( keep_going == 1 )
+        {
+        keep_going = 0;
+
+        for ( i = 1; i <= len - 1; ++i )
+            {
+            if ( arr[i] > arr[i + 1] )
+                {
+                tmp = arr[i];
+                arr[i] = arr[i + 1];
+                arr[i + 1] = tmp;
+                keep_going = 1;
+                }
+            }
+        }
+
+    return len;
+    }
+
+/^#types/ {
+    for ( i = 2; i <= NF; ++i )
+        {
+        if ( index($i, "set[") == 1 )
+            rem[i-1] = 1;
+        }
+}
+
+/^[^#]/ {
+    for ( i in rem )
+        {
+        if ( $i == "-" )
+            # The set has no value, skip sorting it.
+            continue;
+
+        n = split($i, set_contents, ",");
+        n = bubble_sort(set_contents, n);
+        sorted_field = set_contents[1];
+
+        for ( e = 2; e <= n; ++e )
+            sorted_field = sorted_field "," set_contents[e];
+
+        $i = sorted_field;
+        }
+}
+
+{
+    print;
+}
+'