From cd0d55e3d6f7911f4575b9a92f3965cdfce60f5d Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Mon, 27 Mar 2023 20:38:48 +0200 Subject: [PATCH] btest/broker: Add test using Python bindings and zeek -r This came up in community.zeek.org as a use-case and there were a few gaps and holes, so add a test showing minimally the current behavior. --- .../broker.python-bindings/recv..stdout | 23 +++ .../broker.python-bindings/send..stdout | 45 ++++++ testing/btest/broker/python-bindings.zeek | 137 ++++++++++++++++++ 3 files changed, 205 insertions(+) create mode 100644 testing/btest/Baseline/broker.python-bindings/recv..stdout create mode 100644 testing/btest/Baseline/broker.python-bindings/send..stdout create mode 100644 testing/btest/broker/python-bindings.zeek diff --git a/testing/btest/Baseline/broker.python-bindings/recv..stdout b/testing/btest/Baseline/broker.python-bindings/recv..stdout new file mode 100644 index 0000000000..e04c813bff --- /dev/null +++ b/testing/btest/Baseline/broker.python-bindings/recv..stdout @@ -0,0 +1,23 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Received /btest/connections new_conn_added (IPv4Address('141.142.220.118'), 35634/tcp, IPv4Address('208.80.152.2'), 80/tcp) +Received /btest/connections new_conn_added (IPv4Address('141.142.220.118'), 48649/tcp, IPv4Address('208.80.152.118'), 80/tcp) +Received /btest/connections new_conn_added (IPv4Address('141.142.220.118'), 49996/tcp, IPv4Address('208.80.152.3'), 80/tcp) +Received /btest/connections new_conn_added (IPv4Address('141.142.220.118'), 49997/tcp, IPv4Address('208.80.152.3'), 80/tcp) +Received /btest/connections new_conn_added (IPv4Address('141.142.220.118'), 49998/tcp, IPv4Address('208.80.152.3'), 80/tcp) +Received /btest/connections new_conn_added (IPv4Address('141.142.220.118'), 49999/tcp, IPv4Address('208.80.152.3'), 80/tcp) +Received /btest/connections new_conn_added (IPv4Address('141.142.220.118'), 50000/tcp, IPv4Address('208.80.152.3'), 80/tcp) +Received /btest/connections new_conn_added (IPv4Address('141.142.220.118'), 50001/tcp, IPv4Address('208.80.152.3'), 80/tcp) +Received /btest/connections new_conn_added (IPv4Address('141.142.220.118'), 35642/tcp, IPv4Address('208.80.152.2'), 80/tcp) +Received /btest/connections new_conn_added (IPv4Address('173.192.163.128'), 80/tcp, IPv4Address('141.142.220.235'), 6705/tcp) +Received /btest/connections conn_removed (IPv4Address('173.192.163.128'), 80/tcp, IPv4Address('141.142.220.235'), 6705/tcp) +Received /btest/connections conn_removed (IPv4Address('141.142.220.118'), 35634/tcp, IPv4Address('208.80.152.2'), 80/tcp) +Received /btest/connections conn_removed (IPv4Address('141.142.220.118'), 48649/tcp, IPv4Address('208.80.152.118'), 80/tcp) +Received /btest/connections conn_removed (IPv4Address('141.142.220.118'), 49997/tcp, IPv4Address('208.80.152.3'), 80/tcp) +Received /btest/connections conn_removed (IPv4Address('141.142.220.118'), 49996/tcp, IPv4Address('208.80.152.3'), 80/tcp) +Received /btest/connections conn_removed (IPv4Address('141.142.220.118'), 49998/tcp, IPv4Address('208.80.152.3'), 80/tcp) +Received /btest/connections conn_removed (IPv4Address('141.142.220.118'), 50000/tcp, IPv4Address('208.80.152.3'), 80/tcp) +Received /btest/connections conn_removed (IPv4Address('141.142.220.118'), 49999/tcp, IPv4Address('208.80.152.3'), 80/tcp) +Received /btest/connections conn_removed (IPv4Address('141.142.220.118'), 50001/tcp, IPv4Address('208.80.152.3'), 80/tcp) +Received /btest/connections conn_removed (IPv4Address('141.142.220.118'), 35642/tcp, IPv4Address('208.80.152.2'), 80/tcp) +Received /btest/connections Pcap::file_done / +Received Pcap::file_done diff --git a/testing/btest/Baseline/broker.python-bindings/send..stdout b/testing/btest/Baseline/broker.python-bindings/send..stdout new file mode 100644 index 0000000000..5cb65d5227 --- /dev/null +++ b/testing/btest/Baseline/broker.python-bindings/send..stdout @@ -0,0 +1,45 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +0.0, peer added +1300475168.652003, new_connection, 1, CHhAvVGS1DHFjwGM9 +1300475168.724007, new_connection, 2, ClEkJM2Vm5giqnMf4h +1300475168.855305, new_connection, 3, C4J4Th3PJpwUYZZ6gc +1300475168.85533, new_connection, 4, CtPZjS20MLrsMUOJi2 +1300475168.859163, new_connection, 5, CUM0KZ3MLUfNB0cl11 +1300475168.892913, new_connection, 6, CmES5u32sYpV7JYN +1300475168.892936, new_connection, 7, CP5puj4I8PtEU4qzYg +1300475168.895267, new_connection, 8, C37jN32gN3y3AZzyf6 +1300475168.902635, new_connection, 9, C3eiCBGOLw3VtHfOj +1300475169.780331, new_connection, 10, CwjjYJ2WqgTbAqiHl6 +1300475169.780331, Pcap::file_done +1300561569.780331, connection_state_remove, 11, CwjjYJ2WqgTbAqiHl6 +1300561569.780331, connection_state_remove, 12, CHhAvVGS1DHFjwGM9 +1300561569.780331, connection_state_remove, 13, ClEkJM2Vm5giqnMf4h +1300561569.780331, connection_state_remove, 14, CtPZjS20MLrsMUOJi2 +1300561569.780331, connection_state_remove, 15, C4J4Th3PJpwUYZZ6gc +1300561569.780331, connection_state_remove, 16, CUM0KZ3MLUfNB0cl11 +1300561569.780331, connection_state_remove, 17, CP5puj4I8PtEU4qzYg +1300561569.780331, connection_state_remove, 18, CmES5u32sYpV7JYN +1300561569.780331, connection_state_remove, 19, C37jN32gN3y3AZzyf6 +1300561569.780331, connection_state_remove, 20, C3eiCBGOLw3VtHfOj +1300561569.780331, send_pcap_file_done +1300561569.780331, from_python, 1, new_conn_added, CHhAvVGS1DHFjwGM9, [orig_h=141.142.220.118, orig_p=35634/tcp, resp_h=208.80.152.2, resp_p=80/tcp] +1300561569.780331, from_python, 2, new_conn_added, ClEkJM2Vm5giqnMf4h, [orig_h=141.142.220.118, orig_p=48649/tcp, resp_h=208.80.152.118, resp_p=80/tcp] +1300561569.780331, from_python, 3, new_conn_added, C4J4Th3PJpwUYZZ6gc, [orig_h=141.142.220.118, orig_p=49996/tcp, resp_h=208.80.152.3, resp_p=80/tcp] +1300561569.780331, from_python, 4, new_conn_added, CtPZjS20MLrsMUOJi2, [orig_h=141.142.220.118, orig_p=49997/tcp, resp_h=208.80.152.3, resp_p=80/tcp] +1300561569.780331, from_python, 5, new_conn_added, CUM0KZ3MLUfNB0cl11, [orig_h=141.142.220.118, orig_p=49998/tcp, resp_h=208.80.152.3, resp_p=80/tcp] +1300561569.780331, from_python, 6, new_conn_added, CmES5u32sYpV7JYN, [orig_h=141.142.220.118, orig_p=49999/tcp, resp_h=208.80.152.3, resp_p=80/tcp] +1300561569.780331, from_python, 7, new_conn_added, CP5puj4I8PtEU4qzYg, [orig_h=141.142.220.118, orig_p=50000/tcp, resp_h=208.80.152.3, resp_p=80/tcp] +1300561569.780331, from_python, 8, new_conn_added, C37jN32gN3y3AZzyf6, [orig_h=141.142.220.118, orig_p=50001/tcp, resp_h=208.80.152.3, resp_p=80/tcp] +1300561569.780331, from_python, 9, new_conn_added, C3eiCBGOLw3VtHfOj, [orig_h=141.142.220.118, orig_p=35642/tcp, resp_h=208.80.152.2, resp_p=80/tcp] +1300561569.780331, from_python, 10, new_conn_added, CwjjYJ2WqgTbAqiHl6, [orig_h=173.192.163.128, orig_p=80/tcp, resp_h=141.142.220.235, resp_p=6705/tcp] +1300561569.780331, from_python, 11, conn_removed, CwjjYJ2WqgTbAqiHl6, [orig_h=173.192.163.128, orig_p=80/tcp, resp_h=141.142.220.235, resp_p=6705/tcp] +1300561569.780331, from_python, 12, conn_removed, CHhAvVGS1DHFjwGM9, [orig_h=141.142.220.118, orig_p=35634/tcp, resp_h=208.80.152.2, resp_p=80/tcp] +1300561569.780331, from_python, 13, conn_removed, ClEkJM2Vm5giqnMf4h, [orig_h=141.142.220.118, orig_p=48649/tcp, resp_h=208.80.152.118, resp_p=80/tcp] +1300561569.780331, from_python, 14, conn_removed, CtPZjS20MLrsMUOJi2, [orig_h=141.142.220.118, orig_p=49997/tcp, resp_h=208.80.152.3, resp_p=80/tcp] +1300561569.780331, from_python, 15, conn_removed, C4J4Th3PJpwUYZZ6gc, [orig_h=141.142.220.118, orig_p=49996/tcp, resp_h=208.80.152.3, resp_p=80/tcp] +1300561569.780331, from_python, 16, conn_removed, CUM0KZ3MLUfNB0cl11, [orig_h=141.142.220.118, orig_p=49998/tcp, resp_h=208.80.152.3, resp_p=80/tcp] +1300561569.780331, from_python, 17, conn_removed, CP5puj4I8PtEU4qzYg, [orig_h=141.142.220.118, orig_p=50000/tcp, resp_h=208.80.152.3, resp_p=80/tcp] +1300561569.780331, from_python, 18, conn_removed, CmES5u32sYpV7JYN, [orig_h=141.142.220.118, orig_p=49999/tcp, resp_h=208.80.152.3, resp_p=80/tcp] +1300561569.780331, from_python, 19, conn_removed, C37jN32gN3y3AZzyf6, [orig_h=141.142.220.118, orig_p=50001/tcp, resp_h=208.80.152.3, resp_p=80/tcp] +1300561569.780331, from_python, 20, conn_removed, C3eiCBGOLw3VtHfOj, [orig_h=141.142.220.118, orig_p=35642/tcp, resp_h=208.80.152.2, resp_p=80/tcp] +1300561569.780331, peer lost diff --git a/testing/btest/broker/python-bindings.zeek b/testing/btest/broker/python-bindings.zeek new file mode 100644 index 0000000000..30380e7805 --- /dev/null +++ b/testing/btest/broker/python-bindings.zeek @@ -0,0 +1,137 @@ +# @TEST-GROUP: broker +# +# @TEST-PORT: BROKER_PORT +# +# @TEST-DOC: biswa on community.zeek.org has been doing cool stuff with zeek -r, Python broker bindings and suspend_processing(). He ran into a number of issues around suspend_processing(), time management, etc, try to cover some here. +# +# @TEST-REQUIRES: python3 -V +# @TEST-REQUIRES: TOPIC=/btest/connections python3 recv.py check +# +# @TEST-EXEC: TOPIC=/btest/connections btest-bg-run recv "python3 -u ../recv.py" +# @TEST-EXEC: TOPIC=/btest/connections btest-bg-run send "zeek -f 'port 80' -b ../send.zeek -r $TRACES/wikipedia.trace" +# +# @TEST-EXEC: btest-bg-wait 10 +# @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff recv/.stdout +# @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff send/.stdout + +@TEST-START-FILE send.zeek + +global new_conn_added: event(c: connection) &is_used; +global conn_removed: event(c: connection) &is_used; + +global my_topic = getenv("TOPIC"); + +global conn_events = 0; + +event zeek_init() + { + Broker::peer("127.0.0.1", to_port(getenv("BROKER_PORT"))); + Broker::subscribe(my_topic); + suspend_processing(); + } + +event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) + { + print network_time(), "peer lost"; + terminate(); + } + +event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) + { + print network_time(), "peer added"; + continue_processing(); + } + +event new_connection(c: connection) + { + ++conn_events; + print network_time(), "new_connection", conn_events, c$uid; + Broker::publish(my_topic, new_conn_added, c); + } + +event connection_state_remove(c: connection) + { + ++conn_events; + print network_time(), "connection_state_remove", conn_events, c$uid; + Broker::publish(my_topic, conn_removed, c); + } + +event send_pcap_file_done(path: string) + { + print network_time(), "send_pcap_file_done"; + Broker::publish(my_topic, Pcap::file_done, path); + } + +event Pcap::file_done(path: string) + { + # Done reading pcap, forward network_time() by + # 24 hours to expire timers. Send Pcap::file_done + # to Python in a schedule timer to initiate + # termination. + print network_time(), "Pcap::file_done"; + + schedule double_to_interval(24 * 3600 - 1) { send_pcap_file_done(path) }; + + set_network_time(network_time() + double_to_interval(24 * 3600)); + } + +global events_from_python = 0; +event echo_from_python(what: string, c: connection) &is_used + { + ++events_from_python; + print network_time(), "from_python", events_from_python, what, c$uid, c$id; + } +@TEST-END-FILE + + +@TEST-START-FILE recv.py +""" +Python script subscribing to TOPIC +""" +import os +import sys + +# Prep the PYTHONPATH for the build directory. +broker_path = os.path.join(os.environ["BUILD"], "auxil", "broker", "python") +sys.path.insert(0, broker_path) + +import broker + +# 1024/tcp +broker_port = int(os.environ["BROKER_PORT"].split("/")[0]) +broker_topic = os.environ["TOPIC"] + +# We were able to import broker and parse the broker_port, should be good. +if len(sys.argv) > 1 and sys.argv[1] == "check": + sys.exit(0) + +# Setup endpoint and connect to Zeek. +with ( broker.Endpoint() as ep, + ep.make_subscriber(broker_topic) as sub, + ep.make_status_subscriber(True) as ss): + + ep.listen("127.0.0.1", broker_port) + + while True: + statuses = ss.poll() + for s in statuses: + if s.code() in (broker.SC.PeerLost, broker.SC.EndpointUnreachable): + print("peer lost, done") + exit(0) + + # Busy poll for a message or later status + msg = sub.get(0.5) + if msg is None: + continue + (t, d) = msg + my_event = broker.zeek.Event(d) + conn = my_event.args()[0] + print("Received", t, my_event.name(), my_event.args()[0][0]) + + if my_event.name() == "Pcap::file_done": + print("Received Pcap::file_done") + break + + other_event = broker.zeek.Event("echo_from_python", my_event.name(), conn) + ep.publish(broker_topic, other_event) +@TEST-END-FILE