Merge pull request cisagov#44 from cisagov/feature/navv-v2

feature: navv-v2

.gitignore

@@ -1,10 +1,25 @@
 *.pkl
 *.xlsx
 
+## Environment variables ##
+.env 
+
+## MacOS ##
+.DS_Store
+
 ## Python ##
 __pycache__
 .coverage
 .mypy_cache
 .pytest_cache
 .python-version
 *.egg-info
+.venv
+
+## Test Data ##
+analysis
+test-data
+
+## Code Editors ##
+.idea
+.vscode

Makefile

@@ -0,0 +1,27 @@
+.PHONY: run
+include .env
+export
+
+# target: help - Display callable targets.
+help:
+	@egrep "^# target:" [Mm]akefile
+
+# target: install-develop - Install application for development
+install-develop:
+	pip install -e .
+
+# target: install - Install application
+install:
+	pip install navv
+
+# target: generate - Generate analysis from pcap
+generate:
+	navv generate -o analysis -p test-data/test_data.pcap -z test-data/logs test-customer
+
+# target: load-metadata - Load metadata
+load-metadata:
+	navv generate -o analysis -z test-data/logs test-customer
+
+# target: launch - Launch GUI application
+launch:
+	navv launch

setup.cfg

@@ -24,9 +24,11 @@ package_dir =
 packages = find:
 python_requires = >=3.6
 install_requires =
+    click>=8.1.6
+    flask>=2.3.2
     lxml>=4.3.2
-    netaddr>=0.7.19
-    openpyxl>=3.0.6
+    netaddr>=0.8.0
+    openpyxl>=3.1.2
     tqdm>=4.57.0
 
 [options.packages.find]

src/navv/__main__.py

@@ -1,10 +1,6 @@
 #!/usr/bin/env python3
 
-from navv import network_analysis
-
-
-def main():
-    network_analysis.main(network_analysis.parse_args())
+from navv.network_analysis import main
 
 
 if __name__ == "__main__":

src/navv/commands.py

@@ -0,0 +1,162 @@
+"""CLI Commands."""
+import json
+import os
+import webbrowser
+
+
+# Third-Party Libraries
+import click
+
+# cisagov Libraries
+from navv.gui import app
+from navv.message_handler import success_msg, warning_msg
+from navv.spreadsheet_tools import (
+    auto_adjust_width,
+    create_analysis_array,
+    get_inventory_data,
+    get_package_data,
+    get_segments_data,
+    get_workbook,
+    perform_analysis,
+    write_conn_states_sheet,
+    write_externals_sheet,
+    write_inventory_report_sheet,
+    write_macs_sheet,
+    write_stats_sheet,
+    write_unknown_internals_sheet,
+)
+from navv.utilities import pushd, run_zeek, perform_zeekcut, trim_dns_data
+
+
+@click.command("generate")
+@click.option(
+    "-o",
+    "--output-dir",
+    required=False,
+    help="Directory to place resultant analysis files in. Defaults to current working directory.",
+    type=str,
+)
+@click.option(
+    "-p",
+    "--pcap",
+    required=False,
+    help="Path to pcap file. NAVV requires zeek logs or pcap. If used, zeek will run on pcap to create new logs.",
+    type=str,
+)
+@click.option(
+    "-z",
+    "--zeek-logs",
+    required=False,
+    help="Path to store or contain zeek log files. Defaults to current working directory.",
+    type=str,
+)
+@click.argument("customer_name")
+def generate(customer_name, output_dir, pcap, zeek_logs):
+    """Generate excel sheet."""
+    with pushd(output_dir):
+        pass
+    file_name = os.path.join(output_dir, customer_name + "_network_analysis.xlsx")
+
+    wb = get_workbook(file_name)
+
+    services, conn_states = get_package_data()
+    timer_data = dict()
+    segments = get_segments_data(wb["Segments"])
+    inventory = get_inventory_data(wb["Inventory Input"])
+
+    if pcap:
+        run_zeek(os.path.abspath(pcap), zeek_logs, timer=timer_data)
+    else:
+        timer_data["run_zeek"] = "NOT RAN"
+    zeek_data = (
+        perform_zeekcut(
+            fields=[
+                "id.orig_h",
+                "id.resp_h",
+                "id.resp_p",
+                "proto",
+                "conn_state",
+                "orig_l2_addr",
+                "resp_l2_addr",
+            ],
+            log_file=os.path.join(zeek_logs, "conn.log"),
+        )
+        .decode("utf-8")
+        .split("\n")[:-1]
+    )
+
+    # turn zeekcut data into rows for spreadsheet
+    rows, mac_dict = create_analysis_array(zeek_data, timer=timer_data)
+
+    # get dns data for resolution
+    json_path = os.path.join(output_dir, f"{customer_name}_dns_data.json")
+
+    if os.path.exists(json_path):
+        with open(json_path, "rb") as json_file:
+            dns_filtered = json.load(json_file)
+    else:
+        dns_data = perform_zeekcut(
+            fields=["query", "answers", "qtype", "rcode_name"],
+            log_file=os.path.join(zeek_logs, "dns.log"),
+        )
+        dns_filtered = trim_dns_data(dns_data)
+
+    ext_IPs = set()
+    unk_int_IPs = set()
+    perform_analysis(
+        wb,
+        rows,
+        services,
+        conn_states,
+        inventory,
+        segments,
+        dns_filtered,
+        json_path,
+        ext_IPs,
+        unk_int_IPs,
+        timer=timer_data,
+    )
+
+    write_inventory_report_sheet(mac_dict, wb)
+
+    write_macs_sheet(mac_dict, wb)
+
+    write_externals_sheet(ext_IPs, wb)
+
+    write_unknown_internals_sheet(unk_int_IPs, wb)
+
+    auto_adjust_width(wb["Analysis"])
+
+    times = (
+        perform_zeekcut(fields=["ts"], log_file=os.path.join(zeek_logs, "conn.log"))
+        .decode("utf-8")
+        .split("\n")[:-1]
+    )
+    forward = sorted(times)
+    start = float(forward[0])
+    end = float(forward[len(forward) - 1])
+    cap_time = end - start
+    timer_data[
+        "Length of Capture time"
+    ] = "{} day(s) {} hour(s) {} minutes {} seconds".format(
+        int(cap_time / 86400),
+        int(cap_time % 86400 / 3600),
+        int(cap_time % 3600 / 60),
+        int(cap_time % 60),
+    )
+    write_stats_sheet(wb, timer_data)
+    write_conn_states_sheet(conn_states, wb)
+
+    wb.save(file_name)
+
+    if pcap:
+        success_msg(f"Successfully created file: {file_name}")
+
+
+@click.command("launch")
+def launch():
+    """Launch the NAVV GUI."""
+    port = 5000
+    warning_msg("Launching GUI in browser...")
+    webbrowser.open(f"http://127.0.0.1:{port}/")
+    app.run(port=port)