Neyberson/connection

.gitignore

@@ -7,3 +7,6 @@ __pycache__
 
 # editor files
 *.sw*
+
+# ipynb
+.ipynb_checkpoints/

app/app.py

@@ -6,17 +6,36 @@ def main():
     # Interface Test
     # Process/Pipes Test
     with devtools.Browser(headless=False) as browser:
-        ## Create Protocol
-        connection = browser.protocol
+        browser.send_command(command="Target.getTargets")
+        browser.protocol.pipe.read_jsons(debug=True)
+        browser.protocol.pipe.read_jsons(blocking=False, debug=True)
+        browser.protocol.pipe.read_jsons(blocking=False, debug=True)
+        browser.protocol.pipe.read_jsons(blocking=False, debug=True)
+        time.sleep(4)
+        browser.protocol.send_command(
+            command="Target.createTarget", params={"url": "https://www.youtube.com"}
+        )
+        time.sleep(5)
+        browser.protocol.send_command(command="Target.getTargets")
+        browser.protocol.pipe.read_jsons(debug=True)
+        browser.protocol.pipe.read_jsons(blocking=False, debug=True)
+        browser.protocol.pipe.read_jsons(blocking=False, debug=True)
+        browser.protocol.pipe.read_jsons(blocking=False, debug=True)
+        time.sleep(3)
+        browser.protocol.send_command(command="Target.getTargets")
+        browser.protocol.pipe.read_jsons(debug=True)
+        browser.protocol.pipe.read_jsons(blocking=False, debug=True)
+        browser.protocol.pipe.read_jsons(blocking=False, debug=True)
+        browser.protocol.pipe.read_jsons(blocking=False, debug=True)
+        time.sleep(3)
+    time.sleep(3)
 
-        connection.send_command(command="Target.createTarget", params={"url": "https://www.youtube.com/"})
-
-        connection.pipe.read_jsons(debug=True)
-        connection.pipe.read_jsons(blocking=False, debug=True)
-        connection.pipe.read_jsons(blocking=False, debug=True)
-        connection.pipe.read_jsons(blocking=False, debug=True)
-
-        time.sleep(10)
 
 if __name__ == "__main__":
     main()
+
+
+# blocking, regular blocking, you read and write, good luck (we need to be able to piece-meal the thing together)
+
+# blocking with dumping out output off separate thread that you can see
+# that one should probably be working with a queue

devtools/browser.py

@@ -1,35 +1,53 @@
-from .pipe import Pipe
-from .protocol import Protocol
 import platform
 import os
 import sys
 import subprocess
-import signal
+import tempfile
+import warnings
+
+from collections import OrderedDict
+
+from .pipe import Pipe
+from .protocol import Protocol
+from .target import Target
+from .session import Session
+from .tab import Tab
+from .system import which_browser
+
+default_path = which_browser()
 
 
-class Browser:
-    def __init__(self, debug=None, path=None, headless=True):
-        self.pipe = Pipe()
+class Browser(Target):
+    def __init__(
+        self, path=default_path, headless=True, debug=False, debug_browser=None
+    ):
+        if path is None:
+            raise ValueError("You must specify a path")
 
-        if not debug:  # false o None
+        self.pipe = Pipe(debug=debug)
+        self.protocol = Protocol(self.pipe)
+        super().__init__("0", self.protocol)  # TODO not sure about target id "0"
+        self.add_session(Session(self, ""))
+
+        if platform.system() != "Windows":
+            self.temp_dir = tempfile.TemporaryDirectory()
+        else:
+            self.temp_dir = tempfile.TemporaryDirectory(
+                delete=False, ignore_cleanup_errors=True
+            )
+
+        if not debug_browser:  # false o None
             stderr = subprocess.DEVNULL
-        elif debug is True:
+        elif debug_browser is True:
             stderr = None
         else:
             stderr = debug
 
-        if not path:
-            if platform.system() == "Windows":
-                path = r"C:\Program Files\Google\Chrome\Application\chrome.exe"
-            elif platform.system() == "Linux":
-                path = "/usr/bin/google-chrome-stable"
-            else:
-                raise ValueError("You must set path to a chrome-like browser")
-
         new_env = os.environ.copy()
-        new_env["CHROMIUM_PATH"] = path
+        new_env["CHROMIUM_PATH"] = str(path)
+        new_env["USER_DATA_DIR"] = str(self.temp_dir.name)
         if headless:
-            new_env["HEADLESS"] = "--headless"
+            new_env["HEADLESS"] = "--headless"  # unset if false
 
         win_only = {}
         if platform.system() == "Windows":
@@ -50,21 +68,59 @@ def __init__(self, debug=None, path=None, headless=True):
             **win_only,
         )
         self.subprocess = proc
-        self.protocol = Protocol(self.pipe)
+        self.tabs = OrderedDict()
 
     def __enter__(self):
         return self
 
     def __exit__(self, type, value, traceback):
-        return self.close_browser()
+        self.close()
 
-    def close_browser(self):
+    def close(self):
         if platform.system() == "Windows":
-            self.subprocess.send_signal(signal.CTRL_BREAK_EVENT)
-        else:
-            self.subprocess.terminate()
-        self.subprocess.wait(5)
+            # maybe we don't need chrome_wrapper for windows because of how handles are needed
+            # if we're not chaining process, this might not be necessary
+            # otherwise, win behaves strangely in the face of signals, so call a command to kill the process instead
+            # NB: chrome accepts being killed like this because it knows windows is a nightmare
+            subprocess.call(
+                ["taskkill", "/F", "/T", "/PID", str(self.subprocess.pid)]
+            )  # this output should be handled better by where there is debug
+            self.subprocess.wait(2)
+        self.subprocess.terminate()
+        self.subprocess.wait(2)
         self.subprocess.kill()
+        try:
+            self.temp_dir.cleanup()
+        except Exception as e:  # TODO- handle specific errors
+            print(str(e))
+
+        # windows doesn't like python's default cleanup
+        if platform.system() == "Windows":
+            import stat
+            import shutil
+
+            def remove_readonly(func, path, excinfo):
+                os.chmod(path, stat.S_IWUSR)
+                func(path)
+
+            try:
+                shutil.rmtree(self.temp_dir.name, onexc=remove_readonly)
+                del self.temp_dir
+            except PermissionError:
+                warnings.warn(
+                    "The temporary directory could not be deleted, but execution will continue."
+                )
+            except Exception:
+                warnings.warn(
+                    "The temporary directory could not be deleted, but execution will continue."
+                )
+
+    def add_tab(self, tab):
+        if not isinstance(tab, Tab):
+            raise TypeError("tab must be an object of class Tab")
+        self.sessions[tab.target_id] = tab
 
-    def send_command(self, command, params=None, cb=None):
-        return self.protocol.send_command(self, command, params, cb)
+    def remove_session(self, target_id):
+        if isinstance(target_id, Tab):
+            target_id = target_id.target_id
+        del self.sessions[target_id]

devtools/chrome_wrapper.py

@@ -16,21 +16,27 @@
 if system == "Windows":
     import msvcrt  # noqa
 
-path = ""
-if system == "Linux":
-    path = os.environ.get("BROWSER_PATH", "/usr/bin/google-chrome-stable")
-elif system == "Windows":
-    path = os.environ.get(
-        "BROWSER_PATH", r"c:\Program Files\Google\Chrome\Application\chrome.exe"
-    )
-else:
-    path = os.environ["CHROMIUM_PATH"]
+default_paths = {
+    "Linux": "/usr/bin/google-chrome-stable",
+    "Windows": r"c:\Program Files\Google\Chrome\Application\chrome.exe",
+}
+
+path = os.environ.get(
+    "BROWSER_PATH", default_paths.get(system, os.environ.get("CHROMIUM_PATH"))
+)
+
+if path is None:
+    raise ValueError("You must specify a path")
+
+user_data_dir = os.environ["USER_DATA_DIR"]
 
 cli = [
     path,
     "--remote-debugging-pipe",
     "--disable-breakpad",
     "--allow-file-access-from-files",
+    f"--user-data-dir={user_data_dir}",
+    "--no-first-run",
 ]
 
 if "HEADLESS" in os.environ:

devtools/pipe.py

@@ -2,50 +2,55 @@
 import sys
 import json
 
-
 class PipeClosedError(IOError):
     pass
 
-
 class Pipe:
-    def __init__(self):
+    def __init__(self, debug=False):
         self.read_from_chromium, self.write_from_chromium = list(os.pipe())
         self.read_to_chromium, self.write_to_chromium = list(os.pipe())
+        self.debug = debug
 
-    def write_json(
-        self, message_id, method, params=None, session_id=""
-    ):  # this should accept an objects not a string
-        if params:
-            message = {"id": message_id, "method": method, "params": params}
-        else:
-            message = {"id": message_id, "method": method}
-
-        if session_id != "":
-            message["session_id"] = session_id
+    # TODO: accept already formed object
+    def write_json(self, obj, debug=None):
+        if not debug: debug = self.debug
+        if debug:
+            print("write_json:", file=sys.stderr)
 
-        encoded_message = json.dumps(message).encode() + b"\0"
+        encoded_message = json.dumps(obj).encode("utf-8") + b"\0"
 
+        if debug:
+            print(f"write_json: {encoded_message}", file=sys.stderr)
+            # windows may print weird characters if we set utf-8 instead of utf-16
+            # check this TODO
         os.write(self.write_to_chromium, encoded_message)
 
-    def read_jsons(self, blocking=True, debug=False):
+    def read_jsons(self, blocking=True, debug=None):
+        if not debug:
+            debug = self.debug
         if debug:
-            print("Debug enabled", file=sys.stderr)
+            print(f"read_jsons ({'blocking' if blocking else 'not blocking'}):", file=sys.stderr)
         jsons = []
         os.set_blocking(self.read_from_chromium, blocking)
         try:
             raw_buffer = os.read(
                 self.read_from_chromium, 10000
             )  # 10MB buffer, nbd, doesn't matter w/ this
             if not raw_buffer:
+                if debug: print("read_jsons pipe was closed")
                 raise PipeClosedError()
             while raw_buffer[-1] != 0:
+                # still not great, return what you have
                 os.set_blocking(self.read_from_chromium, True)
                 raw_buffer += os.read(self.read_from_chromium, 10000)
         except BlockingIOError:
+            if debug:
+                print("read_jsons: BlockingIOError caught.", file=sys.stderr)
             return jsons
+        decoded_buffer = raw_buffer.decode("utf-8")
         if debug:
-            print(raw_buffer, file=sys.stderr)  # noqa
-        for raw_message in raw_buffer.decode("utf-8").split("\0"):
+            print(f"read_jsons: {decoded_buffer}", file=sys.stderr)
+        for raw_message in decoded_buffer.split("\0"):
             if raw_message:
                 jsons.append(json.loads(raw_message))
         return jsons

devtools/protocol.py

@@ -1,33 +1,62 @@
-from .tab import Tab
-from .session import Session
-from collections import OrderedDict
+import json
+
+from .pipe import PipeClosedError
+from threading import Thread
 
 
 class Protocol:
     def __init__(self, browser_pipe):
-        self.browser_session = Session(self, session_id="")
-        self.target_id = 0
-        self.tabs = OrderedDict()
         self.pipe = browser_pipe
 
-    def create_tab(self):
-        tab_obj = Tab()
-        self.tabs[tab_obj.target_id] = tab_obj
-        print(f"New Tab Created: {tab_obj.target_id}")
-        return tab_obj
-
-    def list_tabs(self):
-        print("Tabs".center(50, "-"))
-        for target_id in self.tabs.keys():
-            print(target_id.center(50, " "))
-        print("End".center(50, "-"))
-
-    def close_tab(self, tab_id):
-        if isinstance(tab_id, str):
-            del self.tabs[tab_id]
+    def write_json(self, obj):
+        n_keys = 0
+        if "id" in obj and "method" in obj:
+            n_keys += 2
+        else:
+            raise RuntimeError("Each message object must contain an id and method key")
+
+        if "params" in obj:
+            n_keys += 1
+        if "sessionId" in obj:
+            n_keys += 1
+
+        if len(obj.keys()) != n_keys:
+            raise RuntimeError(
+                "Message objects must have id and method keys, and may have params and sessionId keys"
+            )
+
+        self.pipe.write_json(obj)
+
+    def verify_response(self, response, session_id, message_id):
+        if "session_id" not in response and session_id == "":
+            pass
+        elif "session_id" in response and response["session_id"] == session_id:
+            pass
         else:
-            del self.tabs[tab_id.target_id]
-        print(f"The following tab was deleted: {tab_id}")
+            return False
+
+        if "id" in response and str(response["id"]) == str(message_id):
+            pass
+        else:
+            return False
+
+        return True
+
+    def run_output_thread(self, debug=False):
+        print("Start run_output_thread() to improve debugging".center(6, "-"))
+
+        def run_print(debug):
+            while True:
+                try:
+                    json_list = self.pipe.read_jsons(debug=debug)
+                    if json_list:
+                        for json_ in json_list:
+                            print(json.dumps(json_, indent=4))
+                except PipeClosedError:
+                    print("Pipe closed".center(10, "-"))
+                    break
+
+        thread_print = Thread(target=run_print, args=(debug,))
+        thread_print.start()
 
-    def send_command(self, command, params=None, cb=None):
-        return self.browser_session.send_command(command, params, cb)
+    # we need to do something for errors