Add readtoken to recognize pipe tokens

src/ansi-shell/Pipe.js

@@ -0,0 +1,17 @@
+export class Pipe {
+    constructor () {
+        this.readableStream = new ReadableStream({
+            start: controller => {
+                this.readController = controller;
+            }
+        });
+        this.writableStream = new WritableStream({
+            start: controller => {
+                this.writableController = controller;
+            },
+            write: item => {
+                this.readController.enqueue(item);
+            }
+        });
+    }
+}

src/ansi-shell/main_shell.js

@@ -12,11 +12,15 @@ const command_registry = {
 // TODO: auto-gen argument parser registry from files
 import SimpleArgParser from "./arg-parsers/simple-parser";
 import { PuterANSIShell } from "../puter-shell/PuterANSIShell";
+import { HiTIDE } from "hitide";
+
 const argparser_registry = {
     [SimpleArgParser.name]: SimpleArgParser
 };
 
 export const main_shell = async () => {
+    const hitide = new HiTIDE();
+
     const ptt = new XDocumentPTT();
     const config = {};
 

src/ansi-shell/readtoken.js

@@ -0,0 +1,86 @@
+// [reference impl](https://github.com/brgl/busybox/blob/master/shell/ash.c)
+
+const list_ws = [' ', '\n', '\t'];
+const list_recorded_tokens = [
+    '|','>','<','&',';','(',')',
+];
+const list_stoptoken = [
+    '|','>','<','&','\\','#',';','(',')',
+    ...list_ws
+];
+
+export const TOKENS = {};
+for ( const k of list_recorded_tokens ) {
+    TOKENS[k] = {};
+}
+
+export const readtoken = str => {
+    let state = null;
+    let buffer = '';
+    let quoteType = '';
+    const tokens = [];
+
+    const actions = {
+        endToken: () => {
+            tokens.push(buffer);
+            buffer = '';
+        }
+    };
+
+    const states = {
+        start: i => {
+            if ( list_ws.includes(str[i]) ) {
+                return;
+            }
+            if ( str[i] === '#' ) return str.length;
+            if ( list_recorded_tokens.includes(str[i]) ) {
+                tokens.push(TOKENS[str[i]]);
+                return;
+            }
+            if ( str[i] === '"' || str[i] === "'" ) {
+                state = states.quote;
+                quoteType = str[i];
+                return;
+            }
+            state = states.text;
+            return i; // prevent increment
+        },
+        text: i => {
+            if ( str[i] === '"' || str[i] === "'" ) {
+                state = states.quote;
+                quoteType = str[i];
+                return;
+            }
+            if ( list_stoptoken.includes(str[i]) ) {
+                state = states.start;
+                actions.endToken();
+                return i; // prevent increment
+            }
+            buffer += str[i];
+        },
+        quote: i => {
+            if ( str[i] === '\\' ) {
+                state = states.quote_esc;
+                return;
+            }
+            if ( str[i] === quoteType ) {
+                state = states.text;
+                return;
+            }
+            buffer += str[i];
+        },
+        quote_esc: i => {
+            buffer += str[i];
+            state = states.quote;
+        }
+    };
+    state = states.start;
+    for ( let i=0 ; i < str.length ; ) {
+        let newI = state(i);
+        i = newI !== undefined ? newI : i+1;
+    }
+
+    if ( buffer !== '' ) actions.endToken();
+
+    return tokens;
+};

src/puter-shell/PuterANSIShell.js

@@ -1,3 +1,5 @@
+import { readtoken } from "../ansi-shell/readtoken";
+
 export class PuterANSIShell {
     constructor (ctx) {
         this.ctx = ctx;
@@ -33,15 +35,13 @@ export class PuterANSIShell {
         const input = await readline(
             this.expandPromptString(this.env.PS1)
         );
-        // TODO: add proper tokenizer
-        const tokens = input.split(' ');
 
-        await this.runCommand(tokens);
+        await this.runCommand(input);
     }
 
     async runCommand (cmdOrTokens) {
         const tokens = typeof cmdOrTokens === 'string'
-            ? this.tokenize(cmdOrTokens)
+            ? readtoken(cmdOrTokens)
             : cmdOrTokens ;
 
         const cmd = tokens.shift();
@@ -81,14 +81,16 @@ export class PuterANSIShell {
         await command.execute(ctx);
     }
 
-    tokenize (cmdString) {
-        return cmdString.split(' ');
-    }
-
     expandPromptString (str) {
         str = str.replace('\\u', this.config['puter.auth.username']);
         str = str.replace('\\w', this.variables.pwd);
         str = str.replace('\\$', '$');
         return str;
     }
-}
+
+    async outputANSI (ctx) {
+        await ctx.iterate(async item => {
+            ctx.externs.out.write(item.name + '\n');
+        });
+    }
+}

test/readtoken.js

@@ -0,0 +1,42 @@
+import assert from 'assert';
+import { readtoken, TOKENS } from '../src/ansi-shell/readtoken.js';
+
+describe('readtoken', () => {
+    const tcases = [
+        {
+            desc: 'should accept unquoted string',
+            input: 'asdf',
+            expected: ['asdf']
+        },
+        {
+            desc: 'should accept leading spaces',
+            input: '   asdf',
+            expected: ['asdf']
+        },
+        {
+            desc: 'should accept trailing spaces',
+            input: 'asdf   ',
+            expected: ['asdf']
+        },
+        {
+            desc: 'should expected quoted string',
+            input: '"asdf"',
+            expected: ['asdf']
+        },
+        {
+            desc: 'should recognize pipe with no whitespace',
+            input: 'asdf|zxcv',
+            expected: ['asdf', TOKENS['|'], 'zxcv']
+        },
+        {
+            desc: 'mixed quoted and unquoted should work',
+            input: '"asdf" zxcv',
+            expected: ['asdf', 'zxcv']
+        },
+    ];
+    for ( const { desc, input, expected } of tcases ) {
+        it(desc, () => {
+            assert.deepEqual(readtoken(input), expected)
+        });
+    }
+})