Use Asynchronous Code for Loading Modules (#1471)

* Add async loading code

* Made infinite loop checking async

* Update stepper to be async

* Add assert and misc utils

* Update transpiler to be async

* Update runners

* Add new import options

* Ignore tests during typechecking

* Relocate typeguards

* Update with assertions and typeguards

* Update repl transpiler to be async

* Abstract out module context initialization and tab loading

* Renamed promise timeout error and added tests

* Remove old code

* Update options

* Add files to avoid line ending issues

* Use POSIX paths on Windows systems

* Ran format

* Update eslint rules to follow exclusion of tests

* Incorporate changes for posix path handling

* Minor change to resolve certain webpack issues with the frontend

* Use a different posix path import

* Allow loading of export default tabs

* Refactor collation of import declarations

---------

Co-authored-by: Ian Yong <[email protected]>

.editorconfig

@@ -0,0 +1,2 @@
+[*]
+end_of_line = lf

.eslintrc.json

@@ -9,6 +9,7 @@
     "plugin:@typescript-eslint/recommended-requiring-type-checking",
     "prettier"
   ],
+  "ignorePatterns": ["src/**/__tests__/**", "src/**/__mocks__/**"],
   "parser": "@typescript-eslint/parser",
   "parserOptions": {
     "project": "tsconfig.json",
@@ -45,5 +46,12 @@
     "@typescript-eslint/unbound-method": "off",
     "prefer-rest-params": "off",
     "simple-import-sort/imports": "warn"
-  }
+  },
+  "overrides": [{
+    "files": ["src/**/__tests__/**", "src/**/__mocks__/**"],
+    "plugins": ["simple-import-sort"],
+    "rules": {
+      "simple-import-sort/imports": "warn"
+    }
+  }]
 }

.prettierignore

@@ -1,2 +1,3 @@
 /src/scm-slang/
-/src/py-slang/
+/src/py-slang/
+/src/**/__tests__/**/__snapshots__

.prettierrc

@@ -4,5 +4,6 @@
   "printWidth": 100,
   "parser": "typescript",
   "trailingComma": "none",
-  "arrowParens": "avoid"
+  "arrowParens": "avoid",
+  "endOfLine": "lf"
 }

src/ec-evaluator/interpreter.ts

@@ -7,18 +7,20 @@
 
 /* tslint:disable:max-classes-per-file */
 import * as es from 'estree'
-import { partition, uniqueId } from 'lodash'
+import { uniqueId } from 'lodash'
 
 import { IOptions } from '..'
 import { UNKNOWN_LOCATION } from '../constants'
 import * as errors from '../errors/errors'
 import { RuntimeSourceError } from '../errors/runtimeSourceError'
 import Closure from '../interpreter/closure'
 import { UndefinedImportError } from '../modules/errors'
-import { loadModuleBundle, loadModuleTabs } from '../modules/moduleLoader'
-import { ModuleFunctions } from '../modules/moduleTypes'
+import { initModuleContext, loadModuleBundle } from '../modules/moduleLoader'
+import { ImportTransformOptions } from '../modules/moduleTypes'
 import { checkEditorBreakpoints } from '../stdlib/inspector'
 import { Context, ContiguousArrayElements, Result, Value } from '../types'
+import assert from '../utils/assert'
+import { filterImportDeclarations } from '../utils/ast/helpers'
 import * as ast from '../utils/astCreator'
 import { evaluateBinaryExpression, evaluateUnaryExpression } from '../utils/operators'
 import * as rttc from '../utils/rttc'
@@ -172,45 +174,31 @@ export function resumeEvaluate(context: Context) {
 function evaluateImports(
   program: es.Program,
   context: Context,
-  loadTabs: boolean,
-  checkImports: boolean
+  { loadTabs, checkImports }: ImportTransformOptions
 ) {
-  const [importNodes] = partition(program.body, ({ type }) => type === 'ImportDeclaration') as [
-    es.ImportDeclaration[],
-    es.Statement[]
-  ]
-  const moduleFunctions: Record<string, ModuleFunctions> = {}
-
   try {
-    for (const node of importNodes) {
-      const moduleName = node.source.value
-      if (typeof moduleName !== 'string') {
-        throw new Error(`ImportDeclarations should have string sources, got ${moduleName}`)
-      }
-
-      if (!(moduleName in moduleFunctions)) {
-        context.moduleContexts[moduleName] = {
-          state: null,
-          tabs: loadTabs ? loadModuleTabs(moduleName, node) : null
-        }
-        moduleFunctions[moduleName] = loadModuleBundle(moduleName, context, node)
-      }
-
-      const functions = moduleFunctions[moduleName]
-      const environment = currentEnvironment(context)
-      for (const spec of node.specifiers) {
-        if (spec.type !== 'ImportSpecifier') {
-          throw new Error(`Only ImportSpecifiers are supported, got: ${spec.type}`)
+    const [importNodeMap] = filterImportDeclarations(program)
+
+    const environment = currentEnvironment(context)
+    Object.entries(importNodeMap).forEach(([moduleName, nodes]) => {
+      initModuleContext(moduleName, context, loadTabs)
+      const functions = loadModuleBundle(moduleName, context, nodes[0])
+      for (const node of nodes) {
+        for (const spec of node.specifiers) {
+          assert(
+            spec.type === 'ImportSpecifier',
+            `Only ImportSpecifiers are supported, got: ${spec.type}`
+          )
+
+          if (checkImports && !(spec.imported.name in functions)) {
+            throw new UndefinedImportError(spec.imported.name, moduleName, spec)
+          }
+
+          declareIdentifier(context, spec.local.name, node, environment)
+          defineVariable(context, spec.local.name, functions[spec.imported.name], true, node)
         }
-
-        if (checkImports && !(spec.imported.name in functions)) {
-          throw new UndefinedImportError(spec.imported.name, moduleName, node)
-        }
-
-        declareIdentifier(context, spec.local.name, node, environment)
-        defineVariable(context, spec.local.name, functions[spec.imported.name], true, node)
       }
-    }
+    })
   } catch (error) {
     // console.log(error)
     handleRuntimeError(context, error)
@@ -335,7 +323,11 @@ const cmdEvaluators: { [type: string]: CmdEvaluator } = {
     if (hasDeclarations(command) || hasImportDeclarations(command)) {
       const environment = createBlockEnvironment(context, 'programEnvironment')
       pushEnvironment(context, environment)
-      evaluateImports(command as unknown as es.Program, context, true, true)
+      evaluateImports(command as unknown as es.Program, context, {
+        wrapSourceModules: true,
+        checkImports: true,
+        loadTabs: true
+      })
       declareFunctionsAndVariables(context, command, environment)
     }
 

src/index.ts

@@ -16,6 +16,7 @@ import {
   FuncDeclWithInferredTypeAnnotation,
   ModuleContext,
   NodeWithInferredType,
+  RecursivePartial,
   Result,
   SourceError,
   SVMProgram,
@@ -31,6 +32,7 @@ import { ECEResultPromise, resumeEvaluate } from './ec-evaluator/interpreter'
 import { CannotFindModuleError } from './errors/localImportErrors'
 import { validateFilePath } from './localImports/filePaths'
 import preprocessFileImports from './localImports/preprocessor'
+import type { ImportTransformOptions } from './modules/moduleTypes'
 import { getKeywords, getProgramNames, NameDeclaration } from './name-extractor'
 import { parse } from './parser/parser'
 import { decodeError, decodeValue } from './parser/scheme'
@@ -56,6 +58,8 @@ export interface IOptions {
   isPrelude: boolean
   throwInfiniteLoops: boolean
   envSteps: number
+
+  importOptions: ImportTransformOptions
 }
 
 // needed to work on browsers
@@ -303,7 +307,7 @@ export function getTypeInformation(
 export async function runInContext(
   code: string,
   context: Context,
-  options: Partial<IOptions> = {}
+  options: RecursivePartial<IOptions> = {}
 ): Promise<Result> {
   const defaultFilePath = '/default.js'
   const files: Partial<Record<string, string>> = {}
@@ -315,7 +319,7 @@ export async function runFilesInContext(
   files: Partial<Record<string, string>>,
   entrypointFilePath: string,
   context: Context,
-  options: Partial<IOptions> = {}
+  options: RecursivePartial<IOptions> = {}
 ): Promise<Result> {
   for (const filePath in files) {
     const filePathError = validateFilePath(filePath)
@@ -340,7 +344,13 @@ export async function runFilesInContext(
     if (program === null) {
       return resolvedErrorPromise
     }
-    return fullJSRunner(program, context, options)
+    const fullImportOptions = {
+      loadTabs: true,
+      checkImports: false,
+      wrapSourceModules: false,
+      ...options.importOptions
+    }
+    return fullJSRunner(program, context, fullImportOptions)
   }
 
   if (context.chapter === Chapter.HTML) {

src/infiniteLoops/__tests__/instrument.ts

@@ -37,7 +37,11 @@ function mockFunctionsAndState() {
  * Returns the value saved in the code using the builtin 'output'.
  * e.g. runWithMock('output(2)') --> 2
  */
-function runWithMock(main: string, codeHistory?: string[], builtins: Map<string, any> = new Map()) {
+async function runWithMock(
+  main: string,
+  codeHistory?: string[],
+  builtins: Map<string, any> = new Map()
+) {
   let output = undefined
   builtins.set('output', (x: any) => (output = x))
   builtins.set('undefined', undefined)
@@ -53,43 +57,45 @@ function runWithMock(main: string, codeHistory?: string[], builtins: Map<string,
     previous = restOfCode as Program[]
   }
   const [mockFunctions, mockState] = mockFunctionsAndState()
-  const instrumentedCode = instrument(previous, program as Program, builtins.keys())
+  const instrumentedCode = await instrument(previous, program as Program, builtins.keys())
   const { builtinsId, functionsId, stateId } = InfiniteLoopRuntimeObjectNames
   const sandboxedRun = new Function('code', functionsId, stateId, builtinsId, `return eval(code)`)
-  sandboxedRun(instrumentedCode, mockFunctions, mockState, builtins)
+  await sandboxedRun(instrumentedCode, mockFunctions, mockState, builtins)
   return output
 }
 
 test('builtins work', () => {
   const main = 'output(2);'
-  expect(runWithMock(main, [])).toBe(2)
+  return expect(runWithMock(main, [])).resolves.toBe(2)
 })
 
 test('binary and unary expressions work', () => {
-  expect(runWithMock('output(1+1);', [])).toBe(2)
-  expect(runWithMock('output(!true);', [])).toBe(false)
+  return Promise.all([
+    expect(runWithMock('output(1+1);', [])).resolves.toBe(2),
+    expect(runWithMock('output(!true);', [])).resolves.toBe(false)
+  ])
 })
 
 test('assignment works as expected', () => {
   const main = `let x = 2;
   let a = [];
   a[0] = 3;
   output(x+a[0]);`
-  expect(runWithMock(main)).toBe(5)
+  return expect(runWithMock(main)).resolves.toBe(5)
 })
 
 test('globals from old code accessible', () => {
   const main = 'output(z+1);'
   const prev = ['const z = w+1;', 'let w = 10;']
-  expect(runWithMock(main, prev)).toBe(12)
+  return expect(runWithMock(main, prev)).resolves.toBe(12)
 })
 
 test('functions run as expected', () => {
   const main = `function f(x,y) {
     return x===0?x:f(x-1,y)+y;
   }
   output(f(5,2));`
-  expect(runWithMock(main)).toBe(10)
+  return expect(runWithMock(main)).resolves.toBe(10)
 })
 
 test('nested functions run as expected', () => {
@@ -100,22 +106,22 @@ test('nested functions run as expected', () => {
     return x===0?x:f(x-1,y)+y;
   }
   output(f(5,2));`
-  expect(runWithMock(main)).toBe(2)
+  return expect(runWithMock(main)).resolves.toBe(2)
 })
 
 test('higher order functions run as expected', () => {
   const main = `function run(f, x) {
     return f(x+1);
   }
   output(run(x=>x+1, 1));`
-  expect(runWithMock(main)).toBe(3)
+  return expect(runWithMock(main)).resolves.toBe(3)
 })
 
 test('loops run as expected', () => {
   const main = `let w = 0;
   for (let i = w; i < 10; i=i+1) {w = i;}
   output(w);`
-  expect(runWithMock(main)).toBe(9)
+  return expect(runWithMock(main)).resolves.toBe(9)
 })
 
 test('nested loops run as expected', () => {
@@ -126,13 +132,13 @@ test('nested loops run as expected', () => {
     }
   }
   output(w);`
-  expect(runWithMock(main)).toBe(100)
+  return expect(runWithMock(main)).resolves.toBe(100)
 })
 
 test('multidimentional arrays work', () => {
   const main = `const x = [[1],[2]];
   output(x[1] === undefined? undefined: x[1][0]);`
-  expect(runWithMock(main)).toBe(2)
+  return expect(runWithMock(main)).resolves.toBe(2)
 })
 
 test('if statements work as expected', () => {
@@ -141,7 +147,7 @@ test('if statements work as expected', () => {
     x = x + 1;
   } else {}
   output(x);`
-  expect(runWithMock(main)).toBe(2)
+  return expect(runWithMock(main)).resolves.toBe(2)
 })
 
 test('combination of loops and functions run as expected', () => {
@@ -158,5 +164,5 @@ test('combination of loops and functions run as expected', () => {
     w = minus(w,1);
   }
   output(z);`
-  expect(runWithMock(main)).toBe(100)
+  return expect(runWithMock(main)).resolves.toBe(100)
 })