From f9cdfb7cbeeb0e46d0b61b757d140e13cfebda8b Mon Sep 17 00:00:00 2001 From: leeyi45 Date: Mon, 21 Aug 2023 02:37:42 +0800 Subject: [PATCH 01/24] Add async loading code --- src/modules/__mocks__/moduleLoader.ts | 15 ++ src/modules/__mocks__/moduleLoaderAsync.ts | 32 ++++ src/modules/__tests__/moduleLoaderAsync.ts | 185 +++++++++++++++++++++ src/modules/errors.ts | 52 +++++- src/modules/moduleLoader.ts | 4 +- src/modules/moduleLoaderAsync.ts | 110 ++++++++++++ src/modules/moduleTypes.ts | 8 +- 7 files changed, 401 insertions(+), 5 deletions(-) create mode 100644 src/modules/__mocks__/moduleLoader.ts create mode 100644 src/modules/__mocks__/moduleLoaderAsync.ts create mode 100644 src/modules/__tests__/moduleLoaderAsync.ts create mode 100644 src/modules/moduleLoaderAsync.ts diff --git a/src/modules/__mocks__/moduleLoader.ts b/src/modules/__mocks__/moduleLoader.ts new file mode 100644 index 000000000..a7391b8e0 --- /dev/null +++ b/src/modules/__mocks__/moduleLoader.ts @@ -0,0 +1,15 @@ +export function loadModuleBundle() { + return { + foo: () => 'foo', + bar: () => 'bar' + } +} + +export function loadModuleTabs() { + return [] +} +export const memoizedGetModuleManifest = () => ({ + one_module: { tabs: [] }, + other_module: { tabs: [] }, + another_module: { tabs: [] } +}) diff --git a/src/modules/__mocks__/moduleLoaderAsync.ts b/src/modules/__mocks__/moduleLoaderAsync.ts new file mode 100644 index 000000000..1c98c18a5 --- /dev/null +++ b/src/modules/__mocks__/moduleLoaderAsync.ts @@ -0,0 +1,32 @@ +export const memoizedGetModuleDocsAsync = jest.fn().mockResolvedValue({ + foo: 'foo', + bar: 'bar' +}) + +export const memoizedGetModuleBundleAsync = jest.fn().mockResolvedValue( + `require => ({ + foo: () => 'foo', + bar: () => 'bar', +})` +) + +export const memoizedGetModuleManifestAsync = jest.fn().mockResolvedValue({ + one_module: { tabs: [] }, + other_module: { tabs: [] }, + another_module: { tabs: [] } +}) + +export function loadModuleBundleAsync() { + return Promise.resolve({ + foo: () => 'foo', + bar: () => 'bar' + }) +} + +export function loadModuleTabsAsync() { + return Promise.resolve([]) +} + +export function checkModuleExists() { + return Promise.resolve(true) +} diff --git a/src/modules/__tests__/moduleLoaderAsync.ts b/src/modules/__tests__/moduleLoaderAsync.ts new file mode 100644 index 000000000..ed2473098 --- /dev/null +++ b/src/modules/__tests__/moduleLoaderAsync.ts @@ -0,0 +1,185 @@ +import type { MockedFunction } from 'jest-mock' + +import { mockContext } from '../../mocks/context' +import { Chapter, Variant } from '../../types' +import { ModuleConnectionError, ModuleInternalError } from '../errors' +import { MODULES_STATIC_URL } from '../moduleLoader' +import * as moduleLoader from '../moduleLoaderAsync' + +jest.mock('lodash', () => ({ + ...jest.requireActual('lodash'), + memoize: jest.fn(x => x) +})) + +global.fetch = jest.fn() +const mockedFetch = fetch as MockedFunction + +function mockResponse(response: string, status: number = 200) { + mockedFetch.mockResolvedValueOnce({ + text: () => Promise.resolve(response), + json: () => Promise.resolve(JSON.parse(response)), + status + } as any) +} + +async function expectSuccess( + correctUrl: string, + expectedResp: T, + func: () => Promise, + callCount: number = 1 +) { + const response = await func() + + expect(fetch).toHaveBeenCalledTimes(callCount) + + const [calledUrl, callOpts] = mockedFetch.mock.calls[0] + expect(calledUrl).toEqual(correctUrl) + expect(callOpts).toMatchObject({ method: 'GET' }) + if (typeof expectedResp === 'string') { + expect(response).toEqual(expectedResp) + } else { + expect(response).toMatchObject(expectedResp) + } +} + +async function expectFailure(sampleUrl: string, expectedErr: any, func: () => Promise) { + await expect(() => func()).rejects.toBeInstanceOf(expectedErr) + + const [calledUrl, callOpts] = mockedFetch.mock.calls[0] + expect(calledUrl).toEqual(sampleUrl) + expect(callOpts).toMatchObject({ method: 'GET' }) +} + +beforeEach(() => { + jest.clearAllMocks() +}) + +describe('Test httpGetAsync', () => { + test('Http GET function httpGetAsync() works correctly', async () => { + const sampleResponse = `{ "repeat": { "contents": ["Repeat"] } }` + const sampleUrl = 'https://www.example.com' + + mockResponse(sampleResponse) + await expectSuccess(sampleUrl, sampleResponse, () => + moduleLoader.httpGetAsync(sampleUrl, 'text') + ) + }) + + test('Http GET function httpGetAsync() throws ModuleConnectionError', async () => { + const sampleUrl = 'https://www.example.com' + mockResponse('', 404) + + await expectFailure(sampleUrl, ModuleConnectionError, () => + moduleLoader.httpGetAsync(sampleUrl, 'text') + ) + }) + + test('Http GET modules manifest correctly', async () => { + const sampleResponse = `{ "repeat": { "contents": ["Repeat"] } }` + const correctUrl = MODULES_STATIC_URL + `/modules.json` + mockResponse(sampleResponse) + + await expectSuccess(correctUrl, JSON.parse(sampleResponse), () => + moduleLoader.memoizedGetModuleManifestAsync() + ) + }) + + test('Http GET returns objects when "json" is specified', async () => { + const sampleResponse = `{ "repeat": { "contents": ["Repeat"] } }` + const correctUrl = MODULES_STATIC_URL + `/modules.json` + mockResponse(sampleResponse) + const result = await moduleLoader.httpGetAsync(correctUrl, 'json') + expect(result).toMatchObject(JSON.parse(sampleResponse)) + }) + + test('Handles TypeErrors thrown by fetch', async () => { + mockedFetch.mockImplementationOnce(() => { + throw new TypeError() + }) + await expectFailure('anyUrl', ModuleConnectionError, () => + moduleLoader.httpGetAsync('anyUrl', 'text') + ) + }) +}) + +describe('Test bundle loading', () => { + const sampleModuleName = 'valid_module' + const sampleModuleUrl = MODULES_STATIC_URL + `/bundles/${sampleModuleName}.js` + + test('Http GET module bundle correctly', async () => { + const sampleResponse = `require => ({ foo: () => 'foo' })` + mockResponse(sampleResponse) + + const bundleText = await moduleLoader.memoizedGetModuleBundleAsync(sampleModuleName) + + expect(fetch).toHaveBeenCalledTimes(1) + + const [calledUrl, callOpts] = mockedFetch.mock.calls[0] + expect(calledUrl).toEqual(sampleModuleUrl) + expect(callOpts).toMatchObject({ method: 'GET' }) + expect(bundleText).toEqual(sampleResponse) + }) + + test('Loading a correctly implemented module bundle', async () => { + const context = mockContext(Chapter.SOURCE_4, Variant.DEFAULT) + const sampleResponse = `require => ({ foo: () => 'foo' })` + mockResponse(sampleResponse) + + const loadedModule = await moduleLoader.loadModuleBundleAsync(sampleModuleName, context, false) + + expect(loadedModule.foo()).toEqual('foo') + expect(fetch).toHaveBeenCalledTimes(1) + + const [calledUrl, callOpts] = mockedFetch.mock.calls[0] + expect(calledUrl).toEqual(sampleModuleUrl) + expect(callOpts).toMatchObject({ method: 'GET' }) + }) + + test('Loading a wrongly implemented module bundle throws ModuleInternalError', async () => { + const context = mockContext(Chapter.SOURCE_4, Variant.DEFAULT) + const wrongModuleText = `export function es6_function(params) {};` + mockResponse(wrongModuleText) + await expect(() => + moduleLoader.loadModuleBundleAsync(sampleModuleName, context, true) + ).rejects.toBeInstanceOf(ModuleInternalError) + + expect(fetch).toHaveBeenCalledTimes(1) + }) +}) + +describe('Test tab loading', () => { + const sampleTabUrl = `${MODULES_STATIC_URL}/tabs/Tab1.js` + const sampleManifest = `{ "one_module": { "tabs": ["Tab1", "Tab2"] } }` + + test('Http GET module tab correctly', async () => { + const sampleResponse = `require => ({ foo: () => 'foo' })` + mockResponse(sampleResponse) + + const bundleText = await moduleLoader.memoizedGetModuleTabAsync('Tab1') + + expect(fetch).toHaveBeenCalledTimes(1) + + const [calledUrl, callOpts] = mockedFetch.mock.calls[0] + expect(calledUrl).toEqual(sampleTabUrl) + expect(callOpts).toMatchObject({ method: 'GET' }) + expect(bundleText).toEqual(sampleResponse) + }) + + test('Loading a wrongly implemented tab throws ModuleInternalError', async () => { + mockResponse(sampleManifest) + + const wrongTabText = `export function es6_function(params) {};` + mockResponse(wrongTabText) + mockResponse(wrongTabText) + + await expect(() => moduleLoader.loadModuleTabsAsync('one_module')).rejects.toBeInstanceOf( + ModuleInternalError + ) + expect(fetch).toHaveBeenCalledTimes(3) + + const [[call0Url], [call1Url], [call2Url]] = mockedFetch.mock.calls + expect(call0Url).toEqual(`${MODULES_STATIC_URL}/modules.json`) + expect(call1Url).toEqual(`${MODULES_STATIC_URL}/tabs/Tab1.js`) + expect(call2Url).toEqual(`${MODULES_STATIC_URL}/tabs/Tab2.js`) + }) +}) diff --git a/src/modules/errors.ts b/src/modules/errors.ts index fa3ebaef0..4f69c1939 100644 --- a/src/modules/errors.ts +++ b/src/modules/errors.ts @@ -1,4 +1,4 @@ -import type { ImportDeclaration } from 'estree' +import type * as es from 'estree' import { RuntimeSourceError } from '../errors/runtimeSourceError' @@ -6,7 +6,7 @@ export class UndefinedImportError extends RuntimeSourceError { constructor( public readonly symbol: string, public readonly moduleName: string, - node?: ImportDeclaration + node?: es.ImportSpecifier ) { super(node) } @@ -19,3 +19,51 @@ export class UndefinedImportError extends RuntimeSourceError { return "Check your imports and make sure what you're trying to import exists!" } } + +export class ModuleConnectionError extends RuntimeSourceError { + private static message: string = `Unable to get modules.` + private static elaboration: string = `You should check your Internet connection, and ensure you have used the correct module path.` + constructor(node?: es.Node) { + super(node) + } + + public explain() { + return ModuleConnectionError.message + } + + public elaborate() { + return ModuleConnectionError.elaboration + } +} + +export class ModuleNotFoundError extends RuntimeSourceError { + constructor(public moduleName: string, node?: es.Node) { + super(node) + } + + public explain() { + return `Module "${this.moduleName}" not found.` + } + + public elaborate() { + return ` + You should check your import declarations, and ensure that all are valid modules. + ` + } +} + +export class ModuleInternalError extends RuntimeSourceError { + constructor(public moduleName: string, public error?: any, node?: es.Node) { + super(node) + } + + public explain() { + return `Error(s) occured when executing the module "${this.moduleName}".` + } + + public elaborate() { + return ` + You may need to contact with the author for this module to fix this error. + ` + } +} diff --git a/src/modules/moduleLoader.ts b/src/modules/moduleLoader.ts index 2801ce81a..2e05e67ce 100644 --- a/src/modules/moduleLoader.ts +++ b/src/modules/moduleLoader.ts @@ -9,7 +9,7 @@ import { } from '../errors/moduleErrors' import { Context } from '../types' import { wrapSourceModule } from '../utils/operators' -import { ModuleBundle, ModuleDocumentation, ModuleFunctions, Modules } from './moduleTypes' +import { ModuleBundle, ModuleDocumentation, ModuleFunctions, ModuleManifest } from './moduleTypes' import { getRequireProvider } from './requireProvider' // Supports both JSDom (Web Browser) environment and Node environment @@ -46,7 +46,7 @@ export function httpGet(url: string): string { * @return Modules */ export const memoizedGetModuleManifest = memoize(getModuleManifest) -function getModuleManifest(): Modules { +function getModuleManifest(): ModuleManifest { const rawManifest = httpGet(`${MODULES_STATIC_URL}/modules.json`) return JSON.parse(rawManifest) } diff --git a/src/modules/moduleLoaderAsync.ts b/src/modules/moduleLoaderAsync.ts new file mode 100644 index 000000000..314050062 --- /dev/null +++ b/src/modules/moduleLoaderAsync.ts @@ -0,0 +1,110 @@ +import type { Node } from 'estree' +import { memoize } from 'lodash' + +import type { Context } from '..' +import { TimeoutError, timeoutPromise } from '../utils/misc' +import { wrapSourceModule } from '../utils/operators' +import { ModuleConnectionError, ModuleInternalError, ModuleNotFoundError } from './errors' +import { MODULES_STATIC_URL } from './moduleLoader' +import type { ModuleBundle, ModuleDocumentation, ModuleManifest } from './moduleTypes' +import { getRequireProvider } from './requireProvider' + +export function httpGetAsync(path: string, type: 'json'): Promise +export function httpGetAsync(path: string, type: 'text'): Promise +export async function httpGetAsync(path: string, type: 'json' | 'text') { + try { + const resp = await timeoutPromise( + fetch(path, { + method: 'GET' + }), + 10000 + ) + + if (resp.status !== 200 && resp.status !== 304) { + throw new ModuleConnectionError() + } + + const promise = type === 'text' ? resp.text() : resp.json() + return timeoutPromise(promise, 10000) + } catch (error) { + if (error instanceof TypeError || error instanceof TimeoutError) { + throw new ModuleConnectionError() + } + if (!(error instanceof ModuleConnectionError)) throw new ModuleInternalError(path, error) + throw error + } +} + +/** + * Send a HTTP GET request to the modules endpoint to retrieve the manifest + * @return Modules + */ +export const memoizedGetModuleManifestAsync = memoize(getModuleManifestAsync) +function getModuleManifestAsync(): Promise { + return httpGetAsync(`${MODULES_STATIC_URL}/modules.json`, 'json') as Promise +} + +async function checkModuleExists(moduleName: string, node?: Node) { + const modules = await memoizedGetModuleManifestAsync() + // Check if the module exists + if (!(moduleName in modules)) throw new ModuleNotFoundError(moduleName, node) + + return modules[moduleName] +} + +export const memoizedGetModuleBundleAsync = memoize(getModuleBundleAsync) +async function getModuleBundleAsync(moduleName: string): Promise { + return httpGetAsync(`${MODULES_STATIC_URL}/bundles/${moduleName}.js`, 'text') +} + +export const memoizedGetModuleTabAsync = memoize(getModuleTabAsync) +function getModuleTabAsync(tabName: string): Promise { + return httpGetAsync(`${MODULES_STATIC_URL}/tabs/${tabName}.js`, 'text') +} + +export const memoizedGetModuleDocsAsync = memoize(getModuleDocsAsync) +async function getModuleDocsAsync(moduleName: string): Promise { + try { + const result = await httpGetAsync(`${MODULES_STATIC_URL}/jsons/${moduleName}.json`, 'json') + return result as ModuleDocumentation + } catch (error) { + console.warn(`Failed to load documentation for ${moduleName}:`, error) + return null + } +} + +export async function loadModuleTabsAsync(moduleName: string, node?: Node) { + const moduleInfo = await checkModuleExists(moduleName, node) + + // Load the tabs for the current module + return Promise.all( + moduleInfo.tabs.map(async path => { + const rawTabFile = await memoizedGetModuleTabAsync(path) + try { + return eval(rawTabFile) + } catch (error) { + // console.error('tab error:', error); + throw new ModuleInternalError(path, error, node) + } + }) + ) +} + +export async function loadModuleBundleAsync( + moduleName: string, + context: Context, + wrapModule: boolean, + node?: Node +) { + // await checkModuleExists(moduleName, node) + const moduleText = await memoizedGetModuleBundleAsync(moduleName) + try { + const moduleBundle: ModuleBundle = eval(moduleText) + + if (wrapModule) return wrapSourceModule(moduleName, moduleBundle, getRequireProvider(context)) + return moduleBundle(getRequireProvider(context)) + } catch (error) { + // console.error("bundle error: ", error) + throw new ModuleInternalError(moduleName, error, node) + } +} diff --git a/src/modules/moduleTypes.ts b/src/modules/moduleTypes.ts index 974f27de4..8182f0a17 100644 --- a/src/modules/moduleTypes.ts +++ b/src/modules/moduleTypes.ts @@ -1,6 +1,6 @@ import type { RequireProvider } from './requireProvider' -export type Modules = { +export type ModuleManifest = { [module: string]: { tabs: string[] } @@ -13,3 +13,9 @@ export type ModuleFunctions = { } export type ModuleDocumentation = Record + +export type ImportTransformOptions = { + wrapSourceModules: boolean + loadTabs: boolean + checkImports: boolean +} From 5efb8a5345b628300249d116f50a237f25499989 Mon Sep 17 00:00:00 2001 From: leeyi45 Date: Mon, 21 Aug 2023 02:38:24 +0800 Subject: [PATCH 02/24] Made infinite loop checking async --- src/infiniteLoops/__tests__/instrument.ts | 38 +++++---- src/infiniteLoops/__tests__/runtime.ts | 99 +++++++++++++---------- src/infiniteLoops/instrument.ts | 35 +++++--- src/infiniteLoops/runtime.ts | 9 ++- 4 files changed, 106 insertions(+), 75 deletions(-) diff --git a/src/infiniteLoops/__tests__/instrument.ts b/src/infiniteLoops/__tests__/instrument.ts index f4f69c924..56886b86a 100644 --- a/src/infiniteLoops/__tests__/instrument.ts +++ b/src/infiniteLoops/__tests__/instrument.ts @@ -37,7 +37,11 @@ function mockFunctionsAndState() { * Returns the value saved in the code using the builtin 'output'. * e.g. runWithMock('output(2)') --> 2 */ -function runWithMock(main: string, codeHistory?: string[], builtins: Map = new Map()) { +async function runWithMock( + main: string, + codeHistory?: string[], + builtins: Map = new Map() +) { let output = undefined builtins.set('output', (x: any) => (output = x)) builtins.set('undefined', undefined) @@ -53,21 +57,23 @@ function runWithMock(main: string, codeHistory?: string[], builtins: Map { const main = 'output(2);' - expect(runWithMock(main, [])).toBe(2) + return expect(runWithMock(main, [])).resolves.toBe(2) }) test('binary and unary expressions work', () => { - expect(runWithMock('output(1+1);', [])).toBe(2) - expect(runWithMock('output(!true);', [])).toBe(false) + return Promise.all([ + expect(runWithMock('output(1+1);', [])).resolves.toBe(2), + expect(runWithMock('output(!true);', [])).resolves.toBe(false) + ]) }) test('assignment works as expected', () => { @@ -75,13 +81,13 @@ test('assignment works as expected', () => { let a = []; a[0] = 3; output(x+a[0]);` - expect(runWithMock(main)).toBe(5) + return expect(runWithMock(main)).resolves.toBe(5) }) test('globals from old code accessible', () => { const main = 'output(z+1);' const prev = ['const z = w+1;', 'let w = 10;'] - expect(runWithMock(main, prev)).toBe(12) + return expect(runWithMock(main, prev)).resolves.toBe(12) }) test('functions run as expected', () => { @@ -89,7 +95,7 @@ test('functions run as expected', () => { return x===0?x:f(x-1,y)+y; } output(f(5,2));` - expect(runWithMock(main)).toBe(10) + return expect(runWithMock(main)).resolves.toBe(10) }) test('nested functions run as expected', () => { @@ -100,7 +106,7 @@ test('nested functions run as expected', () => { return x===0?x:f(x-1,y)+y; } output(f(5,2));` - expect(runWithMock(main)).toBe(2) + return expect(runWithMock(main)).resolves.toBe(2) }) test('higher order functions run as expected', () => { @@ -108,14 +114,14 @@ test('higher order functions run as expected', () => { return f(x+1); } output(run(x=>x+1, 1));` - expect(runWithMock(main)).toBe(3) + return expect(runWithMock(main)).resolves.toBe(3) }) test('loops run as expected', () => { const main = `let w = 0; for (let i = w; i < 10; i=i+1) {w = i;} output(w);` - expect(runWithMock(main)).toBe(9) + return expect(runWithMock(main)).resolves.toBe(9) }) test('nested loops run as expected', () => { @@ -126,13 +132,13 @@ test('nested loops run as expected', () => { } } output(w);` - expect(runWithMock(main)).toBe(100) + return expect(runWithMock(main)).resolves.toBe(100) }) test('multidimentional arrays work', () => { const main = `const x = [[1],[2]]; output(x[1] === undefined? undefined: x[1][0]);` - expect(runWithMock(main)).toBe(2) + return expect(runWithMock(main)).resolves.toBe(2) }) test('if statements work as expected', () => { @@ -141,7 +147,7 @@ test('if statements work as expected', () => { x = x + 1; } else {} output(x);` - expect(runWithMock(main)).toBe(2) + return expect(runWithMock(main)).resolves.toBe(2) }) test('combination of loops and functions run as expected', () => { @@ -158,5 +164,5 @@ test('combination of loops and functions run as expected', () => { w = minus(w,1); } output(z);` - expect(runWithMock(main)).toBe(100) + return expect(runWithMock(main)).resolves.toBe(100) }) diff --git a/src/infiniteLoops/__tests__/runtime.ts b/src/infiniteLoops/__tests__/runtime.ts index 4eb48dae8..d6f6dd1a9 100644 --- a/src/infiniteLoops/__tests__/runtime.ts +++ b/src/infiniteLoops/__tests__/runtime.ts @@ -3,15 +3,15 @@ import * as es from 'estree' import { runInContext } from '../..' import createContext from '../../createContext' import { mockContext } from '../../mocks/context' -import * as moduleLoader from '../../modules/moduleLoader' import { parse } from '../../parser/parser' import { Chapter, Variant } from '../../types' import { stripIndent } from '../../utils/formatters' import { getInfiniteLoopData, InfiniteLoopError, InfiniteLoopErrorType } from '../errors' import { testForInfiniteLoop } from '../runtime' -jest.spyOn(moduleLoader, 'memoizedGetModuleFile').mockImplementationOnce(() => { - return stripIndent` +jest.mock('../../modules/moduleLoaderAsync', () => ({ + memoizedGetModuleBundleAsync: jest.fn(() => + Promise.resolve(stripIndent` require => { 'use strict'; var exports = {}; @@ -36,8 +36,21 @@ jest.spyOn(moduleLoader, 'memoizedGetModuleFile').mockImplementationOnce(() => { }); return exports; } - ` -}) + `) + ), + memoizedGetModuleManifestAsync: jest.fn(() => + Promise.resolve({ + repeat: { tabs: [] } + }) + ), + memoizedGetModuleDocsAsync: jest.fn(() => + Promise.resolve({ + repeat: '', + twice: '', + thrice: '' + }) + ) +})) test('works in runInContext when throwInfiniteLoops is true', async () => { const code = `function fib(x) { @@ -77,84 +90,84 @@ const testForInfiniteLoopWithCode = (code: string, previousPrograms: es.Program[ return testForInfiniteLoop(program, previousPrograms) } -test('non-infinite recursion not detected', () => { +test('non-infinite recursion not detected', async () => { const code = `function fib(x) { return x<=1?x:fib(x-1) + fib(x-2); } fib(100000); ` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result).toBeUndefined() }) -test('non-infinite loop not detected', () => { +test('non-infinite loop not detected', async () => { const code = `for(let i = 0;i<2000;i=i+1){i+1;} let j = 0; while(j<2000) {j=j+1;} ` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result).toBeUndefined() }) -test('no base case function detected', () => { +test('no base case function detected', async () => { const code = `function fib(x) { return fib(x-1) + fib(x-2); } fib(100000); ` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result?.infiniteLoopType).toBe(InfiniteLoopErrorType.NoBaseCase) expect(result?.streamMode).toBe(false) }) -test('no base case loop detected', () => { +test('no base case loop detected', async () => { const code = `for(let i = 0;true;i=i+1){i+1;} ` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result?.infiniteLoopType).toBe(InfiniteLoopErrorType.NoBaseCase) expect(result?.streamMode).toBe(false) }) -test('no variables changing function detected', () => { +test('no variables changing function detected', async () => { const code = `let x = 1; function f() { return x===0?x:f(); } f(); ` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result?.infiniteLoopType).toBe(InfiniteLoopErrorType.Cycle) expect(result?.streamMode).toBe(false) expect(result?.explain()).toContain('None of the variables are being updated.') }) -test('no state change function detected', () => { +test('no state change function detected', async () => { const code = `let x = 1; function f() { return x===0?x:f(); } f(); ` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result?.infiniteLoopType).toBe(InfiniteLoopErrorType.Cycle) expect(result?.streamMode).toBe(false) expect(result?.explain()).toContain('None of the variables are being updated.') }) -test('infinite cycle detected', () => { +test('infinite cycle detected', async () => { const code = `function f(x) { return x[0] === 1? x : f(x); } f([2,3,4]); ` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result?.infiniteLoopType).toBe(InfiniteLoopErrorType.Cycle) expect(result?.streamMode).toBe(false) expect(result?.explain()).toContain('cycle') expect(result?.explain()).toContain('[2,3,4]') }) -test('infinite data structures detected', () => { +test('infinite data structures detected', async () => { const code = `function f(x) { return is_null(x)? x : f(tail(x)); } @@ -162,32 +175,32 @@ test('infinite data structures detected', () => { set_tail(tail(tail(circ)), circ); f(circ); ` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result?.infiniteLoopType).toBe(InfiniteLoopErrorType.Cycle) expect(result?.streamMode).toBe(false) expect(result?.explain()).toContain('cycle') expect(result?.explain()).toContain('(CIRCULAR)') }) -test('functions using SMT work', () => { +test('functions using SMT work', async () => { const code = `function f(x) { return x===0? x: f(x+1); } f(1); ` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result?.infiniteLoopType).toBe(InfiniteLoopErrorType.FromSmt) expect(result?.streamMode).toBe(false) }) -test('detect forcing infinite streams', () => { +test('detect forcing infinite streams', async () => { const code = `stream_to_list(integers_from(0));` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result?.infiniteLoopType).toBe(InfiniteLoopErrorType.NoBaseCase) expect(result?.streamMode).toBe(true) }) -test('detect mutual recursion', () => { +test('detect mutual recursion', async () => { const code = `function e(x){ return x===0?1:1-o(x-1); } @@ -195,23 +208,23 @@ test('detect mutual recursion', () => { return x===1?0:1-e(x-1); } e(9);` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result?.infiniteLoopType).toBe(InfiniteLoopErrorType.FromSmt) expect(result?.streamMode).toBe(false) }) -test('functions passed as arguments not checked', () => { +test('functions passed as arguments not checked', async () => { // if they are checked -> this will throw no base case const code = `const twice = f => x => f(f(x)); const thrice = f => x => f(f(f(x))); const add = x => x + 1; (thrice)(twice(twice))(twice(add))(0);` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result).toBeUndefined() }) -test('detect complicated cycle example', () => { +test('detect complicated cycle example', async () => { const code = `function permutations(s) { return is_null(s) ? list(null) @@ -230,12 +243,12 @@ test('detect complicated cycle example', () => { remove_duplicate(list(list(1,2,3), list(1,2,3))); ` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result?.infiniteLoopType).toBe(InfiniteLoopErrorType.Cycle) expect(result?.streamMode).toBe(false) }) -test('detect complicated cycle example 2', () => { +test('detect complicated cycle example 2', async () => { const code = `function make_big_int_from_number(num){ let output = num; while(output !== 0){ @@ -246,12 +259,12 @@ test('detect complicated cycle example 2', () => { } make_big_int_from_number(1234); ` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result?.infiniteLoopType).toBe(InfiniteLoopErrorType.Cycle) expect(result?.streamMode).toBe(false) }) -test('detect complicated fromSMT example 2', () => { +test('detect complicated fromSMT example 2', async () => { const code = `function fast_power(b,n){ if (n % 2 === 0){ return b* fast_power(b, n-2); @@ -261,47 +274,47 @@ test('detect complicated fromSMT example 2', () => { } fast_power(2,3);` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result?.infiniteLoopType).toBe(InfiniteLoopErrorType.FromSmt) expect(result?.streamMode).toBe(false) }) -test('detect complicated stream example', () => { +test('detect complicated stream example', async () => { const code = `function up(a, b) { return (a > b) ? up(1, 1 + b) : pair(a, () => stream_reverse(up(a + 1, b))); } eval_stream(up(1,1), 22);` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result).toBeDefined() expect(result?.streamMode).toBe(true) }) -test('math functions are disabled in smt solver', () => { +test('math functions are disabled in smt solver', async () => { const code = ` function f(x) { return x===0? x: f(math_floor(x+1)); } f(1);` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result).toBeUndefined() }) -test('cycle detection ignores non deterministic functions', () => { +test('cycle detection ignores non deterministic functions', async () => { const code = ` function f(x) { return x===0?0:f(math_floor(math_random()/2) + 1); } f(1);` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result).toBeUndefined() }) -test('handle imports properly', () => { +test('handle imports properly', async () => { const code = `import {thrice} from "repeat"; function f(x) { return is_number(x) ? f(x) : 42; } display(f(thrice(x=>x+1)(0)));` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result?.infiniteLoopType).toBe(InfiniteLoopErrorType.Cycle) }) diff --git a/src/infiniteLoops/instrument.ts b/src/infiniteLoops/instrument.ts index c8c98d0e8..6b3af9749 100644 --- a/src/infiniteLoops/instrument.ts +++ b/src/infiniteLoops/instrument.ts @@ -574,25 +574,34 @@ function trackLocations(program: es.Program) { }) } -function handleImports(programs: es.Program[]): [string, string[]] { - const [prefixes, imports] = programs.reduce( - ([prefix, moduleNames], program) => { - const [prefixToAdd, importsToAdd, otherNodes] = transformImportDeclarations( +async function handleImports(programs: es.Program[]): Promise<[string, string[]]> { + const transformed = await Promise.all( + programs.map(async program => { + const [prefixToAdd, importsToAdd, otherNodes] = await transformImportDeclarations( program, new Set(), - false + { + wrapSourceModules: false, + checkImports: false, + loadTabs: false + } ) program.body = (importsToAdd as es.Program['body']).concat(otherNodes) - prefix.push(prefixToAdd) - const importedNames = importsToAdd.flatMap(node => node.declarations.map( decl => ((decl.init as es.MemberExpression).object as es.Identifier).name ) ) - return [prefix, moduleNames.concat(importedNames)] - }, - [[] as string[], [] as string[]] + return [prefixToAdd, importedNames] as [string, string[]] + }) + ) + + const [prefixes, imports] = transformed.reduce( + ([prefixes, moduleNames], [prefix, importedNames]) => [ + [...prefixes, prefix], + [...moduleNames, ...importedNames] + ], + [[], []] as [string[], string[]] ) return [prefixes.join('\n'), [...new Set(imports)]] @@ -606,11 +615,11 @@ function handleImports(programs: es.Program[]): [string, string[]] { * @param builtins Names of builtin functions. * @returns code with instrumentations. */ -function instrument( +async function instrument( previous: es.Program[], program: es.Program, builtins: Iterable -): string { +): Promise { const { builtinsId, functionsId, stateId } = globalIds const predefined = {} predefined[builtinsId] = builtinsId @@ -618,7 +627,7 @@ function instrument( predefined[stateId] = stateId const innerProgram = { ...program } - const [prefix, moduleNames] = handleImports([program].concat(previous)) + const [prefix, moduleNames] = await handleImports([program].concat(previous)) for (const name of moduleNames) { predefined[name] = name } diff --git a/src/infiniteLoops/runtime.ts b/src/infiniteLoops/runtime.ts index a9b81b95f..83783e251 100644 --- a/src/infiniteLoops/runtime.ts +++ b/src/infiniteLoops/runtime.ts @@ -305,7 +305,10 @@ functions[FunctionNames.evalU] = sym.evaluateHybridUnary * @param previousProgramsStack Any code previously entered in the REPL & parsed into AST. * @returns SourceError if an infinite loop was detected, undefined otherwise. */ -export function testForInfiniteLoop(program: es.Program, previousProgramsStack: es.Program[]) { +export async function testForInfiniteLoop( + program: es.Program, + previousProgramsStack: es.Program[] +) { const context = createContext(Chapter.SOURCE_4, Variant.DEFAULT, undefined, undefined) const prelude = parse(context.prelude as string, context) as es.Program context.prelude = null @@ -313,7 +316,7 @@ export function testForInfiniteLoop(program: es.Program, previousProgramsStack: const newBuiltins = prepareBuiltins(context.nativeStorage.builtins) const { builtinsId, functionsId, stateId } = InfiniteLoopRuntimeObjectNames - const instrumentedCode = instrument(previous, program, newBuiltins.keys()) + const instrumentedCode = await instrument(previous, program, newBuiltins.keys()) const state = new st.State() const sandboxedRun = new Function( @@ -327,7 +330,7 @@ export function testForInfiniteLoop(program: es.Program, previousProgramsStack: ) try { - sandboxedRun(instrumentedCode, functions, state, newBuiltins, getRequireProvider(context)) + await sandboxedRun(instrumentedCode, functions, state, newBuiltins, getRequireProvider(context)) } catch (error) { if (error instanceof InfiniteLoopError) { if (state.lastLocation !== undefined) { From 8c5a7ac6f768a057d5be5e0ac02bbed0bbcb82d6 Mon Sep 17 00:00:00 2001 From: leeyi45 Date: Mon, 21 Aug 2023 02:40:03 +0800 Subject: [PATCH 03/24] Update stepper to be async --- src/stepper/__tests__/stepper.ts | 371 +++++++++++++++---------------- src/stepper/stepper.ts | 99 +++++---- 2 files changed, 239 insertions(+), 231 deletions(-) diff --git a/src/stepper/__tests__/stepper.ts b/src/stepper/__tests__/stepper.ts index f70f9c0f9..f46e56700 100644 --- a/src/stepper/__tests__/stepper.ts +++ b/src/stepper/__tests__/stepper.ts @@ -1,8 +1,8 @@ -import * as es from 'estree' +import type * as es from 'estree' import { mockContext } from '../../mocks/context' import { parse } from '../../parser/parser' -import { Chapter, substituterNodes } from '../../types' +import { Chapter, Context, substituterNodes } from '../../types' import { codify, getEvaluationSteps } from '../stepper' function getLastStepAsString(steps: [substituterNodes, string[][], string][]): string { @@ -45,11 +45,12 @@ describe('Test codify works on non-circular abstract syntax graphs', () => { }) describe('Test codify works on circular abstract syntax graphs', () => { - test('functions', () => { + test('functions', async () => { const code = ` x => x(); ` const program = parse(code, mockContext())! + const arrowFunctionExpression = ((program as es.Program).body[0] as es.ExpressionStatement) .expression as es.ArrowFunctionExpression const callExpression = arrowFunctionExpression.body as es.CallExpression @@ -62,12 +63,25 @@ describe('Test codify works on circular abstract syntax graphs', () => { }) // source 0 -test('Test basic substitution', () => { +const testEvalSteps = (programStr: string, context?: Context) => { + context = context ?? mockContext() + const program = parse(programStr, context)! + const options = { + stepLimit: 1000, + importOptions: { + loadTabs: false, + wrapSourceModules: false, + checkImports: false + } + } + return getEvaluationSteps(program, context, options) +} + +test('Test basic substitution', async () => { const code = ` (1 + 2) * (3 + 4); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) expect(steps.map(x => codify(x[0])).join('\n')).toMatchInlineSnapshot(` "(1 + 2) * (3 + 4); @@ -88,13 +102,11 @@ test('Test basic substitution', () => { `) }) -test('Test binary operator error', () => { +test('Test binary operator error', async () => { const code = ` (1 + 2) * ('a' + 'string'); ` - const context = mockContext() - const program = parse(code, context)! - const steps = getEvaluationSteps(program, context, 1000) + const steps = await testEvalSteps(code) expect(steps.map(x => codify(x[0])).join('\n')).toMatchInlineSnapshot(` "(1 + 2) * ('a' + 'string'); @@ -111,13 +123,12 @@ test('Test binary operator error', () => { `) }) -test('Test two statement substitution', () => { +test('Test two statement substitution', async () => { const code = ` (1 + 2) * (3 + 4); 3 * 5; ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(Chapter.SOURCE_4), 1000) + const steps = await testEvalSteps(code, mockContext(Chapter.SOURCE_4)) expect(steps.map(x => codify(x[0])).join('\n')).toMatchInlineSnapshot(` "(1 + 2) * (3 + 4); 3 * 5; @@ -154,12 +165,11 @@ test('Test two statement substitution', () => { `) }) -test('Test unary and binary boolean operations', () => { +test('Test unary and binary boolean operations', async () => { const code = ` !!!true || true; ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) expect(steps.map(x => codify(x[0])).join('\n')).toMatchInlineSnapshot(` "!!!true || true; @@ -184,14 +194,13 @@ test('Test unary and binary boolean operations', () => { `) }) -test('Test ternary operator', () => { +test('Test ternary operator', async () => { const code = ` 1 + -1 === 0 ? false ? garbage : Infinity : anotherGarbage; ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) expect(steps.map(x => codify(x[0])).join('\n')).toMatchInlineSnapshot(` "1 + -1 === 0 ? false ? garbage : Infinity : anotherGarbage; @@ -216,15 +225,14 @@ test('Test ternary operator', () => { `) }) -test('Test basic function', () => { +test('Test basic function', async () => { const code = ` function f(n) { return n; } f(5+1*6-40); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) expect(steps.map(x => codify(x[0])).join('\n')).toMatchInlineSnapshot(` "function f(n) { return n; @@ -259,15 +267,14 @@ test('Test basic function', () => { `) }) -test('Test basic bifunction', () => { +test('Test basic bifunction', async () => { const code = ` function f(n, m) { return n * m; } f(5+1*6-40, 2-5); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) expect(steps.map(x => codify(x[0])).join('\n')).toMatchInlineSnapshot(` "function f(n, m) { return n * m; @@ -310,7 +317,7 @@ test('Test basic bifunction', () => { `) }) -test('Test "recursive" function calls', () => { +test('Test "recursive" function calls', async () => { const code = ` function factorial(n) { return n === 0 @@ -319,8 +326,7 @@ test('Test "recursive" function calls', () => { } factorial(5); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) expect(steps.map(x => codify(x[0])).join('\n')).toMatchInlineSnapshot(` "function factorial(n) { return n === 0 ? 1 : n * factorial(n - 1); @@ -452,12 +458,11 @@ test('Test "recursive" function calls', () => { }) // source 0 -test('undefined || 1', () => { +test('undefined || 1', async () => { const code = ` undefined || 1; ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) expect(steps.map(x => codify(x[0])).join('\n')).toMatchInlineSnapshot(` "undefined || 1; @@ -467,12 +472,11 @@ test('undefined || 1', () => { }) // source 0 -test('1 + math_sin', () => { +test('1 + math_sin', async () => { const code = ` 1 + math_sin; ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) expect(steps.map(x => codify(x[0])).join('\n')).toMatchInlineSnapshot(` "1 + math_sin; @@ -482,12 +486,12 @@ test('1 + math_sin', () => { }) // source 0 -test('plus undefined', () => { +test('plus undefined', async () => { const code = ` math_sin(1) + undefined; ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + + const steps = await testEvalSteps(code) expect(steps.map(x => codify(x[0])).join('\n')).toMatchInlineSnapshot(` "math_sin(1) + undefined; @@ -501,12 +505,11 @@ test('plus undefined', () => { }) // source 0 -test('math_pow', () => { +test('math_pow', async () => { const code = ` math_pow(2, 20) || NaN; ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) expect(steps.map(x => codify(x[0])).join('\n')).toMatchInlineSnapshot(` "math_pow(2, 20) || NaN; @@ -520,7 +523,7 @@ test('math_pow', () => { }) // source 0 -test('expmod', () => { +test('expmod', async () => { const code = ` function is_even(n) { return n % 2 === 0; @@ -541,26 +544,24 @@ function expmod(base, exp, m) { expmod(4, 3, 5); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() }) // source 0 -test('Infinite recursion', () => { +test('Infinite recursion', async () => { const code = ` function f() { return f(); } f(); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() }) // source 0 -test('subsets', () => { +test('subsets', async () => { const code = ` function subsets(s) { if (is_null(s)) { @@ -573,20 +574,18 @@ test('subsets', () => { subsets(list(1, 2, 3)); ` - const program = parse(code, mockContext(Chapter.SOURCE_2))! - const steps = getEvaluationSteps(program, mockContext(Chapter.SOURCE_2), 1000) + const steps = await testEvalSteps(code, mockContext(Chapter.SOURCE_2)) expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() }) // source 0 -test('even odd mutual', () => { +test('even odd mutual', async () => { const code = ` const odd = n => n === 0 ? false : even(n-1); const even = n => n === 0 || odd(n-1); even(1); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) expect(getLastStepAsString(steps)).toEqual('false;') expect(steps.map(x => codify(x[0])).join('\n')).toMatchInlineSnapshot(` "const odd = n => n === 0 ? false : even(n - 1); @@ -639,13 +638,12 @@ test('even odd mutual', () => { }) // source 0 -test('assign undefined', () => { +test('assign undefined', async () => { const code = ` const a = undefined; a; ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) expect(getLastStepAsString(steps)).toEqual('undefined;') expect(steps.map(x => codify(x[0])).join('\n')).toMatchInlineSnapshot(` "const a = undefined; @@ -661,12 +659,11 @@ test('assign undefined', () => { `) }) -test('builtins return identifiers', () => { +test('builtins return identifiers', async () => { const code = ` math_sin(); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) expect(getLastStepAsString(steps)).toEqual('NaN;') expect(steps.map(x => codify(x[0])).join('\n')).toMatchInlineSnapshot(` "math_sin(); @@ -680,12 +677,12 @@ test('builtins return identifiers', () => { `) }) -test('negative numbers as arguments', () => { +test('negative numbers as arguments', async () => { const code = ` math_sin(-1); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchInlineSnapshot(` "math_sin(-1); @@ -698,12 +695,11 @@ test('negative numbers as arguments', () => { `) }) -test('is_function checks for builtin', () => { +test('is_function checks for builtin', async () => { const code = ` is_function(is_function); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) expect(steps.map(x => codify(x[0])).join('\n')).toMatchInlineSnapshot(` "is_function(is_function); @@ -716,15 +712,14 @@ test('is_function checks for builtin', () => { `) }) -test('triple equals work on function', () => { +test('triple equals work on function', async () => { const code = ` function f() { return g(); } function g() { return f(); } f === f; g === g; f === g; ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) expect(steps.map(x => codify(x[0])).join('\n')).toMatchInlineSnapshot(` "function f() { return g(); @@ -799,7 +794,7 @@ test('triple equals work on function', () => { `) }) -test('constant declarations in blocks are protected', () => { +test('constant declarations in blocks are protected', async () => { const code = ` const z = 1; @@ -810,8 +805,7 @@ function f(g) { f(y => y + z); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) expect(steps.map(x => codify(x[0])).join('\n')).toMatchInlineSnapshot(` "const z = 1; function f(g) { @@ -877,7 +871,7 @@ f(y => y + z); expect(getLastStepAsString(steps)).toEqual('4;') }) -test('function declarations in blocks are protected', () => { +test('function declarations in blocks are protected', async () => { const code = ` function repeat_pattern(n, p, r) { function twice_p(r) { @@ -897,13 +891,12 @@ function plus_one(x) { repeat_pattern(5, plus_one, 0); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('5;') }) -test('const declarations in blocks subst into call expressions', () => { +test('const declarations in blocks subst into call expressions', async () => { const code = ` const z = 1; function f(g) { @@ -912,13 +905,13 @@ test('const declarations in blocks subst into call expressions', () => { } f(undefined); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('6;') }) -test('scoping test for lambda expressions nested in blocks', () => { +test('scoping test for lambda expressions nested in blocks', async () => { const code = ` { const f = x => g(); @@ -927,26 +920,26 @@ test('scoping test for lambda expressions nested in blocks', () => { f(0); } ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('1;') }) -test('scoping test for blocks nested in lambda expressions', () => { +test('scoping test for blocks nested in lambda expressions', async () => { const code = ` const f = x => { g(); }; const g = () => { x; }; const x = 1; f(0); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('undefined;') }) -test('scoping test for function expressions', () => { +test('scoping test for function expressions', async () => { const code = ` function f(x) { return g(); @@ -957,26 +950,26 @@ test('scoping test for function expressions', () => { const x = 1; f(0); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('1;') }) -test('scoping test for lambda expressions', () => { +test('scoping test for lambda expressions', async () => { const code = ` const f = x => g(); const g = () => x; const x = 1; f(0); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('1;') }) -test('scoping test for block expressions', () => { +test('scoping test for block expressions', async () => { const code = ` function f(x) { const y = x; @@ -988,13 +981,13 @@ test('scoping test for block expressions', () => { const y = 1; f(0); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('1;') }) -test('scoping test for block expressions, no renaming', () => { +test('scoping test for block expressions, no renaming', async () => { const code = ` function h(w) { function f(w) { @@ -1007,13 +1000,13 @@ test('scoping test for block expressions, no renaming', () => { } h(1); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('1;') }) -test('scoping test for block expressions, with renaming', () => { +test('scoping test for block expressions, with renaming', async () => { const code = ` function f(w) { return g(); @@ -1027,37 +1020,37 @@ test('scoping test for block expressions, with renaming', () => { } h(f); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('g();') }) -test('return in nested blocks', () => { +test('return in nested blocks', async () => { const code = ` function f(x) {{ return 1; }} f(0); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('1;') }) -test('renaming clash test for lambda function', () => { +test('renaming clash test for lambda function', async () => { const code = ` const f = w_11 => w_10 => w_11 + w_10 + g(); const g = () => w_10; const w_10 = 0; f(1)(2); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('3;') }) -test('renaming clash test for functions', () => { +test('renaming clash test for functions', async () => { const code = ` function f(w_8) { function h(w_9) { @@ -1073,13 +1066,13 @@ function g() { const w_9 = 0; f(1)(2); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('3;') }) -test('renaming clash in replacement for lambda function', () => { +test('renaming clash in replacement for lambda function', async () => { const code = ` const g = () => x_1 + x_2; const f = x_1 => x_2 => g(); @@ -1087,13 +1080,13 @@ test('renaming clash in replacement for lambda function', () => { const x_2 = 0; f(1)(1); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('0;') }) -test(`renaming clash in replacement for function expression`, () => { +test(`renaming clash in replacement for function expression`, async () => { const code = ` function f(x_1) { function h(x_2) { @@ -1108,13 +1101,13 @@ test(`renaming clash in replacement for function expression`, () => { const x_2 = 0; f(1)(1); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('0;') }) -test(`renaming clash in replacement for function declaration`, () => { +test(`renaming clash in replacement for function declaration`, async () => { const code = ` function g() { return x_1 + x_2; @@ -1129,13 +1122,13 @@ test(`renaming clash in replacement for function declaration`, () => { const x_2 = 0; f(1)(1); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('0;') }) -test(`multiple clash for function declaration`, () => { +test(`multiple clash for function declaration`, async () => { const code = ` function g() { return x_2 + x_3; @@ -1151,13 +1144,13 @@ test(`multiple clash for function declaration`, () => { const x_4 = 2; f(1)(1); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('4;') }) -test(`multiple clash for function expression`, () => { +test(`multiple clash for function expression`, async () => { const code = ` function f(x_2) { function h(x_3) { @@ -1173,13 +1166,13 @@ test(`multiple clash for function expression`, () => { const x_4 = 2; f(1)(1); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('4;') }) -test(`multiple clash for lambda function`, () => { +test(`multiple clash for lambda function`, async () => { const code = ` const f = x_2 => x_3 => x_4 + g(); const g = () => x_2 + x_3; @@ -1188,13 +1181,13 @@ test(`multiple clash for lambda function`, () => { const x_4 = 2; f(1)(1); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('4;') }) -test(`multiple clash 2 for lambda function`, () => { +test(`multiple clash 2 for lambda function`, async () => { const code = ` const f = x => x_1 => x_2 + g(); const g = () => x + x_1; @@ -1203,13 +1196,13 @@ test(`multiple clash 2 for lambda function`, () => { const x = 1; f(1)(1); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('3;') }) -test(`multiple clash 2 for function expression`, () => { +test(`multiple clash 2 for function expression`, async () => { const code = ` function f(x) { function h(x_1) { @@ -1225,13 +1218,13 @@ test(`multiple clash 2 for function expression`, () => { const x = 1; f(1)(1); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('3;') }) -test(`multiple clash 2 for function declaration`, () => { +test(`multiple clash 2 for function declaration`, async () => { const code = ` function g() { return x + x_1; @@ -1247,13 +1240,13 @@ test(`multiple clash 2 for function declaration`, () => { const x = 1; f(1)(1); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('3;') }) -test(`renaming clash with declaration in replacement for function declaration`, () => { +test(`renaming clash with declaration in replacement for function declaration`, async () => { const code = ` function g() { const x_2 = 2; @@ -1271,13 +1264,13 @@ test(`renaming clash with declaration in replacement for function declaration`, const x = 0; f(1)(1); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('3;') }) -test(`renaming clash with declaration in replacement for function expression`, () => { +test(`renaming clash with declaration in replacement for function expression`, async () => { const code = ` function f(x) { function h(x_1) { @@ -1295,13 +1288,13 @@ test(`renaming clash with declaration in replacement for function expression`, ( const x = 0; f(1)(1); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('2;') }) -test(`renaming clash with declaration in replacement for lambda function`, () => { +test(`renaming clash with declaration in replacement for lambda function`, async () => { const code = ` const f = x => x_1 => g(); const g = () => { const x_2 = 2; return x_1 + x + x_2; }; @@ -1309,13 +1302,13 @@ test(`renaming clash with declaration in replacement for lambda function`, () => const x_1 = 0; f(1)(1); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('2;') }) -test(`renaming clash with parameter of lambda function declaration in block`, () => { +test(`renaming clash with parameter of lambda function declaration in block`, async () => { const code = ` const g = () => x_1; const f = x_1 => { @@ -1326,13 +1319,13 @@ test(`renaming clash with parameter of lambda function declaration in block`, () const x_1 = 1; f(3)(2); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('4;') }) -test(`renaming clash with parameter of function declaration in block`, () => { +test(`renaming clash with parameter of function declaration in block`, async () => { const code = ` function g() { return x_1; @@ -1346,39 +1339,39 @@ test(`renaming clash with parameter of function declaration in block`, () => { const x_1 = 1; f(3)(2); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('4;') }) -test(`renaming of outer parameter in lambda function`, () => { +test(`renaming of outer parameter in lambda function`, async () => { const code = ` const g = () => w_1; const f = w_1 => w_2 => w_1 + g(); const w_1 = 0; f(1)(1); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('1;') }) -test(`correctly avoids capture by other parameter names`, () => { +test(`correctly avoids capture by other parameter names`, async () => { const code = ` function f(g, x) { return g(x); } f(y => x + 1, 2); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('x + 1;') }) -test(`removes debugger statements`, () => { +test(`removes debugger statements`, async () => { const code = ` function f(n) { debugger; @@ -1387,71 +1380,67 @@ test(`removes debugger statements`, () => { debugger; f(3); ` - const program = parse(code, mockContext())! - const steps = getEvaluationSteps(program, mockContext(), 1000) + const steps = await testEvalSteps(code) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('6;') }) describe(`redeclaration of predeclared functions work`, () => { - test('control', () => { + test('control', async () => { const code = ` length(list(1, 2, 3)); ` - const context = mockContext(Chapter.SOURCE_2) - const program = parse(code, context)! - const steps = getEvaluationSteps(program, context, 1000) + + const steps = await testEvalSteps(code, mockContext(Chapter.SOURCE_2)) expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('3;') }) - test('test', () => { + test('test', async () => { const code = ` function length(xs) { return 0; } length(list(1, 2, 3)); ` - const context = mockContext(Chapter.SOURCE_2) - const program = parse(code, context)! - const steps = getEvaluationSteps(program, context, 1000) + const steps = await testEvalSteps(code, mockContext(Chapter.SOURCE_2)) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('0;') }) }) describe(`#1109: Empty function bodies don't break execution`, () => { - test('Function declaration', () => { + test('Function declaration', async () => { const code = ` function a() {} "other statement"; a(); "Gets returned by normal run"; ` - const context = mockContext(Chapter.SOURCE_2) - const program = parse(code, context)! - const steps = getEvaluationSteps(program, context, 1000) + const steps = await testEvalSteps(code, mockContext(Chapter.SOURCE_2)) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('"Gets returned by normal run";') }) - test('Constant declaration of lambda', () => { + test('Constant declaration of lambda', async () => { const code = ` const a = () => {}; "other statement"; a(); "Gets returned by normal run"; ` - const context = mockContext(Chapter.SOURCE_2) - const program = parse(code, context)! - const steps = getEvaluationSteps(program, context, 1000) + const steps = await testEvalSteps(code, mockContext(Chapter.SOURCE_2)) + expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('"Gets returned by normal run";') }) }) describe(`#1342: Test the fix of #1341: Stepper limit off by one`, () => { - test('Program steps equal to Stepper limit', () => { + test('Program steps equal to Stepper limit', async () => { const code = ` function factorial(n) { return n === 1 @@ -1460,22 +1449,20 @@ describe(`#1342: Test the fix of #1341: Stepper limit off by one`, () => { } factorial(100); ` - const context = mockContext(Chapter.SOURCE_2) - const program = parse(code, context)! - const steps = getEvaluationSteps(program, context, 1000) + const steps = await testEvalSteps(code, mockContext(Chapter.SOURCE_2)) expect(steps.map(x => codify(x[0])).join('\n')).toMatchSnapshot() expect(getLastStepAsString(steps)).toEqual('9.33262154439441e+157;') }) }) // describe(`#1223: Stepper: Import statements cause errors`, () => { -// test('import a module and invoke its functions', () => { +// test('import a module and invoke its functions', async () => { // const code = ` // import {circle, show, red, stack} from "rune"; // show(stack(red(circle), circle)); // ` -// const program = parse(code, mockContext())! -// const steps = getEvaluationSteps(program, mockContext(), 1000) +// const steps = await testEvalSteps(code) +// // expect(steps.map(x => codify(x[0])).join('\n')).toMatchInlineSnapshot(` // "show(stack(red(circle), circle)); @@ -1496,13 +1483,13 @@ describe(`#1342: Test the fix of #1341: Stepper limit off by one`, () => { // `) // }) -// test('return function from module function and invoke built-in with lambda', () => { +// test('return function from module function and invoke built-in with lambda', async () => { // const code = ` // import {draw_points, make_point} from "curve"; // draw_points(100)(t => make_point(t, t)); // ` -// const program = parse(code, mockContext())! -// const steps = getEvaluationSteps(program, mockContext(), 1000) +// const steps = await testEvalSteps(code) +// // expect(steps.map(x => codify(x[0])).join('\n')).toMatchInlineSnapshot(` // "draw_points(100)(t => make_point(t, t)); @@ -1519,7 +1506,7 @@ describe(`#1342: Test the fix of #1341: Stepper limit off by one`, () => { // `) // }) -// test('invoke built-in with function expression', () => { +// test('invoke built-in with function expression', async () => { // const code = ` // import {draw_3D_points, make_3D_point} from "curve"; // function f(t) { @@ -1527,8 +1514,8 @@ describe(`#1342: Test the fix of #1341: Stepper limit off by one`, () => { // } // draw_3D_points(100)(f); // ` -// const program = parse(code, mockContext())! -// const steps = getEvaluationSteps(program, mockContext(), 1000) +// const steps = await testEvalSteps(code) +// // expect(steps.map(x => codify(x[0])).join('\n')).toMatchInlineSnapshot(` // "function f(t) { // return make_3D_point(t, t, t); @@ -1555,7 +1542,7 @@ describe(`#1342: Test the fix of #1341: Stepper limit off by one`, () => { // `) // }) -// test('recursive function and invoking with module function and object', () => { +// test('recursive function and invoking with module function and object', async () => { // const code = ` // import { stack, heart, show, make_cross } from "rune"; // function repeat(n, f, i) { @@ -1565,8 +1552,8 @@ describe(`#1342: Test the fix of #1341: Stepper limit off by one`, () => { // } // show(repeat(1, make_cross, heart)); // ` -// const program = parse(code, mockContext())! -// const steps = getEvaluationSteps(program, mockContext(), 1000) +// const steps = await testEvalSteps(code) +// // expect(steps.map(x => codify(x[0])).join('\n')).toMatchInlineSnapshot(` // "function repeat(n, f, i) { // return n === 0 ? i : repeat(n - 1, f, f(i)); @@ -1621,13 +1608,13 @@ describe(`#1342: Test the fix of #1341: Stepper limit off by one`, () => { // `) // }) -// test('display unnamed object', () => { +// test('display unnamed object', async () => { // const code = ` // import {play, sine_sound} from "sound"; // play(sine_sound(440, 5)); // ` -// const program = parse(code, mockContext())! -// const steps = getEvaluationSteps(program, mockContext(), 1000) +// const steps = await testEvalSteps(code) +// // expect(steps.map(x => codify(x[0])).join('\n')).toMatchInlineSnapshot(` // "play(sine_sound(440, 5)); diff --git a/src/stepper/stepper.ts b/src/stepper/stepper.ts index 96b6806de..c3cae4178 100644 --- a/src/stepper/stepper.ts +++ b/src/stepper/stepper.ts @@ -1,10 +1,12 @@ import { generate } from 'astring' -import * as es from 'estree' +import type * as es from 'estree' +import { partition } from 'lodash' +import { type IOptions } from '..' import * as errors from '../errors/errors' import { UndefinedImportError } from '../modules/errors' -import { loadModuleBundle, loadModuleTabs } from '../modules/moduleLoader' -import { ModuleFunctions } from '../modules/moduleTypes' +import { loadModuleBundleAsync, loadModuleTabsAsync } from '../modules/moduleLoaderAsync' +import type { ImportTransformOptions } from '../modules/moduleTypes' import { parse } from '../parser/parser' import { BlockExpression, @@ -14,6 +16,8 @@ import { FunctionDeclarationExpression, substituterNodes } from '../types' +import assert from '../utils/assert' +import { isImportDeclaration } from '../utils/ast/typeGuards' import * as ast from '../utils/astCreator' import { dummyBlockExpression, @@ -3155,63 +3159,80 @@ function removeDebuggerStatements(program: es.Program): es.Program { return program } -function evaluateImports( +async function evaluateImports( program: es.Program, context: Context, - loadTabs: boolean, - checkImports: boolean + { loadTabs, checkImports, wrapSourceModules }: ImportTransformOptions ) { - const importNodes = program.body.filter( - ({ type }) => type === 'ImportDeclaration' - ) as es.ImportDeclaration[] - program.body = program.body.filter(({ type }) => !(type === 'ImportDeclaration')) - const moduleFunctions: Record = {} + const [importNodes, otherNodes] = partition(program.body, isImportDeclaration) - try { - for (const node of importNodes) { - const moduleName = node.source.value - if (typeof moduleName !== 'string') { - throw new Error(`ImportDeclarations should have string sources, got ${moduleName}`) - } + const importNodeMap = importNodes.reduce((res, node) => { + const moduleName = node.source.value + assert( + typeof moduleName === 'string', + `ImportDeclarations should have string sources, got ${moduleName}` + ) - if (!(moduleName in moduleFunctions)) { - context.moduleContexts[moduleName] = { - state: null, - tabs: loadTabs ? loadModuleTabs(moduleName, node) : null - } - moduleFunctions[moduleName] = loadModuleBundle(moduleName, context, node) - } + if (!(moduleName in res)) { + res[moduleName] = [] + } + + res[moduleName].push(node) + return res + }, {} as Record) - const functions = moduleFunctions[moduleName] - const environment = currentEnvironment(context) - for (const spec of node.specifiers) { - if (spec.type !== 'ImportSpecifier') { - throw new Error(`Only ImportSpecifiers are supported, got: ${spec.type}`) + try { + await Promise.all( + Object.entries(importNodeMap).map(async ([moduleName, nodes]) => { + if (!(moduleName in context.moduleContexts)) { + context.moduleContexts[moduleName] = { + state: null, + tabs: loadTabs ? await loadModuleTabsAsync(moduleName) : null + } + } else if (!context.moduleContexts[moduleName].tabs && loadTabs) { + context.moduleContexts[moduleName].tabs = await loadModuleTabsAsync(moduleName) } - if (checkImports && !(spec.imported.name in functions)) { - throw new UndefinedImportError(spec.imported.name, moduleName, node) + const functions = await loadModuleBundleAsync( + moduleName, + context, + wrapSourceModules, + nodes[0] + ) + const environment = currentEnvironment(context) + for (const node of nodes) { + for (const spec of node.specifiers) { + assert( + spec.type === 'ImportSpecifier', + `Only ImportSpecifiers are supported, got: ${spec.type}` + ) + + if (checkImports && !(spec.imported.name in functions)) { + throw new UndefinedImportError(spec.imported.name, moduleName, spec) + } + declareIdentifier(context, spec.local.name, node, environment) + defineVariable(context, spec.local.name, functions[spec.imported.name], true, node) + } } - declareIdentifier(context, spec.local.name, node, environment) - defineVariable(context, spec.local.name, functions[spec.imported.name], true, node) - } - } + }) + ) } catch (error) { // console.log(error) handleRuntimeError(context, error) } + program.body = otherNodes } // the context here is for builtins -export function getEvaluationSteps( +export async function getEvaluationSteps( program: es.Program, context: Context, - stepLimit: number | undefined -): [es.Program, string[][], string][] { + { importOptions, stepLimit }: Pick +): Promise<[es.Program, string[][], string][]> { const steps: [es.Program, string[][], string][] = [] try { const limit = stepLimit === undefined ? 1000 : stepLimit % 2 === 0 ? stepLimit : stepLimit + 1 - evaluateImports(program, context, true, true) + await evaluateImports(program, context, importOptions) // starts with substituting predefined constants let start = substPredefinedConstants(program) // and predefined fns From 82f6e604ecc56507c13e4fdfaf28b4b05292b69c Mon Sep 17 00:00:00 2001 From: leeyi45 Date: Mon, 21 Aug 2023 02:41:00 +0800 Subject: [PATCH 04/24] Add assert and misc utils --- src/localImports/directedGraph.ts | 25 ++++++----- src/localImports/preprocessor.ts | 31 +++++++------- src/utils/__tests__/arrayMap.ts | 27 ++++++++++++ src/utils/arrayMap.ts | 71 +++++++++++++++++++++++++++++++ src/utils/assert.ts | 27 ++++++++++++ src/utils/misc.ts | 18 ++++++++ 6 files changed, 172 insertions(+), 27 deletions(-) create mode 100644 src/utils/__tests__/arrayMap.ts create mode 100644 src/utils/arrayMap.ts create mode 100644 src/utils/assert.ts create mode 100644 src/utils/misc.ts diff --git a/src/localImports/directedGraph.ts b/src/localImports/directedGraph.ts index c7ff436ae..a6997a51d 100644 --- a/src/localImports/directedGraph.ts +++ b/src/localImports/directedGraph.ts @@ -1,3 +1,5 @@ +import assert from '../utils/assert' + /** * The result of attempting to find a topological ordering * of nodes on a DirectedGraph. @@ -113,9 +115,10 @@ export class DirectedGraph { // all nodes have an in-degree of 0 after running Kahn's algorithm. // This in turn implies that Kahn's algorithm was able to find a // valid topological ordering & that the graph contains no cycles. - if (startingNodeInCycle === null) { - throw new Error('There are no cycles in this graph. This should never happen.') - } + assert( + startingNodeInCycle !== null, + 'There are no cycles in this graph. This should never happen.' + ) const cycle = [startingNodeInCycle] // Then, we keep picking arbitrary nodes with non-zero in-degrees until @@ -132,9 +135,10 @@ export class DirectedGraph { // An in-degree of 0 implies that the node is not part of a cycle, // which is a contradiction since the current node was picked because // it is part of a cycle. - if (neighbours.size === 0) { - throw new Error(`Node '${currentNode}' has no incoming edges. This should never happen.`) - } + assert( + neighbours.size > 0, + `Node '${currentNode}' has no incoming edges. This should never happen.` + ) let nextNodeInCycle: string | null = null for (const neighbour of neighbours) { @@ -146,11 +150,10 @@ export class DirectedGraph { // By the invariant stated above, if the current node is part of a cycle, // then one of its neighbours must also be part of the same cycle. This // is because a cycle contains at least 2 nodes. - if (nextNodeInCycle === null) { - throw new Error( - `None of the neighbours of node '${currentNode}' are part of the same cycle. This should never happen.` - ) - } + assert( + nextNodeInCycle !== null, + `None of the neighbours of node '${currentNode}' are part of the same cycle. This should never happen.` + ) // If the next node we pick is already part of the cycle, // we drop all elements before the first instance of the diff --git a/src/localImports/preprocessor.ts b/src/localImports/preprocessor.ts index d4f6464c1..820cd858e 100644 --- a/src/localImports/preprocessor.ts +++ b/src/localImports/preprocessor.ts @@ -1,10 +1,12 @@ import es from 'estree' -import * as path from 'path' +import { posix as pathlib } from 'path' import { CannotFindModuleError, CircularImportError } from '../errors/localImportErrors' import { parse } from '../parser/parser' import { AcornOptions } from '../parser/types' import { Context } from '../types' +import assert from '../utils/assert' +import { isImportDeclaration, isModuleDeclaration } from '../utils/ast/typeGuards' import { isIdentifier } from '../utils/rttc' import { createInvokedFunctionResultVariableDeclaration } from './constructors/contextSpecificConstructors' import { DirectedGraph } from './directedGraph' @@ -23,7 +25,6 @@ import { getInvokedFunctionResultVariableNameToImportSpecifiersMap, transformProgramToFunctionDeclaration } from './transformers/transformProgramToFunctionDeclaration' -import { isImportDeclaration, isModuleDeclaration } from './typeGuards' /** * Returns all absolute local module paths which should be imported. @@ -39,11 +40,11 @@ export const getImportedLocalModulePaths = ( program: es.Program, currentFilePath: string ): Set => { - if (!path.isAbsolute(currentFilePath)) { + if (!pathlib.isAbsolute(currentFilePath)) { throw new Error(`Current file path '${currentFilePath}' is not absolute.`) } - const baseFilePath = path.resolve(currentFilePath, '..') + const baseFilePath = pathlib.resolve(currentFilePath, '..') const importedLocalModuleNames: Set = new Set() const importDeclarations = program.body.filter(isImportDeclaration) importDeclarations.forEach((importDeclaration: es.ImportDeclaration): void => { @@ -52,7 +53,7 @@ export const getImportedLocalModulePaths = ( throw new Error('Module names must be strings.') } if (!isSourceModule(modulePath)) { - const absoluteModulePath = path.resolve(baseFilePath, modulePath) + const absoluteModulePath = pathlib.resolve(baseFilePath, modulePath) importedLocalModuleNames.add(absoluteModulePath) } }) @@ -192,7 +193,7 @@ const preprocessFileImports = ( // We want to operate on the entrypoint program to get the eventual // preprocessed program. const entrypointProgram = programs[entrypointFilePath] - const entrypointDirPath = path.resolve(entrypointFilePath, '..') + const entrypointDirPath = pathlib.resolve(entrypointFilePath, '..') // Create variables to hold the imported statements. const entrypointProgramModuleDeclarations = entrypointProgram.body.filter(isModuleDeclaration) @@ -218,11 +219,10 @@ const preprocessFileImports = ( const functionDeclaration = transformProgramToFunctionDeclaration(program, filePath) const functionName = functionDeclaration.id?.name - if (functionName === undefined) { - throw new Error( - 'A transformed function declaration is missing its name. This should never happen.' - ) - } + assert( + functionName !== undefined, + 'A transformed function declaration is missing its name. This should never happen.' + ) functionDeclarations[functionName] = functionDeclaration } @@ -242,11 +242,10 @@ const preprocessFileImports = ( const functionDeclaration = functionDeclarations[functionName] const functionParams = functionDeclaration.params.filter(isIdentifier) - if (functionParams.length !== functionDeclaration.params.length) { - throw new Error( - 'Function declaration contains non-Identifier AST nodes as params. This should never happen.' - ) - } + assert( + functionParams.length === functionDeclaration.params.length, + 'Function declaration contains non-Identifier AST nodes as params. This should never happen.' + ) const invokedFunctionResultVariableDeclaration = createInvokedFunctionResultVariableDeclaration( functionName, diff --git a/src/utils/__tests__/arrayMap.ts b/src/utils/__tests__/arrayMap.ts new file mode 100644 index 000000000..b9a423f18 --- /dev/null +++ b/src/utils/__tests__/arrayMap.ts @@ -0,0 +1,27 @@ +import { arrayMapFrom } from '../arrayMap' + +test('arrayMapFrom', () => { + const arrMap = arrayMapFrom([ + [1, [1, 2, 3]], + [2, [2, 4, 6]] + ]) + + expect(arrMap.get(1)).toEqual([1, 2, 3]) + expect(arrMap.get(2)).toEqual([2, 4, 6]) +}) + +test('mapAsync', async () => { + const arrMap = arrayMapFrom([ + [1, [1, 2, 3]], + [2, [2, 4, 6]] + ]) + + const mapper = jest.fn((k: number, entry: number[]) => + Promise.resolve([k, entry.map(each => each * 2)] as [number, number[]]) + ) + const newMap = await arrMap.mapAsync(mapper) + + expect(newMap.get(1)).toEqual([2, 4, 6]) + expect(newMap.get(2)).toEqual([4, 8, 12]) + expect(mapper).toHaveBeenCalledTimes(2) +}) diff --git a/src/utils/arrayMap.ts b/src/utils/arrayMap.ts new file mode 100644 index 000000000..e0d934c74 --- /dev/null +++ b/src/utils/arrayMap.ts @@ -0,0 +1,71 @@ +/** + * Convenience class for maps that store an array of values + */ +export default class ArrayMap { + constructor(private readonly map: Map = new Map()) {} + + public get(key: K) { + return this.map.get(key) + } + + public add(key: K, item: V) { + if (!this.map.has(key)) { + this.map.set(key, []) + } + this.map.get(key)!.push(item) + } + + public entries() { + return Array.from(this.map.entries()) + } + + public keys() { + return new Set(this.map.keys()) + } + + /** + * Similar to `mapAsync`, but for an async mapping function that does not return any value + */ + public async forEachAsync Promise>(forEach: F): Promise { + await Promise.all(this.entries().map(([key, value]) => forEach(key, value))) + } + + /** + * Using a mapping function that returns a promise, transform an array map + * to another array map with different keys and values. All calls to the mapping function + * execute asynchronously + */ + public async mapAsync Promise<[any, any[]]>>(mapper: F) { + const pairs = await Promise.all(this.entries().map(([key, value]) => mapper(key, value))) + + type U = Awaited> + const tempMap = new Map(pairs) + return new ArrayMap(tempMap) + } + + public [Symbol.toStringTag]() { + return this.entries().map(([key, value]) => `${key}: ${value}`) + } +} + +/** + * Create an ArrayMap from an iterable of key value pairs + */ +export function arrayMapFrom>( + pairs: Iterable<[K, V]> +): ArrayMap +export function arrayMapFrom(pairs: Iterable<[K, V]>): ArrayMap +export function arrayMapFrom(pairs: Iterable<[K, V | V[]]>) { + const res = new ArrayMap() + for (const [k, v] of pairs) { + if (Array.isArray(v)) { + for (const each of v) { + res.add(k, each) + } + } else { + res.add(k, v) + } + } + + return res +} diff --git a/src/utils/assert.ts b/src/utils/assert.ts new file mode 100644 index 000000000..d405343e2 --- /dev/null +++ b/src/utils/assert.ts @@ -0,0 +1,27 @@ +/* + * Why not use the nodejs builtin assert? It needs polyfills to work in the browser. + * With this we have a lightweight assert that doesn't need any further packages. + * Plus, we can customize our own assert messages and handling + */ + +import { RuntimeSourceError } from '../errors/runtimeSourceError' + +export class AssertionError extends RuntimeSourceError { + constructor(public readonly message: string) { + super() + } + + public explain(): string { + return this.message + } + + public elaborate(): string { + return 'Please contact the administrators to let them know that this error has occurred' + } +} + +export default function assert(condition: boolean, message: string): asserts condition { + if (!condition) { + throw new AssertionError(message) + } +} diff --git a/src/utils/misc.ts b/src/utils/misc.ts new file mode 100644 index 000000000..3cb5f79fe --- /dev/null +++ b/src/utils/misc.ts @@ -0,0 +1,18 @@ +import { RuntimeSourceError } from '../errors/runtimeSourceError' + +export class TimeoutError extends RuntimeSourceError {} + +export const timeoutPromise = (promise: Promise, timeout: number) => + new Promise((resolve, reject) => { + const timeoutid = setTimeout(() => reject(new TimeoutError()), timeout) + + promise + .then(res => { + clearTimeout(timeoutid) + resolve(res) + }) + .catch(e => { + clearTimeout(timeoutid) + reject(e) + }) + }) From 78117c50dcf0b0d0876e78bfe482bc1c7727a290 Mon Sep 17 00:00:00 2001 From: leeyi45 Date: Mon, 21 Aug 2023 02:41:35 +0800 Subject: [PATCH 05/24] Update transpiler to be async --- src/transpiler/__tests__/modules.ts | 106 ++++------ src/transpiler/__tests__/transpiled-code.ts | 8 +- src/transpiler/transpiler.ts | 211 ++++++++++++-------- 3 files changed, 171 insertions(+), 154 deletions(-) diff --git a/src/transpiler/__tests__/modules.ts b/src/transpiler/__tests__/modules.ts index c7b8c1ae2..8cea33aba 100644 --- a/src/transpiler/__tests__/modules.ts +++ b/src/transpiler/__tests__/modules.ts @@ -1,51 +1,32 @@ import type { Identifier, Literal, MemberExpression, VariableDeclaration } from 'estree' -import type { FunctionLike, MockedFunction } from 'jest-mock' +// import type { FunctionLike, MockedFunction } from 'jest-mock' +import { runInContext } from '../..' import { mockContext } from '../../mocks/context' import { UndefinedImportError } from '../../modules/errors' -import { memoizedGetModuleFile } from '../../modules/moduleLoader' import { parse } from '../../parser/parser' -import { Chapter } from '../../types' +import { Chapter, Value } from '../../types' import { stripIndent } from '../../utils/formatters' import { transformImportDeclarations, transpile } from '../transpiler' -jest.mock('../../modules/moduleLoader', () => ({ - ...jest.requireActual('../../modules/moduleLoader'), - memoizedGetModuleFile: jest.fn(), - memoizedGetModuleManifest: jest.fn().mockReturnValue({ - one_module: { - tabs: [] - }, - another_module: { - tabs: [] - } - }), - memoizedloadModuleDocs: jest.fn().mockReturnValue({ - foo: 'foo', - bar: 'bar' - }) -})) - -const asMock = (func: T) => func as MockedFunction -const mockedModuleFile = asMock(memoizedGetModuleFile) +// const asMock = (func: T) => func as MockedFunction +// const mockedModuleFile = asMock(memoizedGetModuleFile) -test('Transform import declarations into variable declarations', () => { - mockedModuleFile.mockImplementation((name, type) => { - if (type === 'json') { - return name === 'one_module' ? "{ foo: 'foo' }" : "{ bar: 'bar' }" - } else { - return 'undefined' - } - }) +jest.mock('../../modules/moduleLoaderAsync') +test('Transform import declarations into variable declarations', async () => { const code = stripIndent` - import { foo } from "test/one_module"; - import { bar } from "test/another_module"; + import { foo } from "one_module"; + import { bar } from "another_module"; foo(bar); ` const context = mockContext(Chapter.SOURCE_4) const program = parse(code, context)! - const [, importNodes] = transformImportDeclarations(program, new Set(), false) + const [, importNodes] = await transformImportDeclarations(program, new Set(), { + wrapSourceModules: true, + loadTabs: false, + checkImports: true + }) expect(importNodes[0].type).toBe('VariableDeclaration') expect((importNodes[0].declarations[0].id as Identifier).name).toEqual('foo') @@ -54,28 +35,24 @@ test('Transform import declarations into variable declarations', () => { expect((importNodes[1].declarations[0].id as Identifier).name).toEqual('bar') }) -test('Transpiler accounts for user variable names when transforming import statements', () => { - mockedModuleFile.mockImplementation((name, type) => { - if (type === 'json') { - return name === 'one_module' ? "{ foo: 'foo' }" : "{ bar: 'bar' }" - } else { - return 'undefined' - } - }) - +test('Transpiler accounts for user variable names when transforming import statements', async () => { const code = stripIndent` - import { foo } from "test/one_module"; - import { bar as __MODULE__2 } from "test/another_module"; + import { foo } from "one_module"; + import { bar as __MODULE__2 } from "another_module"; const __MODULE__ = 'test0'; const __MODULE__0 = 'test1'; foo(bar); ` const context = mockContext(4) const program = parse(code, context)! - const [, importNodes, [varDecl0, varDecl1]] = transformImportDeclarations( + const [, importNodes, [varDecl0, varDecl1]] = await transformImportDeclarations( program, new Set(['__MODULE__', '__MODULE__0']), - false + { + loadTabs: false, + wrapSourceModules: false, + checkImports: true + } ) expect(importNodes[0].type).toBe('VariableDeclaration') @@ -95,40 +72,31 @@ test('Transpiler accounts for user variable names when transforming import state ).toEqual('__MODULE__3') }) -test('checkForUndefinedVariables accounts for import statements', () => { - mockedModuleFile.mockImplementation((name, type) => { - if (type === 'json') { - return "{ hello: 'hello' }" - } else { - return 'undefined' - } - }) - +test('Module loading functionality', async () => { const code = stripIndent` - import { foo } from "one_module"; - foo; + import { foo } from 'one_module'; + foo(); ` const context = mockContext(Chapter.SOURCE_4) - const program = parse(code, context)! - transpile(program, context, false) -}) + const result = await runInContext(code, context) + expect(result.status).toEqual('finished') -test('importing undefined variables should throw errors', () => { - mockedModuleFile.mockImplementation((name, type) => { - if (type === 'json') { - return '{}' - } else { - return 'undefined' - } - }) + expect((result as Value).value).toEqual('foo') +}) +test('importing undefined variables should throw errors', async () => { const code = stripIndent` import { hello } from 'one_module'; ` const context = mockContext(Chapter.SOURCE_4) const program = parse(code, context)! try { - transpile(program, context, false) + await transpile( + program, + context, + { checkImports: true, loadTabs: false, wrapSourceModules: false }, + false + ) } catch (error) { expect(error).toBeInstanceOf(UndefinedImportError) expect((error as UndefinedImportError).symbol).toEqual('hello') diff --git a/src/transpiler/__tests__/transpiled-code.ts b/src/transpiler/__tests__/transpiled-code.ts index 104ff6cbd..54d5ba3fb 100644 --- a/src/transpiler/__tests__/transpiled-code.ts +++ b/src/transpiler/__tests__/transpiled-code.ts @@ -9,10 +9,10 @@ import { transpile } from '../transpiler' * code being tested from being transformed into eval. * Check for variables being stored back by looking at all the tests. */ -test('builtins do get prepended', () => { +test('builtins do get prepended', async () => { const code = '"ensure_builtins";' const context = mockContext(Chapter.SOURCE_4) - const transpiled = transpile(parse(code, context)!, context).transpiled + const { transpiled } = await transpile(parse(code, context)!, context) // replace native[] as they may be inconsistent const replacedNative = transpiled.replace(/native\[\d+]/g, 'native') // replace the line hiding globals as they may differ between environments @@ -20,7 +20,7 @@ test('builtins do get prepended', () => { expect({ code, transpiled: replacedGlobalsLine }).toMatchSnapshot() }) -test('Ensure no name clashes', () => { +test('Ensure no name clashes', async () => { const code = stripIndent` const boolOrErr = 1; boolOrErr[123] = 1; @@ -32,7 +32,7 @@ test('Ensure no name clashes', () => { const native = 123; ` const context = mockContext(Chapter.SOURCE_4) - const transpiled = transpile(parse(code, context)!, context).transpiled + const { transpiled } = await transpile(parse(code, context)!, context) const replacedNative = transpiled.replace(/native0\[\d+]/g, 'native') const replacedGlobalsLine = replacedNative.replace(/\n\(\(.*\)/, '\n(( )') expect(replacedGlobalsLine).toMatchSnapshot() diff --git a/src/transpiler/transpiler.ts b/src/transpiler/transpiler.ts index b33bb6b66..9124410bc 100644 --- a/src/transpiler/transpiler.ts +++ b/src/transpiler/transpiler.ts @@ -6,10 +6,24 @@ import { RawSourceMap, SourceMapGenerator } from 'source-map' import { NATIVE_STORAGE_ID, REQUIRE_PROVIDER_ID, UNKNOWN_LOCATION } from '../constants' import { UndefinedVariable } from '../errors/errors' -import { UndefinedImportError } from '../modules/errors' -import { memoizedGetModuleFile, memoizedloadModuleDocs } from '../modules/moduleLoader' -import { ModuleDocumentation } from '../modules/moduleTypes' -import { AllowedDeclarations, Chapter, Context, NativeStorage, Variant } from '../types' +import { ModuleNotFoundError } from '../errors/moduleErrors' +import { ModuleInternalError, UndefinedImportError } from '../modules/errors' +import { + memoizedGetModuleBundleAsync, + memoizedGetModuleDocsAsync, + memoizedGetModuleManifestAsync +} from '../modules/moduleLoaderAsync' +import { ImportTransformOptions } from '../modules/moduleTypes' +import { + AllowedDeclarations, + Chapter, + Context, + NativeStorage, + RecursivePartial, + Variant +} from '../types' +import assert from '../utils/assert' +import { isImportDeclaration } from '../utils/ast/typeGuards' import * as create from '../utils/astCreator' import { getIdentifiersInNativeStorage, @@ -40,88 +54,101 @@ const globalIdNames = [ export type NativeIds = Record -export function transformImportDeclarations( +export async function transformImportDeclarations( program: es.Program, usedIdentifiers: Set, - checkImports: boolean, + { wrapSourceModules, checkImports }: ImportTransformOptions, nativeId?: es.Identifier, useThis: boolean = false -): [string, es.VariableDeclaration[], es.Program['body']] { - const [importNodes, otherNodes] = partition( - program.body, - node => node.type === 'ImportDeclaration' - ) +): Promise<[string, es.VariableDeclaration[], es.Program['body']]> { + const [importNodes, otherNodes] = partition(program.body, isImportDeclaration) if (importNodes.length === 0) return ['', [], otherNodes] + const importNodeMap = importNodes.reduce((res, node) => { + const moduleName = node.source.value + assert( + typeof moduleName === 'string', + `Expected ImportDeclaration to have a source of type string, got ${moduleName}` + ) - const moduleInfos = importNodes.reduce( - (res, node: es.ImportDeclaration) => { - const moduleName = node.source.value - if (typeof moduleName !== 'string') { - throw new Error( - `Expected ImportDeclaration to have a source of type string, got ${moduleName}` - ) - } + if (!(moduleName in res)) { + res[moduleName] = [] + } - if (!(moduleName in res)) { - res[moduleName] = { - text: memoizedGetModuleFile(moduleName, 'bundle'), - nodes: [], - docs: checkImports ? memoizedloadModuleDocs(moduleName, node) : null - } - } + res[moduleName].push(node) - res[moduleName].nodes.push(node) - node.specifiers.forEach(spec => usedIdentifiers.add(spec.local.name)) - return res - }, - {} as Record< - string, - { - nodes: es.ImportDeclaration[] - text: string - docs: ModuleDocumentation | null + node.specifiers.forEach(({ local: { name } }) => usedIdentifiers.add(name)) + return res + }, {} as Record) + + const manifest = await memoizedGetModuleManifestAsync() + + const loadedModules = await Promise.all( + Object.entries(importNodeMap).map(async ([moduleName, nodes]) => { + if (!(moduleName in manifest)) { + throw new ModuleNotFoundError(moduleName, nodes[0]) } - > - ) - const prefix: string[] = [] - const declNodes = Object.entries(moduleInfos).flatMap(([moduleName, { nodes, text, docs }]) => { - const namespaced = getUniqueId(usedIdentifiers, '__MODULE__') - prefix.push(`// ${moduleName} module`) + const [text, docs] = await Promise.all([ + memoizedGetModuleBundleAsync(moduleName), + memoizedGetModuleDocsAsync(moduleName) + ]) - const modifiedText = nativeId - ? `${NATIVE_STORAGE_ID}.operators.get("wrapSourceModule")("${moduleName}", ${text}, ${REQUIRE_PROVIDER_ID})` - : `(${text})(${REQUIRE_PROVIDER_ID})` - prefix.push(`const ${namespaced} = ${modifiedText}\n`) + const namespaced = getUniqueId(usedIdentifiers, '__MODULE__') - return nodes.flatMap(node => - node.specifiers.map(specifier => { - if (specifier.type !== 'ImportSpecifier') { - throw new Error(`Expected import specifier, found: ${specifier.type}`) - } + if (checkImports && !docs) { + throw new ModuleInternalError( + moduleName, + new Error('checkImports was true, but failed to load docs'), + nodes[0] + ) + } + + const declNodes = nodes.flatMap(({ specifiers }) => + specifiers.map(spec => { + assert(spec.type === 'ImportSpecifier', `Expected ImportSpecifier, got ${spec.type}`) - if (checkImports) { - if (!docs) { - console.warn(`Failed to load docs for ${moduleName}, skipping typechecking`) - } else if (!(specifier.imported.name in docs)) { - throw new UndefinedImportError(specifier.imported.name, moduleName, node) + if (!(spec.imported.name in docs!)) { + throw new UndefinedImportError(spec.imported.name, moduleName, spec) } - } - // Convert each import specifier to its corresponding local variable declaration - return create.constantDeclaration( - specifier.local.name, - create.memberExpression( - create.identifier(`${useThis ? 'this.' : ''}${namespaced}`), - specifier.imported.name + // Convert each import specifier to its corresponding local variable declaration + return create.constantDeclaration( + spec.local.name, + create.memberExpression( + create.identifier(`${useThis ? 'this.' : ''}${namespaced}`), + spec.imported.name + ) ) - ) - }) - ) - }) + }) + ) + + return [moduleName, { text, nodes: declNodes, namespaced }] as [ + string, + { + text: string + nodes: es.VariableDeclaration[] + namespaced: string + } + ] + }) + ) - return [prefix.join('\n'), declNodes, otherNodes] + const [prefixes, declNodes] = loadedModules.reduce( + ([prefix, decls], [moduleName, { text, nodes, namespaced }]) => { + const modifiedText = wrapSourceModules + ? `${NATIVE_STORAGE_ID}.operators.get("wrapSourceModule")("${moduleName}", ${text}, ${REQUIRE_PROVIDER_ID})` + : `(${text})(${REQUIRE_PROVIDER_ID})` + + return [ + [...prefix, `const ${namespaced} = ${modifiedText}\n`], + [...decls, ...nodes] + ] + }, + [[], []] as [string[], es.VariableDeclaration[]] + ) + + return [prefixes.join('\n'), declNodes, otherNodes] } export function getGloballyDeclaredIdentifiers(program: es.Program): string[] { @@ -601,11 +628,12 @@ function getDeclarationsToAccessTranspilerInternals( export type TranspiledResult = { transpiled: string; sourceMapJson?: RawSourceMap } -function transpileToSource( +async function transpileToSource( program: es.Program, context: Context, - skipUndefined: boolean -): TranspiledResult { + skipUndefined: boolean, + importOptions: ImportTransformOptions +): Promise { const usedIdentifiers = new Set([ ...getIdentifiersInProgram(program), ...getIdentifiersInNativeStorage(context.nativeStorage) @@ -628,10 +656,10 @@ function transpileToSource( wrapArrowFunctionsToAllowNormalCallsAndNiceToString(program, functionsToStringMap, globalIds) addInfiniteLoopProtection(program, globalIds, usedIdentifiers) - const [modulePrefix, importNodes, otherNodes] = transformImportDeclarations( + const [modulePrefix, importNodes, otherNodes] = await transformImportDeclarations( program, usedIdentifiers, - true, + importOptions, globalIds.native ) program.body = (importNodes as es.Program['body']).concat(otherNodes) @@ -658,11 +686,12 @@ function transpileToSource( return { transpiled, sourceMapJson } } -function transpileToFullJS( +async function transpileToFullJS( program: es.Program, context: Context, + importOptions: ImportTransformOptions, skipUndefined: boolean -): TranspiledResult { +): Promise { const usedIdentifiers = new Set([ ...getIdentifiersInProgram(program), ...getIdentifiersInNativeStorage(context.nativeStorage) @@ -671,10 +700,10 @@ function transpileToFullJS( const globalIds = getNativeIds(program, usedIdentifiers) checkForUndefinedVariables(program, context.nativeStorage, globalIds, skipUndefined) - const [modulePrefix, importNodes, otherNodes] = transformImportDeclarations( + const [modulePrefix, importNodes, otherNodes] = await transformImportDeclarations( program, usedIdentifiers, - false, + importOptions, globalIds.native ) @@ -698,13 +727,33 @@ function transpileToFullJS( export function transpile( program: es.Program, context: Context, + importOptions: RecursivePartial = {}, skipUndefined = false -): TranspiledResult { +): Promise { if (context.chapter === Chapter.FULL_JS || context.chapter === Chapter.PYTHON_1) { - return transpileToFullJS(program, context, true) + const fullImportOptions = { + checkImports: false, + loadTabs: true, + wrapSourceModules: false, + ...importOptions + } + + return transpileToFullJS(program, context, fullImportOptions, true) } else if (context.variant == Variant.NATIVE) { - return transpileToFullJS(program, context, false) + const fullImportOptions = { + checkImports: true, + loadTabs: true, + wrapSourceModules: true, + ...importOptions + } + return transpileToFullJS(program, context, fullImportOptions, false) } else { - return transpileToSource(program, context, skipUndefined) + const fullImportOptions = { + checkImports: true, + loadTabs: true, + wrapSourceModules: true, + ...importOptions + } + return transpileToSource(program, context, skipUndefined, fullImportOptions) } } From 7999355d56f895afeb4ac53ebe7fedf93dc0e86f Mon Sep 17 00:00:00 2001 From: leeyi45 Date: Mon, 21 Aug 2023 02:42:29 +0800 Subject: [PATCH 06/24] Update runners --- src/runner/fullJSRunner.ts | 15 ++++++------- src/runner/htmlRunner.ts | 4 ++-- src/runner/sourceRunner.ts | 43 +++++++++++++++++++++++++------------- src/runner/utils.ts | 40 ++++++++++++++++++++++------------- 4 files changed, 64 insertions(+), 38 deletions(-) diff --git a/src/runner/fullJSRunner.ts b/src/runner/fullJSRunner.ts index ea9ad0d16..3d4ef8517 100644 --- a/src/runner/fullJSRunner.ts +++ b/src/runner/fullJSRunner.ts @@ -1,12 +1,13 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ import { generate } from 'astring' -import * as es from 'estree' +import type * as es from 'estree' import { RawSourceMap } from 'source-map' -import { IOptions, Result } from '..' +import type { Result } from '..' import { NATIVE_STORAGE_ID } from '../constants' import { RuntimeSourceError } from '../errors/runtimeSourceError' import { hoistAndMergeImports } from '../localImports/transformers/hoistAndMergeImports' +import { ImportTransformOptions } from '../modules/moduleTypes' import { getRequireProvider, RequireProvider } from '../modules/requireProvider' import { parse } from '../parser/parser' import { evallerReplacer, getBuiltins, transpile } from '../transpiler/transpiler' @@ -49,7 +50,7 @@ function containsPrevEval(context: Context): boolean { export async function fullJSRunner( program: es.Program, context: Context, - options: Partial = {} + importOptions: ImportTransformOptions ): Promise { // prelude & builtins // only process builtins and preludes if it is a fresh eval context @@ -63,7 +64,7 @@ export async function fullJSRunner( // modules hoistAndMergeImports(program) - appendModulesToContext(program, context) + await appendModulesToContext(program, context) // evaluate and create a separate block for preludes and builtins const preEvalProgram: es.Program = create.program([ @@ -80,12 +81,12 @@ export async function fullJSRunner( let transpiled let sourceMapJson: RawSourceMap | undefined try { - ;({ transpiled, sourceMapJson } = transpile(program, context)) - return Promise.resolve({ + ;({ transpiled, sourceMapJson } = await transpile(program, context, importOptions)) + return { status: 'finished', context, value: await fullJSEval(transpiled, requireProvider, context.nativeStorage) - }) + } } catch (error) { context.errors.push( error instanceof RuntimeSourceError ? error : await toSourceError(error, sourceMapJson) diff --git a/src/runner/htmlRunner.ts b/src/runner/htmlRunner.ts index f52c688f8..ba17dd062 100644 --- a/src/runner/htmlRunner.ts +++ b/src/runner/htmlRunner.ts @@ -1,5 +1,5 @@ import { IOptions, Result } from '..' -import { Context } from '../types' +import { Context, RecursivePartial } from '../types' const HTML_ERROR_HANDLING_SCRIPT_TEMPLATE = `