From 4189cc1ab5493ab0aef48c06416c4d16f6806245 Mon Sep 17 00:00:00 2001 From: "Kevin (Kun) \"Kassimo\" Qian" Date: Thu, 14 Nov 2019 05:31:39 -0800 Subject: [PATCH] Loader: support .wasm imports (#3328) * loader: support .wasm imports * http_server: true * Support named exports * Clippy --- .eslintignore | 1 + .prettierignore | 1 + Cargo.lock | 7 ++ cli/Cargo.toml | 1 + cli/compilers/mod.rs | 2 + cli/compilers/wasm.rs | 174 ++++++++++++++++++++++++++ cli/compilers/wasm_wrap.js | 19 +++ cli/file_fetcher.rs | 9 ++ cli/global_state.rs | 6 + cli/js/compiler.ts | 54 +++++++- cli/msg.rs | 4 +- cli/ops/compiler.rs | 32 ++++- cli/tests/051_wasm_import.ts | 22 ++++ cli/tests/051_wasm_import.ts.out | 1 + cli/tests/051_wasm_import/remote.ts | 3 + cli/tests/051_wasm_import/simple.wasm | Bin 0 -> 226 bytes cli/tests/051_wasm_import/simple.wat | 31 +++++ cli/tests/051_wasm_import/wasm-dep.js | 17 +++ cli/tests/integration_tests.rs | 6 + tools/lint.py | 7 +- 20 files changed, 388 insertions(+), 9 deletions(-) create mode 100644 cli/compilers/wasm.rs create mode 100644 cli/compilers/wasm_wrap.js create mode 100644 cli/tests/051_wasm_import.ts create mode 100644 cli/tests/051_wasm_import.ts.out create mode 100644 cli/tests/051_wasm_import/remote.ts create mode 100644 cli/tests/051_wasm_import/simple.wasm create mode 100644 cli/tests/051_wasm_import/simple.wat create mode 100644 cli/tests/051_wasm_import/wasm-dep.js diff --git a/.eslintignore b/.eslintignore index 9ccc3ac20b..40bd83b988 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1,3 +1,4 @@ +cli/compilers/wasm_wrap.js cli/tests/error_syntax.js std/deno.d.ts std/prettier/vendor diff --git a/.prettierignore b/.prettierignore index 589a2a3a49..b54ad06177 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,3 +1,4 @@ +cli/compilers/wasm_wrap.js cli/tests/error_syntax.js cli/tests/badly_formatted.js cli/tests/top_level_for_await.js diff --git a/Cargo.lock b/Cargo.lock index a764fbdb16..dcb3c7ab55 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -91,6 +91,11 @@ dependencies = [ "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "base64" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "bitflags" version = "1.2.1" @@ -282,6 +287,7 @@ version = "0.23.0" dependencies = [ "ansi_term 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)", "atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)", + "base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", "deno 0.23.0", @@ -1944,6 +1950,7 @@ dependencies = [ "checksum backtrace 0.3.40 (registry+https://github.com/rust-lang/crates.io-index)" = "924c76597f0d9ca25d762c25a4d369d51267536465dc5064bdf0eb073ed477ea" "checksum backtrace-sys 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)" = "5d6575f128516de27e3ce99689419835fce9643a9b215a14d2b5b685be018491" "checksum base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e" +"checksum base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" "checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" "checksum blake2b_simd 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)" = "b83b7baab1e671718d78204225800d6b170e648188ac7dc992e9d6bddf87d0c0" "checksum bumpalo 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ad807f2fc2bf185eeb98ff3a901bd46dc5ad58163d0fa4577ba0d25674d71708" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 792b5aec8a..4e2ab18c83 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -27,6 +27,7 @@ deno_typescript = { path = "../deno_typescript", version = "0.23.0" } ansi_term = "0.12.1" atty = "0.2.13" +base64 = "0.11.0" byteorder = "1.3.2" clap = "2.33.0" dirs = "2.0.2" diff --git a/cli/compilers/mod.rs b/cli/compilers/mod.rs index fdc18d2bcf..dca5bc7b6e 100644 --- a/cli/compilers/mod.rs +++ b/cli/compilers/mod.rs @@ -5,10 +5,12 @@ use futures::Future; mod js; mod json; mod ts; +mod wasm; pub use js::JsCompiler; pub use json::JsonCompiler; pub use ts::TsCompiler; +pub use wasm::WasmCompiler; #[derive(Debug, Clone)] pub struct CompiledModule { diff --git a/cli/compilers/wasm.rs b/cli/compilers/wasm.rs new file mode 100644 index 0000000000..e0a715f84a --- /dev/null +++ b/cli/compilers/wasm.rs @@ -0,0 +1,174 @@ +// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. +use crate::compilers::CompiledModule; +use crate::compilers::CompiledModuleFuture; +use crate::file_fetcher::SourceFile; +use crate::global_state::ThreadSafeGlobalState; +use crate::startup_data; +use crate::state::*; +use crate::worker::Worker; +use deno::Buf; +use futures::Future; +use futures::IntoFuture; +use serde_derive::Deserialize; +use serde_json; +use std::collections::HashMap; +use std::sync::atomic::Ordering; +use std::sync::{Arc, Mutex}; +use url::Url; + +// TODO(kevinkassimo): This is a hack to encode/decode data as base64 string. +// (Since Deno namespace might not be available, Deno.read can fail). +// Binary data is already available through source_file.source_code. +// If this is proven too wasteful in practice, refactor this. + +// Ref: https://webassembly.github.io/esm-integration/js-api/index.html#esm-integration +// https://github.com/nodejs/node/blob/35ec01097b2a397ad0a22aac536fe07514876e21/lib/internal/modules/esm/translators.js#L190-L210 + +// Dynamically construct JS wrapper with custom static imports and named exports. +// Boots up an internal worker to resolve imports/exports through query from V8. + +static WASM_WRAP: &str = include_str!("./wasm_wrap.js"); + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +struct WasmModuleInfo { + import_list: Vec, + export_list: Vec, +} + +#[derive(Default)] +pub struct WasmCompiler { + cache: Arc>>, +} + +impl WasmCompiler { + /// Create a new V8 worker with snapshot of WASM compiler and setup compiler's runtime. + fn setup_worker(global_state: ThreadSafeGlobalState) -> Worker { + let (int, ext) = ThreadSafeState::create_channels(); + let worker_state = + ThreadSafeState::new(global_state.clone(), None, true, int) + .expect("Unable to create worker state"); + + // Count how many times we start the compiler worker. + global_state + .metrics + .compiler_starts + .fetch_add(1, Ordering::SeqCst); + + let mut worker = Worker::new( + "WASM".to_string(), + startup_data::compiler_isolate_init(), + worker_state, + ext, + ); + worker.execute("denoMain('WASM')").unwrap(); + worker.execute("workerMain()").unwrap(); + worker.execute("wasmCompilerMain()").unwrap(); + worker + } + + pub fn compile_async( + self: &Self, + global_state: ThreadSafeGlobalState, + source_file: &SourceFile, + ) -> Box { + let cache = self.cache.clone(); + let maybe_cached = { cache.lock().unwrap().get(&source_file.url).cloned() }; + if let Some(m) = maybe_cached { + return Box::new(futures::future::ok(m.clone())); + } + let cache_ = self.cache.clone(); + + debug!(">>>>> wasm_compile_async START"); + let base64_data = base64::encode(&source_file.source_code); + let worker = WasmCompiler::setup_worker(global_state.clone()); + let worker_ = worker.clone(); + let url = source_file.url.clone(); + + let fut = worker + .post_message( + serde_json::to_string(&base64_data) + .unwrap() + .into_boxed_str() + .into_boxed_bytes(), + ) + .into_future() + .then(move |_| worker) + .then(move |result| { + if let Err(err) = result { + // TODO(ry) Need to forward the error instead of exiting. + eprintln!("{}", err.to_string()); + std::process::exit(1); + } + debug!("Sent message to worker"); + worker_.get_message() + }) + .map_err(|_| panic!("not handled")) + .and_then(move |maybe_msg: Option| { + debug!("Received message from worker"); + let json_msg = maybe_msg.unwrap(); + let module_info: WasmModuleInfo = + serde_json::from_slice(&json_msg).unwrap(); + debug!("WASM module info: {:#?}", &module_info); + let code = wrap_wasm_code( + &base64_data, + &module_info.import_list, + &module_info.export_list, + ); + debug!("Generated code: {}", &code); + let module = CompiledModule { + code, + name: url.to_string(), + }; + { + cache_.lock().unwrap().insert(url.clone(), module.clone()); + } + debug!("<<<<< wasm_compile_async END"); + Ok(module) + }); + Box::new(fut) + } +} + +fn build_single_import(index: usize, origin: &str) -> String { + let origin_json = serde_json::to_string(origin).unwrap(); + format!( + r#"import * as m{} from {}; +importObject[{}] = m{}; +"#, + index, &origin_json, &origin_json, index + ) +} + +fn build_imports(imports: &[String]) -> String { + let mut code = String::from(""); + for (index, origin) in imports.iter().enumerate() { + code.push_str(&build_single_import(index, origin)); + } + code +} + +fn build_single_export(name: &str) -> String { + format!("export const {} = instance.exports.{};\n", name, name) +} + +fn build_exports(exports: &[String]) -> String { + let mut code = String::from(""); + for e in exports { + code.push_str(&build_single_export(e)); + } + code +} + +fn wrap_wasm_code( + base64_data: &str, + imports: &[String], + exports: &[String], +) -> String { + let imports_code = build_imports(imports); + let exports_code = build_exports(exports); + String::from(WASM_WRAP) + .replace("//IMPORTS\n", &imports_code) + .replace("//EXPORTS\n", &exports_code) + .replace("BASE64_DATA", base64_data) +} diff --git a/cli/compilers/wasm_wrap.js b/cli/compilers/wasm_wrap.js new file mode 100644 index 0000000000..c90bd55407 --- /dev/null +++ b/cli/compilers/wasm_wrap.js @@ -0,0 +1,19 @@ +const importObject = Object.create(null); +//IMPORTS + +function base64ToUint8Array(data) { + const binString = window.atob(data); + const size = binString.length; + const bytes = new Uint8Array(size); + for (let i = 0; i < size; i++) { + bytes[i] = binString.charCodeAt(i); + } + return bytes; +} + +const buffer = base64ToUint8Array("BASE64_DATA"); +const compiled = await WebAssembly.compile(buffer); + +const instance = new WebAssembly.Instance(compiled, importObject); + +//EXPORTS diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs index 99ea61795f..d28ed0e26e 100644 --- a/cli/file_fetcher.rs +++ b/cli/file_fetcher.rs @@ -491,6 +491,7 @@ fn map_file_extension(path: &Path) -> msg::MediaType { Some("jsx") => msg::MediaType::JSX, Some("mjs") => msg::MediaType::JavaScript, Some("json") => msg::MediaType::Json, + Some("wasm") => msg::MediaType::Wasm, _ => msg::MediaType::Unknown, }, } @@ -1503,6 +1504,10 @@ mod tests { map_file_extension(Path::new("foo/bar.json")), msg::MediaType::Json ); + assert_eq!( + map_file_extension(Path::new("foo/bar.wasm")), + msg::MediaType::Wasm + ); assert_eq!( map_file_extension(Path::new("foo/bar.txt")), msg::MediaType::Unknown @@ -1544,6 +1549,10 @@ mod tests { map_content_type(Path::new("foo/bar.json"), None), msg::MediaType::Json ); + assert_eq!( + map_content_type(Path::new("foo/bar.wasm"), None), + msg::MediaType::Wasm + ); assert_eq!( map_content_type(Path::new("foo/bar"), None), msg::MediaType::Unknown diff --git a/cli/global_state.rs b/cli/global_state.rs index 3e102cb4ee..b0c2821707 100644 --- a/cli/global_state.rs +++ b/cli/global_state.rs @@ -3,6 +3,7 @@ use crate::compilers::CompiledModule; use crate::compilers::JsCompiler; use crate::compilers::JsonCompiler; use crate::compilers::TsCompiler; +use crate::compilers::WasmCompiler; use crate::deno_dir; use crate::deno_error::permission_denied; use crate::file_fetcher::SourceFileFetcher; @@ -45,6 +46,7 @@ pub struct GlobalState { pub js_compiler: JsCompiler, pub json_compiler: JsonCompiler, pub ts_compiler: TsCompiler, + pub wasm_compiler: WasmCompiler, pub lockfile: Option>, } @@ -111,6 +113,7 @@ impl ThreadSafeGlobalState { ts_compiler, js_compiler: JsCompiler {}, json_compiler: JsonCompiler {}, + wasm_compiler: WasmCompiler::default(), lockfile, }; @@ -130,6 +133,9 @@ impl ThreadSafeGlobalState { .and_then(move |out| match out.media_type { msg::MediaType::Unknown => state1.js_compiler.compile_async(&out), msg::MediaType::Json => state1.json_compiler.compile_async(&out), + msg::MediaType::Wasm => { + state1.wasm_compiler.compile_async(state1.clone(), &out) + } msg::MediaType::TypeScript | msg::MediaType::TSX | msg::MediaType::JSX => { diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts index 89c1107402..775277cdd4 100644 --- a/cli/js/compiler.ts +++ b/cli/js/compiler.ts @@ -28,7 +28,8 @@ enum MediaType { TypeScript = 2, TSX = 3, Json = 4, - Unknown = 5 + Wasm = 5, + Unknown = 6 } // Warning! The values in this enum are duplicated in cli/msg.rs @@ -44,8 +45,8 @@ enum CompilerRequestType { const console = new Console(core.print); window.console = console; window.workerMain = workerMain; -function denoMain(): void { - os.start(true, "TS"); +function denoMain(compilerType?: string): void { + os.start(true, compilerType || "TS"); } window["denoMain"] = denoMain; @@ -371,6 +372,9 @@ function getExtension(fileName: string, mediaType: MediaType): ts.Extension { return ts.Extension.Tsx; case MediaType.Json: return ts.Extension.Json; + case MediaType.Wasm: + // Custom marker for Wasm type. + return ts.Extension.Js; case MediaType.Unknown: default: throw TypeError("Cannot resolve extension."); @@ -724,3 +728,47 @@ window.compilerMain = function compilerMain(): void { workerClose(); }; }; + +function base64ToUint8Array(data: string): Uint8Array { + const binString = window.atob(data); + const size = binString.length; + const bytes = new Uint8Array(size); + for (let i = 0; i < size; i++) { + bytes[i] = binString.charCodeAt(i); + } + return bytes; +} + +window.wasmCompilerMain = function wasmCompilerMain(): void { + // workerMain should have already been called since a compiler is a worker. + window.onmessage = async ({ + data: binary + }: { + data: string; + }): Promise => { + const buffer = base64ToUint8Array(binary); + // @ts-ignore + const compiled = await WebAssembly.compile(buffer); + + util.log(">>> WASM compile start"); + + const importList = Array.from( + // @ts-ignore + new Set(WebAssembly.Module.imports(compiled).map(({ module }) => module)) + ); + const exportList = Array.from( + // @ts-ignore + new Set(WebAssembly.Module.exports(compiled).map(({ name }) => name)) + ); + + postMessage({ + importList, + exportList + }); + + util.log("<<< WASM compile end"); + + // The compiler isolate exits after a single message. + workerClose(); + }; +}; diff --git a/cli/msg.rs b/cli/msg.rs index 7599d874be..dbfb3316f0 100644 --- a/cli/msg.rs +++ b/cli/msg.rs @@ -74,7 +74,8 @@ pub enum MediaType { TypeScript = 2, TSX = 3, Json = 4, - Unknown = 5, + Wasm = 5, + Unknown = 6, } pub fn enum_name_media_type(mt: MediaType) -> &'static str { @@ -84,6 +85,7 @@ pub fn enum_name_media_type(mt: MediaType) -> &'static str { MediaType::TypeScript => "TypeScript", MediaType::TSX => "TSX", MediaType::Json => "Json", + MediaType::Wasm => "Wasm", MediaType::Unknown => "Unknown", } } diff --git a/cli/ops/compiler.rs b/cli/ops/compiler.rs index e7d38f3643..a722db6af5 100644 --- a/cli/ops/compiler.rs +++ b/cli/ops/compiler.rs @@ -2,6 +2,7 @@ use super::dispatch_json::{Deserialize, JsonOp, Value}; use crate::futures::future::join_all; use crate::futures::Future; +use crate::msg; use crate::ops::json_op; use crate::state::ThreadSafeState; use deno::Loader; @@ -74,17 +75,44 @@ fn op_fetch_source_files( futures.push(fut); } + let global_state = state.global_state.clone(); + let future = join_all(futures) .map_err(ErrBox::from) .and_then(move |files| { - let res = files + // We want to get an array of futures that resolves to + let v: Vec<_> = files .into_iter() .map(|file| { + // Special handling of Wasm files: + // compile them into JS first! + // This allows TS to do correct export types. + if file.media_type == msg::MediaType::Wasm { + return futures::future::Either::A( + global_state + .wasm_compiler + .compile_async(global_state.clone(), &file) + .and_then(|compiled_mod| Ok((file, Some(compiled_mod.code)))), + ); + } + futures::future::Either::B(futures::future::ok((file, None))) + }) + .collect(); + join_all(v) + }) + .and_then(move |files_with_code| { + let res = files_with_code + .into_iter() + .map(|(file, maybe_code)| { json!({ "url": file.url.to_string(), "filename": file.filename.to_str().unwrap(), "mediaType": file.media_type as i32, - "sourceCode": String::from_utf8(file.source_code).unwrap(), + "sourceCode": if let Some(code) = maybe_code { + code + } else { + String::from_utf8(file.source_code).unwrap() + }, }) }) .collect(); diff --git a/cli/tests/051_wasm_import.ts b/cli/tests/051_wasm_import.ts new file mode 100644 index 0000000000..7000657c3b --- /dev/null +++ b/cli/tests/051_wasm_import.ts @@ -0,0 +1,22 @@ +import { add, addImported, addRemote } from "./051_wasm_import/simple.wasm"; +import { state } from "./051_wasm_import/wasm-dep.js"; + +function assertEquals(actual: unknown, expected: unknown, msg?: string): void { + if (actual !== expected) { + throw new Error(msg); + } +} + +assertEquals(state, "WASM Start Executed", "Incorrect state"); + +assertEquals(add(10, 20), 30, "Incorrect add"); + +assertEquals(addImported(0), 42, "Incorrect addImported"); + +assertEquals(state, "WASM JS Function Executed", "Incorrect state"); + +assertEquals(addImported(1), 43, "Incorrect addImported"); + +assertEquals(addRemote(1), 2020, "Incorrect addRemote"); + +console.log("Passed"); diff --git a/cli/tests/051_wasm_import.ts.out b/cli/tests/051_wasm_import.ts.out new file mode 100644 index 0000000000..863339fb8c --- /dev/null +++ b/cli/tests/051_wasm_import.ts.out @@ -0,0 +1 @@ +Passed diff --git a/cli/tests/051_wasm_import/remote.ts b/cli/tests/051_wasm_import/remote.ts new file mode 100644 index 0000000000..761a5248e2 --- /dev/null +++ b/cli/tests/051_wasm_import/remote.ts @@ -0,0 +1,3 @@ +export function jsRemoteFn(): number { + return 2019; +} diff --git a/cli/tests/051_wasm_import/simple.wasm b/cli/tests/051_wasm_import/simple.wasm new file mode 100644 index 0000000000000000000000000000000000000000..8e544fe30594fa96e020262683ec284c9975224a GIT binary patch literal 226 zcmY*RT?>Lh5Zv|7G$jyv5BmY#X(*9DAn3I}pwb=Wkso+RJ^I%>t(RDsncbZksCA0~ zfLmGtP8C4qOp_I8!FiDl<8Ny@2stpNsnVAY;Q#0=eeD`!A9#KkGZdVs#}iBJq(zoz zIln&}ZlpGvCwW@VePz=Q{b=|o+ulfKbfENY(Q`j>DLRCBdqQd