From 7c3b9b4f4f2f4ec8fdeb0e77bb853fd22ffaa476 Mon Sep 17 00:00:00 2001 From: Andy Finch Date: Thu, 5 Dec 2019 15:30:20 -0500 Subject: [PATCH] feat: first pass at native plugins (#3372) --- Cargo.lock | 68 ++++++++++++++++++ Cargo.toml | 1 + cli/Cargo.toml | 1 + cli/deno_error.rs | 15 ++++ cli/flags.rs | 17 +++++ cli/js/deno.ts | 1 + cli/js/dispatch.ts | 18 ++++- cli/js/lib.deno_runtime.d.ts | 35 ++++++++++ cli/js/permissions.ts | 5 ++ cli/js/permissions_test.ts | 14 +++- cli/js/plugins.ts | 66 ++++++++++++++++++ cli/js/test_util.ts | 14 +++- cli/ops/mod.rs | 1 + cli/ops/permissions.rs | 2 + cli/ops/plugins.rs | 96 ++++++++++++++++++++++++++ cli/ops/workers.rs | 7 +- cli/permissions.rs | 29 ++++++++ cli/state.rs | 5 ++ cli/worker.rs | 2 + core/lib.rs | 2 + core/ops.rs | 2 +- core/plugins.rs | 22 ++++++ test_plugin/Cargo.toml | 14 ++++ test_plugin/src/lib.rs | 53 ++++++++++++++ test_plugin/tests/integration_tests.rs | 44 ++++++++++++ test_plugin/tests/test.js | 47 +++++++++++++ 26 files changed, 574 insertions(+), 7 deletions(-) create mode 100644 cli/js/plugins.ts create mode 100644 cli/ops/plugins.rs create mode 100644 core/plugins.rs create mode 100644 test_plugin/Cargo.toml create mode 100644 test_plugin/src/lib.rs create mode 100644 test_plugin/tests/integration_tests.rs create mode 100644 test_plugin/tests/test.js diff --git a/Cargo.lock b/Cargo.lock index 9624ea88af..fddd566423 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -293,6 +293,7 @@ dependencies = [ "deno 0.25.0", "deno_typescript 0.25.0", "dirs 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "dlopen 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "fwdansi 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "http 0.1.19 (registry+https://github.com/rust-lang/crates.io-index)", @@ -357,6 +358,27 @@ dependencies = [ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "dlopen" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "dlopen_derive 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "dlopen_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.44 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "downcast-rs" version = "1.1.1" @@ -969,6 +991,14 @@ name = "proc-macro-nested" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "proc-macro2" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "proc-macro2" version = "1.0.6" @@ -989,6 +1019,14 @@ dependencies = [ "url 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "quote" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "quote" version = "1.0.2" @@ -1436,6 +1474,16 @@ name = "strsim" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "syn" +version = "0.15.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "syn" version = "1.0.7" @@ -1487,6 +1535,15 @@ dependencies = [ "wincolor 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "test_plugin" +version = "0.0.1" +dependencies = [ + "deno 0.25.0", + "deno_cli 0.25.0", + "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "textwrap" version = "0.11.0" @@ -1786,6 +1843,11 @@ name = "unicode-width" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "unicode-xid" version = "0.2.0" @@ -2075,6 +2137,8 @@ dependencies = [ "checksum ct-logs 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4d3686f5fa27dbc1d76c751300376e167c5a43387f44bb451fd1c24776e49113" "checksum dirs 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "13aea89a5c93364a98e9b37b2fa237effbb694d5cfe01c5b70941f7eb087d5e3" "checksum dirs-sys 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "afa0b23de8fd801745c471deffa6e12d248f962c9fd4b4c33787b055599bde7b" +"checksum dlopen 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "71e80ad39f814a9abe68583cd50a2d45c8a67561c3361ab8da240587dda80937" +"checksum dlopen_derive 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "f236d9e1b1fbd81cea0f9cbdc8dcc7e8ebcd80e6659cd7cb2ad5f6c05946c581" "checksum downcast-rs 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "52ba6eb47c2131e784a38b726eb54c1e1484904f013e576a25354d0124161af6" "checksum dtoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "ea57b42383d091c85abcc2706240b94ab2a8fa1fc81c10ff23c4de06e2a90b5e" "checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3" @@ -2144,8 +2208,10 @@ dependencies = [ "checksum ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" "checksum proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)" = "ecd45702f76d6d3c75a80564378ae228a85f0b59d2f3ed43c91b4a69eb2ebfc5" "checksum proc-macro-nested 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "369a6ed065f249a159e06c45752c780bda2fb53c995718f9e484d08daa9eb42e" +"checksum proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)" = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" "checksum proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "9c9e470a8dc4aeae2dee2f335e8f533e2d4b347e1434e5671afc49b054592f27" "checksum publicsuffix 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "9bf259a81de2b2eb9850ec990ec78e6a25319715584fd7652b9b26f96fcb1510" +"checksum quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" "checksum quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe" "checksum rand 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" "checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" @@ -2194,6 +2260,7 @@ dependencies = [ "checksum spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" "checksum string 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d24114bfcceb867ca7f71a0d3fe45d45619ec47a6fbfa98cb14e14250bfa5d6d" "checksum strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" +"checksum syn 0.15.44 (registry+https://github.com/rust-lang/crates.io-index)" = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" "checksum syn 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)" = "0e7bedb3320d0f3035594b0b723c8a28d7d336a3eda3881db79e61d676fb644c" "checksum synstructure 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)" = "575be94ccb86e8da37efb894a87e2b660be299b41d8ef347f9d6d79fbe61b1ba" "checksum sys-info 0.5.8 (registry+https://github.com/rust-lang/crates.io-index)" = "0079fe39cec2c8215e21b0bc4ccec9031004c160b88358f531b601e96b77f0df" @@ -2226,6 +2293,7 @@ dependencies = [ "checksum unicode-normalization 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "09c8070a9942f5e7cfccd93f490fdebd230ee3c3c9f107cb25bad5351ef671cf" "checksum unicode-segmentation 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "49f5526225fd8b77342d5986ab5f6055552e9c0776193b5b63fd53b46debfad7" "checksum unicode-width 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7007dbd421b92cc6e28410fe7362e2e0a2503394908f417b68ec8d1c364c4e20" +"checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" "checksum unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" "checksum untrusted 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60369ef7a31de49bcb3f6ca728d4ba7300d9a1658f94c727d4cab8c8d9f4aece" "checksum url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dd4e7c0d531266369519a4aa4f399d748bd37043b00bde1e4ff1f60a120b355a" diff --git a/Cargo.toml b/Cargo.toml index a54477a120..b273e4222f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,4 +4,5 @@ members = [ "core", "tools/hyper_hello", "deno_typescript", + "test_plugin" ] diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 4645f24455..417141ba36 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -31,6 +31,7 @@ base64 = "0.11.0" byteorder = "1.3.2" clap = "2.33.0" dirs = "2.0.2" +dlopen = "0.1.8" futures = { version = "0.3", features = [ "compat", "io-compat" ] } http = "0.1.19" hyper = "0.12.35" diff --git a/cli/deno_error.rs b/cli/deno_error.rs index 8cff29d30a..8d0eea2012 100644 --- a/cli/deno_error.rs +++ b/cli/deno_error.rs @@ -6,6 +6,7 @@ pub use crate::msg::ErrorKind; use deno::AnyError; use deno::ErrBox; use deno::ModuleResolutionError; +use dlopen::Error as DlopenError; use http::uri; use hyper; use reqwest; @@ -292,6 +293,19 @@ mod unix { } } +impl GetErrorKind for DlopenError { + fn kind(&self) -> ErrorKind { + use dlopen::Error::*; + match self { + NullCharacter(_) => ErrorKind::Other, + OpeningLibraryError(e) => GetErrorKind::kind(e), + SymbolGettingError(e) => GetErrorKind::kind(e), + NullSymbol => ErrorKind::Other, + AddrNotMatchingDll(e) => GetErrorKind::kind(e), + } + } +} + impl GetErrorKind for dyn AnyError { fn kind(&self) -> ErrorKind { use self::GetErrorKind as Get; @@ -325,6 +339,7 @@ impl GetErrorKind for dyn AnyError { .downcast_ref::() .map(Get::kind) }) + .or_else(|| self.downcast_ref::().map(Get::kind)) .or_else(|| unix_error_kind(self)) .unwrap_or_else(|| { panic!("Can't get ErrorKind for {:?}", self); diff --git a/cli/flags.rs b/cli/flags.rs index f23879e0fd..770f422aad 100644 --- a/cli/flags.rs +++ b/cli/flags.rs @@ -82,6 +82,7 @@ pub struct DenoFlags { pub net_whitelist: Vec, pub allow_env: bool, pub allow_run: bool, + pub allow_plugin: bool, pub allow_hrtime: bool, pub no_prompts: bool, pub no_remote: bool, @@ -346,6 +347,7 @@ fn xeval_parse(flags: &mut DenoFlags, matches: &clap::ArgMatches) { flags.allow_run = true; flags.allow_read = true; flags.allow_write = true; + flags.allow_plugin = true; flags.allow_hrtime = true; flags.argv.push(XEVAL_URL.to_string()); @@ -373,6 +375,7 @@ fn repl_parse(flags: &mut DenoFlags, matches: &clap::ArgMatches) { flags.allow_run = true; flags.allow_read = true; flags.allow_write = true; + flags.allow_plugin = true; flags.allow_hrtime = true; } @@ -383,6 +386,7 @@ fn eval_parse(flags: &mut DenoFlags, matches: &clap::ArgMatches) { flags.allow_run = true; flags.allow_read = true; flags.allow_write = true; + flags.allow_plugin = true; flags.allow_hrtime = true; let code: &str = matches.value_of("code").unwrap(); flags.argv.extend(vec![code.to_string()]); @@ -465,6 +469,9 @@ fn run_test_args_parse(flags: &mut DenoFlags, matches: &clap::ArgMatches) { if matches.is_present("allow-run") { flags.allow_run = true; } + if matches.is_present("allow-plugin") { + flags.allow_plugin = true; + } if matches.is_present("allow-hrtime") { flags.allow_hrtime = true; } @@ -475,6 +482,7 @@ fn run_test_args_parse(flags: &mut DenoFlags, matches: &clap::ArgMatches) { flags.allow_run = true; flags.allow_read = true; flags.allow_write = true; + flags.allow_plugin = true; flags.allow_hrtime = true; } if matches.is_present("cached-only") { @@ -942,6 +950,11 @@ fn run_test_args<'a, 'b>(app: App<'a, 'b>) -> App<'a, 'b> { .long("allow-run") .help("Allow running subprocesses"), ) + .arg( + Arg::with_name("allow-plugin") + .long("allow-plugin") + .help("Allow loading plugins"), + ) .arg( Arg::with_name("allow-hrtime") .long("allow-hrtime") @@ -1408,6 +1421,7 @@ mod tests { allow_run: true, allow_read: true, allow_write: true, + allow_plugin: true, allow_hrtime: true, ..DenoFlags::default() } @@ -1581,6 +1595,7 @@ mod tests { allow_run: true, allow_read: true, allow_write: true, + allow_plugin: true, allow_hrtime: true, ..DenoFlags::default() } @@ -1600,6 +1615,7 @@ mod tests { allow_run: true, allow_read: true, allow_write: true, + allow_plugin: true, allow_hrtime: true, ..DenoFlags::default() } @@ -1635,6 +1651,7 @@ mod tests { allow_run: true, allow_read: true, allow_write: true, + allow_plugin: true, allow_hrtime: true, ..DenoFlags::default() } diff --git a/cli/js/deno.ts b/cli/js/deno.ts index 6f07bef675..27a7bb3bd0 100644 --- a/cli/js/deno.ts +++ b/cli/js/deno.ts @@ -76,6 +76,7 @@ export { } from "./permissions.ts"; export { truncateSync, truncate } from "./truncate.ts"; export { FileInfo } from "./file_info.ts"; +export { openPlugin } from "./plugins.ts"; export { connect, dial, listen, Listener, Conn } from "./net.ts"; export { dialTLS, listenTLS } from "./tls.ts"; export { metrics, Metrics } from "./metrics.ts"; diff --git a/cli/js/dispatch.ts b/cli/js/dispatch.ts index 35806c3add..ed6f570520 100644 --- a/cli/js/dispatch.ts +++ b/cli/js/dispatch.ts @@ -1,6 +1,7 @@ // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. import * as minimal from "./dispatch_minimal.ts"; import * as json from "./dispatch_json.ts"; +import { AsyncHandler } from "./plugins.ts"; // These consts are shared with Rust. Update with care. export let OP_READ: number; @@ -67,6 +68,16 @@ export let OP_CWD: number; export let OP_FETCH_ASSET: number; export let OP_DIAL_TLS: number; export let OP_HOSTNAME: number; +export let OP_OPEN_PLUGIN: number; + +const PLUGIN_ASYNC_HANDLER_MAP: Map = new Map(); + +export function setPluginAsyncHandler( + opId: number, + handler: AsyncHandler +): void { + PLUGIN_ASYNC_HANDLER_MAP.set(opId, handler); +} export function asyncMsgFromRust(opId: number, ui8: Uint8Array): void { switch (opId) { @@ -111,6 +122,11 @@ export function asyncMsgFromRust(opId: number, ui8: Uint8Array): void { json.asyncMsgFromRust(opId, ui8); break; default: - throw Error("bad async opId"); + const handler = PLUGIN_ASYNC_HANDLER_MAP.get(opId); + if (handler) { + handler(ui8); + } else { + throw Error("bad async opId"); + } } } diff --git a/cli/js/lib.deno_runtime.d.ts b/cli/js/lib.deno_runtime.d.ts index fb7767aa63..0d34638696 100644 --- a/cli/js/lib.deno_runtime.d.ts +++ b/cli/js/lib.deno_runtime.d.ts @@ -907,6 +907,7 @@ declare namespace Deno { | "write" | "net" | "env" + | "plugin" | "hrtime"; /** https://w3c.github.io/permissions/#status-of-a-permission */ export type PermissionState = "granted" | "denied" | "prompt"; @@ -924,6 +925,9 @@ declare namespace Deno { interface EnvPermissionDescriptor { name: "env"; } + interface PluginPermissionDescriptor { + name: "plugin"; + } interface HrtimePermissionDescriptor { name: "hrtime"; } @@ -933,6 +937,7 @@ declare namespace Deno { | ReadWritePermissionDescriptor | NetPermissionDescriptor | EnvPermissionDescriptor + | PluginPermissionDescriptor | HrtimePermissionDescriptor; export class Permissions { @@ -982,6 +987,36 @@ declare namespace Deno { */ export function truncate(name: string, len?: number): Promise; + // @url js/plugins.d.ts + + export interface AsyncHandler { + (msg: Uint8Array): void; + } + + export interface PluginOp { + dispatch( + control: Uint8Array, + zeroCopy?: ArrayBufferView | null + ): Uint8Array | null; + setAsyncHandler(handler: AsyncHandler): void; + } + + export interface Plugin { + ops: { + [name: string]: PluginOp; + }; + } + + /** Open and initalize a plugin. + * Requires the `--allow-plugin` flag. + * + * const plugin = Deno.openPlugin("./path/to/some/plugin.so"); + * const some_op = plugin.ops.some_op; + * const response = some_op.dispatch(new Uint8Array([1,2,3,4])); + * console.log(`Response from plugin ${response}`); + */ + export function openPlugin(filename: string): Plugin; + // @url js/net.d.ts type Transport = "tcp"; diff --git a/cli/js/permissions.ts b/cli/js/permissions.ts index c3530e9700..e0fb8a84c1 100644 --- a/cli/js/permissions.ts +++ b/cli/js/permissions.ts @@ -11,6 +11,7 @@ export type PermissionName = | "net" | "env" | "run" + | "plugin" | "hrtime"; // NOTE: Keep in sync with cli/permissions.rs @@ -31,6 +32,9 @@ interface NetPermissionDescriptor { interface EnvPermissionDescriptor { name: "env"; } +interface PluginPermissionDescriptor { + name: "plugin"; +} interface HrtimePermissionDescriptor { name: "hrtime"; } @@ -40,6 +44,7 @@ type PermissionDescriptor = | ReadWritePermissionDescriptor | NetPermissionDescriptor | EnvPermissionDescriptor + | PluginPermissionDescriptor | HrtimePermissionDescriptor; /** https://w3c.github.io/permissions/#permissionstatus */ diff --git a/cli/js/permissions_test.ts b/cli/js/permissions_test.ts index d9ba538f0e..a50718652f 100644 --- a/cli/js/permissions_test.ts +++ b/cli/js/permissions_test.ts @@ -7,11 +7,12 @@ const knownPermissions: Deno.PermissionName[] = [ "write", "net", "env", + "plugin", "hrtime" ]; -for (const grant of knownPermissions) { - testPerm({ [grant]: true }, async function envGranted(): Promise { +function genFunc(grant: Deno.PermissionName): () => Promise { + const gen: () => Promise = async function Granted(): Promise { const status0 = await Deno.permissions.query({ name: grant }); assert(status0 != null); assertEquals(status0.state, "granted"); @@ -19,7 +20,14 @@ for (const grant of knownPermissions) { const status1 = await Deno.permissions.revoke({ name: grant }); assert(status1 != null); assertEquals(status1.state, "prompt"); - }); + }; + // Properly name these generated functions. + Object.defineProperty(gen, "name", { value: grant + "Granted" }); + return gen; +} + +for (const grant of knownPermissions) { + testPerm({ [grant]: true }, genFunc(grant)); } test(async function permissionInvalidName(): Promise { diff --git a/cli/js/plugins.ts b/cli/js/plugins.ts new file mode 100644 index 0000000000..324ae34082 --- /dev/null +++ b/cli/js/plugins.ts @@ -0,0 +1,66 @@ +import { sendSync } from "./dispatch_json.ts"; +import { OP_OPEN_PLUGIN, setPluginAsyncHandler } from "./dispatch.ts"; +import { core } from "./core.ts"; + +export interface AsyncHandler { + (msg: Uint8Array): void; +} + +interface PluginOp { + dispatch( + control: Uint8Array, + zeroCopy?: ArrayBufferView | null + ): Uint8Array | null; + setAsyncHandler(handler: AsyncHandler): void; +} + +class PluginOpImpl implements PluginOp { + constructor(private readonly opId: number) {} + + dispatch( + control: Uint8Array, + zeroCopy?: ArrayBufferView | null + ): Uint8Array | null { + return core.dispatch(this.opId, control, zeroCopy); + } + + setAsyncHandler(handler: AsyncHandler): void { + setPluginAsyncHandler(this.opId, handler); + } +} + +// TODO(afinch7): add close method. + +interface Plugin { + ops: { + [name: string]: PluginOp; + }; +} + +class PluginImpl implements Plugin { + private _ops: { [name: string]: PluginOp } = {}; + + constructor(private readonly rid: number, ops: { [name: string]: number }) { + for (const op in ops) { + this._ops[op] = new PluginOpImpl(ops[op]); + } + } + + get ops(): { [name: string]: PluginOp } { + return Object.assign({}, this._ops); + } +} + +interface OpenPluginResponse { + rid: number; + ops: { + [name: string]: number; + }; +} + +export function openPlugin(filename: string): Plugin { + const response: OpenPluginResponse = sendSync(OP_OPEN_PLUGIN, { + filename + }); + return new PluginImpl(response.rid, response.ops); +} diff --git a/cli/js/test_util.ts b/cli/js/test_util.ts index 85fffabe64..aac70d2ede 100644 --- a/cli/js/test_util.ts +++ b/cli/js/test_util.ts @@ -26,6 +26,7 @@ interface TestPermissions { net?: boolean; env?: boolean; run?: boolean; + plugin?: boolean; hrtime?: boolean; } @@ -35,6 +36,7 @@ export interface Permissions { net: boolean; env: boolean; run: boolean; + plugin: boolean; hrtime: boolean; } @@ -48,6 +50,7 @@ async function getProcessPermissions(): Promise { write: await isGranted("write"), net: await isGranted("net"), env: await isGranted("env"), + plugin: await isGranted("plugin"), hrtime: await isGranted("hrtime") }; } @@ -75,8 +78,9 @@ function permToString(perms: Permissions): string { const n = perms.net ? 1 : 0; const e = perms.env ? 1 : 0; const u = perms.run ? 1 : 0; + const p = perms.plugin ? 1 : 0; const h = perms.hrtime ? 1 : 0; - return `permR${r}W${w}N${n}E${e}U${u}H${h}`; + return `permR${r}W${w}N${n}E${e}U${u}P${p}H${h}`; } function registerPermCombination(perms: Permissions): void { @@ -93,6 +97,7 @@ function normalizeTestPermissions(perms: TestPermissions): Permissions { net: !!perms.net, run: !!perms.run, env: !!perms.env, + plugin: !!perms.plugin, hrtime: !!perms.hrtime }; } @@ -120,6 +125,7 @@ export function test(fn: testing.TestFunction): void { net: false, env: false, run: false, + plugin: false, hrtime: false }, fn @@ -176,6 +182,7 @@ test(function permissionsMatches(): void { net: false, env: false, run: false, + plugin: false, hrtime: false }, normalizeTestPermissions({ read: true }) @@ -190,6 +197,7 @@ test(function permissionsMatches(): void { net: false, env: false, run: false, + plugin: false, hrtime: false }, normalizeTestPermissions({}) @@ -204,6 +212,7 @@ test(function permissionsMatches(): void { net: true, env: true, run: true, + plugin: true, hrtime: true }, normalizeTestPermissions({ read: true }) @@ -219,6 +228,7 @@ test(function permissionsMatches(): void { net: true, env: false, run: false, + plugin: false, hrtime: false }, normalizeTestPermissions({ read: true }) @@ -234,6 +244,7 @@ test(function permissionsMatches(): void { net: true, env: true, run: true, + plugin: true, hrtime: true }, { @@ -242,6 +253,7 @@ test(function permissionsMatches(): void { net: true, env: true, run: true, + plugin: true, hrtime: true } ) diff --git a/cli/ops/mod.rs b/cli/ops/mod.rs index 9b33d59189..28bf7e2176 100644 --- a/cli/ops/mod.rs +++ b/cli/ops/mod.rs @@ -16,6 +16,7 @@ pub mod io; pub mod net; pub mod os; pub mod permissions; +pub mod plugins; pub mod process; pub mod random; pub mod repl; diff --git a/cli/ops/permissions.rs b/cli/ops/permissions.rs index bd8340bf0c..f513f2fabe 100644 --- a/cli/ops/permissions.rs +++ b/cli/ops/permissions.rs @@ -55,6 +55,7 @@ pub fn op_revoke_permission( "write" => permissions.allow_write.revoke(), "net" => permissions.allow_net.revoke(), "env" => permissions.allow_env.revoke(), + "plugin" => permissions.allow_plugin.revoke(), "hrtime" => permissions.allow_hrtime.revoke(), _ => {} }; @@ -83,6 +84,7 @@ pub fn op_request_permission( } "net" => permissions.request_net(&args.url.as_ref().map(String::as_str)), "env" => Ok(permissions.request_env()), + "plugin" => Ok(permissions.request_plugin()), "hrtime" => Ok(permissions.request_hrtime()), n => Err(type_error(format!("No such permission name: {}", n))), }?; diff --git a/cli/ops/plugins.rs b/cli/ops/plugins.rs new file mode 100644 index 0000000000..2673e3d6aa --- /dev/null +++ b/cli/ops/plugins.rs @@ -0,0 +1,96 @@ +use super::dispatch_json::{Deserialize, JsonOp, Value}; +use crate::fs as deno_fs; +use crate::ops::json_op; +use crate::state::ThreadSafeState; +use deno::*; +use dlopen::symbor::Library; +use std::collections::HashMap; +use std::ffi::OsStr; +use std::sync::Arc; + +pub fn init(i: &mut Isolate, s: &ThreadSafeState, r: Arc) { + let r_ = r.clone(); + i.register_op( + "open_plugin", + s.core_op(json_op(s.stateful_op(move |state, args, zero_copy| { + op_open_plugin(&r_, state, args, zero_copy) + }))), + ); +} + +fn open_plugin>(lib_path: P) -> Result { + debug!("Loading Plugin: {:#?}", lib_path.as_ref()); + + Library::open(lib_path).map_err(ErrBox::from) +} + +struct PluginResource { + lib: Library, + ops: HashMap, +} + +impl Resource for PluginResource {} + +struct InitContext { + ops: HashMap>, +} + +impl PluginInitContext for InitContext { + fn register_op(&mut self, name: &str, op: Box) { + let existing = self.ops.insert(name.to_string(), op); + assert!( + existing.is_none(), + format!("Op already registered: {}", name) + ); + } +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct OpenPluginArgs { + filename: String, +} + +pub fn op_open_plugin( + registry: &Arc, + state: &ThreadSafeState, + args: Value, + _zero_copy: Option, +) -> Result { + let args: OpenPluginArgs = serde_json::from_value(args)?; + let (filename, filename_) = deno_fs::resolve_from_cwd(&args.filename)?; + + state.check_plugin(&filename_)?; + + let lib = open_plugin(filename)?; + let plugin_resource = PluginResource { + lib, + ops: HashMap::new(), + }; + let mut table = state.lock_resource_table(); + let rid = table.add("plugin", Box::new(plugin_resource)); + let plugin_resource = table.get_mut::(rid).unwrap(); + + let init_fn = *unsafe { + plugin_resource + .lib + .symbol::("deno_plugin_init") + }?; + let mut init_context = InitContext { + ops: HashMap::new(), + }; + init_fn(&mut init_context); + for op in init_context.ops { + // Register each plugin op in the `OpRegistry` with the name + // formated like this `plugin_{plugin_rid}_{name}`. + // The inclusion of prefix and rid is designed to avoid any + // op name collision beyond the bound of a single loaded + // plugin instance. + let op_id = registry.register(&format!("plugin_{}_{}", rid, op.0), op.1); + plugin_resource.ops.insert(op.0, op_id); + } + + Ok(JsonOp::Sync( + json!({ "rid": rid, "ops": plugin_resource.ops }), + )) +} diff --git a/cli/ops/workers.rs b/cli/ops/workers.rs index 4bde382227..2b4d11e75e 100644 --- a/cli/ops/workers.rs +++ b/cli/ops/workers.rs @@ -227,7 +227,12 @@ fn op_host_get_worker_closed( }; let op = future.then(move |_result| { let mut workers_table = state_.workers.lock().unwrap(); - workers_table.remove(&id); + let maybe_worker = workers_table.remove(&id); + if let Some(worker) = maybe_worker { + let mut channels = worker.state.worker_channels.lock().unwrap(); + channels.sender.close_channel(); + channels.receiver.close(); + }; futures::future::ok(json!({})) }); diff --git a/cli/permissions.rs b/cli/permissions.rs index 8bad2f7953..72a1a928ac 100644 --- a/cli/permissions.rs +++ b/cli/permissions.rs @@ -108,6 +108,7 @@ pub struct DenoPermissions { pub net_whitelist: HashSet, pub allow_env: PermissionState, pub allow_run: PermissionState, + pub allow_plugin: PermissionState, pub allow_hrtime: PermissionState, } @@ -122,6 +123,7 @@ impl DenoPermissions { net_whitelist: flags.net_whitelist.iter().cloned().collect(), allow_env: PermissionState::from(flags.allow_env), allow_run: PermissionState::from(flags.allow_run), + allow_plugin: PermissionState::from(flags.allow_plugin), allow_hrtime: PermissionState::from(flags.allow_hrtime), } } @@ -207,6 +209,13 @@ impl DenoPermissions { ) } + pub fn check_plugin(&self, filename: &str) -> Result<(), ErrBox> { + self.allow_plugin.check( + &format!("access to open a plugin: {}", filename), + "run again with the --allow-plugin flag", + ) + } + pub fn request_run(&mut self) -> PermissionState { self .allow_run @@ -258,6 +267,10 @@ impl DenoPermissions { .request("Deno requests to access to high precision time.") } + pub fn request_plugin(&mut self) -> PermissionState { + self.allow_plugin.request("Deno requests to open plugins.") + } + pub fn get_permission_state( &self, name: &str, @@ -270,6 +283,7 @@ impl DenoPermissions { "write" => Ok(self.get_state_write(path)), "net" => self.get_state_net_url(url), "env" => Ok(self.allow_env), + "plugin" => Ok(self.allow_plugin), "hrtime" => Ok(self.allow_hrtime), n => Err(type_error(format!("No such permission name: {}", n))), } @@ -652,6 +666,21 @@ mod tests { assert_eq!(perms1.request_env(), PermissionState::Deny); } + #[test] + fn test_permissions_request_plugin() { + let mut perms0 = DenoPermissions::from_flags(&DenoFlags { + ..Default::default() + }); + set_prompt_result(true); + assert_eq!(perms0.request_plugin(), PermissionState::Allow); + + let mut perms1 = DenoPermissions::from_flags(&DenoFlags { + ..Default::default() + }); + set_prompt_result(false); + assert_eq!(perms1.request_plugin(), PermissionState::Deny); + } + #[test] fn test_permissions_request_hrtime() { let mut perms0 = DenoPermissions::from_flags(&DenoFlags { diff --git a/cli/state.rs b/cli/state.rs index e99bff08ea..d31f667b52 100644 --- a/cli/state.rs +++ b/cli/state.rs @@ -295,6 +295,11 @@ impl ThreadSafeState { self.permissions.lock().unwrap().check_run() } + #[inline] + pub fn check_plugin(&self, filename: &str) -> Result<(), ErrBox> { + self.permissions.lock().unwrap().check_plugin(filename) + } + pub fn check_dyn_import( self: &Self, module_specifier: &ModuleSpecifier, diff --git a/cli/worker.rs b/cli/worker.rs index a8585b74e7..814e7f4402 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -50,6 +50,7 @@ impl Worker { let isolate = Arc::new(Mutex::new(deno::Isolate::new(startup_data, false))); { let mut i = isolate.lock().unwrap(); + let op_registry = i.op_registry.clone(); ops::compiler::init(&mut i, &state); ops::errors::init(&mut i, &state); @@ -57,6 +58,7 @@ impl Worker { ops::files::init(&mut i, &state); ops::fs::init(&mut i, &state); ops::io::init(&mut i, &state); + ops::plugins::init(&mut i, &state, op_registry); ops::net::init(&mut i, &state); ops::tls::init(&mut i, &state); ops::os::init(&mut i, &state); diff --git a/core/lib.rs b/core/lib.rs index 31f717769e..f1becb5d7e 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -14,6 +14,7 @@ mod libdeno; mod module_specifier; mod modules; mod ops; +mod plugins; mod resources; mod shared_queue; @@ -27,6 +28,7 @@ pub use crate::libdeno::PinnedBuf; pub use crate::module_specifier::*; pub use crate::modules::*; pub use crate::ops::*; +pub use crate::plugins::*; pub use crate::resources::*; pub fn v8_version() -> &'static str { diff --git a/core/ops.rs b/core/ops.rs index 6dc0a73239..c840ed979e 100644 --- a/core/ops.rs +++ b/core/ops.rs @@ -27,7 +27,7 @@ pub type CoreError = (); pub type CoreOp = Op; /// Main type describing op -type OpDispatcher = +pub type OpDispatcher = dyn Fn(&[u8], Option) -> CoreOp + Send + Sync + 'static; #[derive(Default)] diff --git a/core/plugins.rs b/core/plugins.rs new file mode 100644 index 0000000000..50271d53a4 --- /dev/null +++ b/core/plugins.rs @@ -0,0 +1,22 @@ +use crate::libdeno::PinnedBuf; +use crate::ops::CoreOp; + +pub type PluginInitFn = fn(context: &mut dyn PluginInitContext); + +pub trait PluginInitContext { + fn register_op( + &mut self, + name: &str, + op: Box) -> CoreOp + Send + Sync + 'static>, + ); +} + +#[macro_export] +macro_rules! init_fn { + ($fn:path) => { + #[no_mangle] + pub fn deno_plugin_init(context: &mut dyn PluginInitContext) { + $fn(context) + } + }; +} diff --git a/test_plugin/Cargo.toml b/test_plugin/Cargo.toml new file mode 100644 index 0000000000..d6c4eb1de3 --- /dev/null +++ b/test_plugin/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "test_plugin" +version = "0.0.1" +authors = ["the deno authors"] +edition = "2018" +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies] +futures = "0.3" +deno = { path = "../core" } +deno_cli = { path = "../cli" } \ No newline at end of file diff --git a/test_plugin/src/lib.rs b/test_plugin/src/lib.rs new file mode 100644 index 0000000000..30df114b94 --- /dev/null +++ b/test_plugin/src/lib.rs @@ -0,0 +1,53 @@ +#[macro_use] +extern crate deno; +extern crate futures; + +use deno::CoreOp; +use deno::Op; +use deno::PluginInitContext; +use deno::{Buf, PinnedBuf}; +use futures::future::FutureExt; + +fn init(context: &mut dyn PluginInitContext) { + context.register_op("testSync", Box::new(op_test_sync)); + context.register_op("testAsync", Box::new(op_test_async)); +} +init_fn!(init); + +pub fn op_test_sync(data: &[u8], zero_copy: Option) -> CoreOp { + if let Some(buf) = zero_copy { + let data_str = std::str::from_utf8(&data[..]).unwrap(); + let buf_str = std::str::from_utf8(&buf[..]).unwrap(); + println!( + "Hello from plugin. data: {} | zero_copy: {}", + data_str, buf_str + ); + } + let result = b"test"; + let result_box: Buf = Box::new(*result); + Op::Sync(result_box) +} + +pub fn op_test_async(data: &[u8], zero_copy: Option) -> CoreOp { + let data_str = std::str::from_utf8(&data[..]).unwrap().to_string(); + let fut = async move { + if let Some(buf) = zero_copy { + let buf_str = std::str::from_utf8(&buf[..]).unwrap(); + println!( + "Hello from plugin. data: {} | zero_copy: {}", + data_str, buf_str + ); + } + let (tx, rx) = futures::channel::oneshot::channel::>(); + std::thread::spawn(move || { + std::thread::sleep(std::time::Duration::from_secs(1)); + tx.send(Ok(())).unwrap(); + }); + assert!(rx.await.is_ok()); + let result = b"test"; + let result_box: Buf = Box::new(*result); + Ok(result_box) + }; + + Op::Async(fut.boxed()) +} diff --git a/test_plugin/tests/integration_tests.rs b/test_plugin/tests/integration_tests.rs new file mode 100644 index 0000000000..a6790d013b --- /dev/null +++ b/test_plugin/tests/integration_tests.rs @@ -0,0 +1,44 @@ +use deno_cli::test_util::*; +use std::process::Command; + +fn deno_cmd() -> Command { + assert!(deno_exe_path().exists()); + Command::new(deno_exe_path()) +} + +#[cfg(debug_assertions)] +const BUILD_VARIANT: &str = "debug"; + +#[cfg(not(debug_assertions))] +const BUILD_VARIANT: &str = "release"; + +#[test] +fn basic() { + let mut build_plugin_base = Command::new("cargo"); + let mut build_plugin = + build_plugin_base.arg("build").arg("-p").arg("test_plugin"); + if BUILD_VARIANT == "release" { + build_plugin = build_plugin.arg("--release"); + } + let _build_plugin_output = build_plugin.output().unwrap(); + let output = deno_cmd() + .arg("--allow-plugin") + .arg("tests/test.js") + .arg(BUILD_VARIANT) + .output() + .unwrap(); + let stdout = std::str::from_utf8(&output.stdout).unwrap(); + let stderr = std::str::from_utf8(&output.stderr).unwrap(); + if !output.status.success() { + println!("stdout {}", stdout); + println!("stderr {}", stderr); + } + assert!(output.status.success()); + let expected = if cfg!(target_os = "windows") { + "Hello from plugin. data: test | zero_copy: test\nPlugin Sync Response: test\r\nHello from plugin. data: test | zero_copy: test\nPlugin Async Response: test\r\n" + } else { + "Hello from plugin. data: test | zero_copy: test\nPlugin Sync Response: test\nHello from plugin. data: test | zero_copy: test\nPlugin Async Response: test\n" + }; + assert_eq!(stdout, expected); + assert_eq!(stderr, ""); +} diff --git a/test_plugin/tests/test.js b/test_plugin/tests/test.js new file mode 100644 index 0000000000..5a127d3286 --- /dev/null +++ b/test_plugin/tests/test.js @@ -0,0 +1,47 @@ +const filenameBase = "test_plugin"; + +let filenameSuffix = ".so"; +let filenamePrefix = "lib"; + +if (Deno.build.os === "win") { + filenameSuffix = ".dll"; + filenamePrefix = ""; +} +if (Deno.build.os === "mac") { + filenameSuffix = ".dylib"; +} + +const filename = `../target/${Deno.args[1]}/${filenamePrefix}${filenameBase}${filenameSuffix}`; + +const plugin = Deno.openPlugin(filename); + +const { testSync, testAsync } = plugin.ops; + +const textDecoder = new TextDecoder(); + +function runTestSync() { + const response = testSync.dispatch( + new Uint8Array([116, 101, 115, 116]), + new Uint8Array([116, 101, 115, 116]) + ); + + console.log(`Plugin Sync Response: ${textDecoder.decode(response)}`); +} + +testAsync.setAsyncHandler(response => { + console.log(`Plugin Async Response: ${textDecoder.decode(response)}`); +}); + +function runTestAsync() { + const response = testAsync.dispatch( + new Uint8Array([116, 101, 115, 116]), + new Uint8Array([116, 101, 115, 116]) + ); + + if (response != null || response != undefined) { + throw new Error("Expected null response!"); + } +} + +runTestSync(); +runTestAsync();