From aba952397ae668714add770c5b6fa6edf1cf3eb5 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Sat, 11 May 2019 10:23:19 -0400 Subject: [PATCH] Add progress bar (#2309) --- cli/compiler.rs | 9 ++ cli/deno_dir.rs | 25 +++- cli/flags.rs | 2 - cli/main.rs | 13 +- cli/msg.rs | 1 - cli/ops.rs | 11 +- cli/permissions.rs | 1 - cli/progress.rs | 161 ++++++++++++++++++++++ cli/resources.rs | 1 - cli/state.rs | 19 ++- cli/worker.rs | 53 ++++++- js/compiler.ts | 3 +- tests/config.ts.out | 2 +- tests/error_004_missing_module.ts.out | 3 +- tests/error_006_import_ext_failure.ts.out | 3 +- tests/unbuffered_stderr.ts.out | 2 +- 16 files changed, 268 insertions(+), 41 deletions(-) create mode 100644 cli/progress.rs diff --git a/cli/compiler.rs b/cli/compiler.rs index 522002b0b0..d4913a4e2d 100644 --- a/cli/compiler.rs +++ b/cli/compiler.rs @@ -95,6 +95,7 @@ fn lazy_start(parent_state: ThreadSafeState) -> ResourceId { parent_state.flags.clone(), parent_state.argv.clone(), op_selector_compiler, + parent_state.progress.clone(), ); let rid = child_state.resource.rid; let resource = child_state.resource.clone(); @@ -192,6 +193,10 @@ pub fn compile_async( let compiler_rid = lazy_start(parent_state.clone()); + let compiling_job = parent_state + .progress + .add(format!("Compiling {}", module_meta_data_.module_name)); + let (local_sender, local_receiver) = oneshot::channel::>>(); @@ -218,6 +223,10 @@ pub fn compile_async( let res_data = res["data"].as_object().expect( "Error decoding compiler response: expected object field 'data'", ); + + // Explicit drop to keep reference alive until future completes. + drop(compiling_job); + match res["success"].as_bool() { Some(true) => Ok(ModuleMetaData { maybe_output_code: res_data["outputCode"] diff --git a/cli/deno_dir.rs b/cli/deno_dir.rs index 9d83ad044e..c0ee051b83 100644 --- a/cli/deno_dir.rs +++ b/cli/deno_dir.rs @@ -8,6 +8,7 @@ use crate::fs as deno_fs; use crate::http_util; use crate::js_errors::SourceMapGetter; use crate::msg; +use crate::progress::Progress; use crate::tokio_util; use crate::version; use dirs; @@ -44,6 +45,8 @@ pub struct DenoDir { /// The active configuration file contents (or empty array) which applies to /// source code cached by `DenoDir`. pub config: Vec, + + pub progress: Progress, } impl DenoDir { @@ -52,6 +55,7 @@ impl DenoDir { pub fn new( custom_root: Option, state_config: &Option>, + progress: Progress, ) -> std::io::Result { // Only setup once. let home_dir = dirs::home_dir().expect("Could not get home directory."); @@ -85,6 +89,7 @@ impl DenoDir { deps_http, deps_https, config, + progress, }; // TODO Lazily create these directories. @@ -578,9 +583,10 @@ fn fetch_remote_source_async( filename: &str, ) -> impl Future, Error = DenoError> { use crate::http_util::FetchOnceResult; - { - eprintln!("Downloading {}", module_name); - } + + let download_job = deno_dir + .progress + .add(format!("Downloading {}", module_name)); let filename = filename.to_owned(); let module_name = module_name.to_owned(); @@ -682,7 +688,11 @@ fn fetch_remote_source_async( } }) }, - ) + ).then(move |r| { + // Explicit drop to keep reference alive until future completes. + drop(download_job); + r + }) } /// Fetch remote source code. @@ -912,8 +922,11 @@ mod tests { fn test_setup() -> (TempDir, DenoDir) { let temp_dir = TempDir::new().expect("tempdir fail"); let config = Some(b"{}".to_vec()); - let deno_dir = DenoDir::new(Some(temp_dir.path().to_path_buf()), &config) - .expect("setup fail"); + let deno_dir = DenoDir::new( + Some(temp_dir.path().to_path_buf()), + &config, + Progress::new(), + ).expect("setup fail"); (temp_dir, deno_dir) } diff --git a/cli/flags.rs b/cli/flags.rs index 442d106659..e1ae09ebbb 100644 --- a/cli/flags.rs +++ b/cli/flags.rs @@ -7,7 +7,6 @@ macro_rules! svec { ($($x:expr),*) => (vec![$($x.to_string()),*]); } -#[cfg_attr(feature = "cargo-clippy", allow(stutter))] #[derive(Clone, Debug, PartialEq, Default)] pub struct DenoFlags { pub log_debug: bool, @@ -307,7 +306,6 @@ fn resolve_paths(paths: Vec) -> Vec { /// Parse ArgMatches into internal DenoFlags structure. /// This method should not make any side effects. -#[cfg_attr(feature = "cargo-clippy", allow(stutter))] pub fn parse_flags(matches: ArgMatches) -> DenoFlags { let mut flags = DenoFlags::default(); diff --git a/cli/main.rs b/cli/main.rs index a864c2db31..2423c4936d 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -27,6 +27,7 @@ pub mod msg; pub mod msg_util; pub mod ops; pub mod permissions; +mod progress; mod repl; pub mod resolve_addr; pub mod resources; @@ -39,6 +40,7 @@ pub mod version; pub mod worker; use crate::errors::RustOrJsError; +use crate::progress::Progress; use crate::state::ThreadSafeState; use crate::worker::root_specifier_to_url; use crate::worker::Worker; @@ -134,7 +136,16 @@ fn create_worker_and_state( flags: DenoFlags, argv: Vec, ) -> (Worker, ThreadSafeState) { - let state = ThreadSafeState::new(flags, argv, ops::op_selector_std); + let progress = Progress::new(); + progress.set_callback(|done, completed, total, msg| { + if done { + eprintln!(""); + } else { + eprint!("\r[{}/{}] {}", completed, total, msg); + eprint!("\x1B[K"); // Clear to end of line. + } + }); + let state = ThreadSafeState::new(flags, argv, ops::op_selector_std, progress); let worker = Worker::new( "main".to_string(), startup_data::deno_isolate_init(), diff --git a/cli/msg.rs b/cli/msg.rs index d97940a345..0aaf368e10 100644 --- a/cli/msg.rs +++ b/cli/msg.rs @@ -1,5 +1,4 @@ // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. -#![allow(unused_imports)] #![allow(dead_code)] #![cfg_attr( feature = "cargo-clippy", diff --git a/cli/ops.rs b/cli/ops.rs index 322e41242b..07def46e88 100644 --- a/cli/ops.rs +++ b/cli/ops.rs @@ -859,19 +859,9 @@ fn op_chmod( debug!("op_chmod {}", &path_); // Still check file/dir exists on windows let _metadata = fs::metadata(&path)?; - // Only work in unix #[cfg(any(unix))] { - // We need to use underscore to compile in Windows. - #[cfg_attr( - feature = "cargo-clippy", - allow(clippy::used_underscore_binding) - )] let mut permissions = _metadata.permissions(); - #[cfg_attr( - feature = "cargo-clippy", - allow(clippy::used_underscore_binding) - )] permissions.set_mode(_mode); fs::set_permissions(&path, permissions)?; } @@ -2049,6 +2039,7 @@ fn op_create_worker( parent_state.flags.clone(), parent_state.argv.clone(), op_selector_std, + parent_state.progress.clone(), ); let rid = child_state.resource.rid; let name = format!("USER-WORKER-{}", specifier); diff --git a/cli/permissions.rs b/cli/permissions.rs index 304b6edfef..a02c6bb07a 100644 --- a/cli/permissions.rs +++ b/cli/permissions.rs @@ -124,7 +124,6 @@ impl Default for PermissionAccessor { } } -#[cfg_attr(feature = "cargo-clippy", allow(stutter))] #[derive(Debug, Default)] pub struct DenoPermissions { // Keep in sync with src/permissions.ts diff --git a/cli/progress.rs b/cli/progress.rs new file mode 100644 index 0000000000..a9bfcc37df --- /dev/null +++ b/cli/progress.rs @@ -0,0 +1,161 @@ +// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. +use std::sync::Arc; +use std::sync::Mutex; + +#[derive(Clone, Default)] +pub struct Progress(Arc>); + +impl Progress { + pub fn new() -> Self { + Progress::default() + } + + pub fn set_callback(&self, f: F) + where + F: Fn(bool, usize, usize, &str) + Send + Sync + 'static, + { + let mut s = self.0.lock().unwrap(); + assert!(s.callback.is_none()); + s.callback = Some(Arc::new(f)); + } + + /// Returns job counts: (complete, total) + pub fn progress(&self) -> (usize, usize) { + let s = self.0.lock().unwrap(); + s.progress() + } + + pub fn history(&self) -> Vec { + let s = self.0.lock().unwrap(); + s.job_names.clone() + } + + pub fn add(&self, name: String) -> Job { + let mut s = self.0.lock().unwrap(); + let id = s.job_names.len(); + s.maybe_call_callback(false, s.complete, s.job_names.len() + 1, &name); + s.job_names.push(name); + Job { + id, + inner: self.0.clone(), + } + } + + pub fn done(&self) { + let s = self.0.lock().unwrap(); + s.maybe_call_callback(true, s.complete, s.job_names.len(), ""); + } +} + +type Callback = dyn Fn(bool, usize, usize, &str) + Send + Sync; + +#[derive(Default)] +struct Inner { + job_names: Vec, + complete: usize, + callback: Option>, +} + +impl Inner { + pub fn maybe_call_callback( + &self, + done: bool, + complete: usize, + total: usize, + msg: &str, + ) { + if let Some(ref cb) = self.callback { + cb(done, complete, total, msg); + } + } + + /// Returns job counts: (complete, total) + pub fn progress(&self) -> (usize, usize) { + let total = self.job_names.len(); + (self.complete, total) + } +} + +pub struct Job { + inner: Arc>, + id: usize, +} + +impl Drop for Job { + fn drop(&mut self) { + let mut s = self.inner.lock().unwrap(); + s.complete += 1; + let name = &s.job_names[self.id]; + let (complete, total) = s.progress(); + s.maybe_call_callback(false, complete, total, name); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn progress() { + let p = Progress::new(); + assert_eq!(p.progress(), (0, 0)); + { + let _j1 = p.add("hello".to_string()); + assert_eq!(p.progress(), (0, 1)); + } + assert_eq!(p.progress(), (1, 1)); + { + let _j2 = p.add("hello".to_string()); + assert_eq!(p.progress(), (1, 2)); + } + assert_eq!(p.progress(), (2, 2)); + } + + #[test] + fn history() { + let p = Progress::new(); + let _a = p.add("a".to_string()); + let _b = p.add("b".to_string()); + assert_eq!(p.history(), vec!["a", "b"]); + } + + #[test] + fn callback() { + let callback_history: Arc>> = + Arc::new(Mutex::new(Vec::new())); + { + let p = Progress::new(); + let callback_history_ = callback_history.clone(); + + p.set_callback(move |_done, complete, total, msg| { + // println!("callback: {}, {}, {}", complete, total, msg); + let mut h = callback_history_.lock().unwrap(); + h.push((complete, total, String::from(msg))); + }); + { + let _a = p.add("a".to_string()); + let _b = p.add("b".to_string()); + } + let _c = p.add("c".to_string()); + } + + let h = callback_history.lock().unwrap(); + assert_eq!( + h.to_vec(), + vec![ + (0, 1, "a".to_string()), + (0, 2, "b".to_string()), + (1, 2, "b".to_string()), + (2, 2, "a".to_string()), + (2, 3, "c".to_string()), + (3, 3, "c".to_string()), + ] + ); + } + + #[test] + fn thread_safe() { + fn f(_: S) {} + f(Progress::new()); + } +} diff --git a/cli/resources.rs b/cli/resources.rs index 3a7121d4cc..8473fc6f1a 100644 --- a/cli/resources.rs +++ b/cli/resources.rs @@ -374,7 +374,6 @@ pub fn get_message_stream_from_worker(rid: ResourceId) -> WorkerReceiverStream { WorkerReceiverStream { rid } } -#[cfg_attr(feature = "cargo-clippy", allow(stutter))] pub struct ChildResources { pub child_rid: ResourceId, pub stdin_rid: Option, diff --git a/cli/state.rs b/cli/state.rs index f27aa95a4d..2254f8e253 100644 --- a/cli/state.rs +++ b/cli/state.rs @@ -5,6 +5,7 @@ use crate::flags; use crate::global_timer::GlobalTimer; use crate::ops; use crate::permissions::DenoPermissions; +use crate::progress::Progress; use crate::resources; use crate::resources::ResourceId; use crate::worker::Worker; @@ -28,7 +29,6 @@ pub type WorkerReceiver = async_mpsc::Receiver; pub type WorkerChannels = (WorkerSender, WorkerReceiver); pub type UserWorkerTable = HashMap>; -// AtomicU64 is currently unstable #[derive(Default)] pub struct Metrics { pub ops_dispatched: AtomicUsize, @@ -39,13 +39,11 @@ pub struct Metrics { pub resolve_count: AtomicUsize, } -// Wrap State so that it can implement Dispatch. +/// Isolate cannot be passed between threads but ThreadSafeState can. +/// ThreadSafeState satisfies Send and Sync. So any state that needs to be +/// accessed outside the main V8 thread should be inside ThreadSafeState. pub struct ThreadSafeState(Arc); -// Isolate cannot be passed between threads but ThreadSafeState can. -// ThreadSafeState satisfies Send and Sync. -// So any state that needs to be accessed outside the main V8 thread should be -// inside ThreadSafeState. #[cfg_attr(feature = "cargo-clippy", allow(stutter))] pub struct State { pub dir: deno_dir::DenoDir, @@ -63,8 +61,11 @@ pub struct State { pub global_timer: Mutex, pub workers: Mutex, pub start_time: Instant, + /// A reference to this worker's resource. pub resource: resources::Resource, pub dispatch_selector: ops::OpSelector, + /// Reference to global progress bar. + pub progress: Progress, } impl Clone for ThreadSafeState { @@ -91,6 +92,7 @@ impl ThreadSafeState { flags: flags::DenoFlags, argv_rest: Vec, dispatch_selector: ops::OpSelector, + progress: Progress, ) -> Self { let custom_root = env::var("DENO_DIR").map(String::into).ok(); @@ -140,7 +142,8 @@ impl ThreadSafeState { }; ThreadSafeState(Arc::new(State { - dir: deno_dir::DenoDir::new(custom_root, &config).unwrap(), + dir: deno_dir::DenoDir::new(custom_root, &config, progress.clone()) + .unwrap(), argv: argv_rest, permissions: DenoPermissions::from_flags(&flags), flags, @@ -153,6 +156,7 @@ impl ThreadSafeState { start_time: Instant::now(), resource, dispatch_selector, + progress, })) } @@ -210,6 +214,7 @@ impl ThreadSafeState { flags::DenoFlags::default(), argv, ops::op_selector_std, + Progress::new(), ) } diff --git a/cli/worker.rs b/cli/worker.rs index 317b104a0f..98bea6eb83 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -69,6 +69,7 @@ impl Worker { let recursive_load = deno::RecursiveLoad::new(js_url.as_str(), self); recursive_load.and_then( move |(id, mut self_)| -> Result, Self)> { + self_.state.progress.done(); if is_prefetch { Ok(self_) } else { @@ -82,6 +83,7 @@ impl Worker { }, ) .map_err(|(err, self_)| { + self_.state.progress.done(); // Convert to RustOrJsError AND apply_source_map. let err = match err { deno::JSErrorOr::JSError(err) => RustOrJsError::Js(self_.apply_source_map(err)), @@ -256,6 +258,7 @@ mod tests { use super::*; use crate::flags; use crate::ops::op_selector_std; + use crate::progress::Progress; use crate::resources; use crate::startup_data; use crate::state::ThreadSafeState; @@ -272,8 +275,12 @@ mod tests { let js_url = Url::from_file_path(filename).unwrap(); let argv = vec![String::from("./deno"), js_url.to_string()]; - let state = - ThreadSafeState::new(flags::DenoFlags::default(), argv, op_selector_std); + let state = ThreadSafeState::new( + flags::DenoFlags::default(), + argv, + op_selector_std, + Progress::new(), + ); let state_ = state.clone(); tokio_util::run(lazy(move || { let worker = Worker::new("TEST".to_string(), StartupData::None, state); @@ -298,8 +305,12 @@ mod tests { let js_url = Url::from_file_path(filename).unwrap(); let argv = vec![String::from("./deno"), js_url.to_string()]; - let state = - ThreadSafeState::new(flags::DenoFlags::default(), argv, op_selector_std); + let state = ThreadSafeState::new( + flags::DenoFlags::default(), + argv, + op_selector_std, + Progress::new(), + ); let state_ = state.clone(); tokio_util::run(lazy(move || { let worker = Worker::new("TEST".to_string(), StartupData::None, state); @@ -318,6 +329,40 @@ mod tests { assert_eq!(metrics.resolve_count.load(Ordering::SeqCst), 2); } + #[test] + fn execute_006_url_imports() { + let filename = std::env::current_dir() + .unwrap() + .join("tests/006_url_imports.ts"); + let js_url = Url::from_file_path(filename).unwrap(); + let argv = vec![String::from("deno"), js_url.to_string()]; + let mut flags = flags::DenoFlags::default(); + flags.reload = true; + let state = + ThreadSafeState::new(flags, argv, op_selector_std, Progress::new()); + let state_ = state.clone(); + tokio_util::run(lazy(move || { + let mut worker = Worker::new( + "TEST".to_string(), + startup_data::deno_isolate_init(), + state, + ); + js_check(worker.execute("denoMain()")); + let result = worker.execute_mod(&js_url, false); + let worker = match result { + Err((err, worker)) => { + eprintln!("execute_mod err {:?}", err); + worker + } + Ok(worker) => worker, + }; + tokio_util::panic_on_error(worker) + })); + + let metrics = &state_.metrics; + assert_eq!(metrics.resolve_count.load(Ordering::SeqCst), 3); + } + fn create_test_worker() -> Worker { let state = ThreadSafeState::mock(); let mut worker = diff --git a/js/compiler.ts b/js/compiler.ts index 5711e103a9..d830dd7113 100644 --- a/js/compiler.ts +++ b/js/compiler.ts @@ -400,9 +400,8 @@ class Compiler implements ts.LanguageServiceHost, ts.FormatDiagnosticsHost { ): { outputCode: OutputCode; sourceMap: SourceMap } { this._log("compiler.compile", { moduleSpecifier, containingFile }); const moduleMetaData = this._resolveModule(moduleSpecifier, containingFile); - const { fileName, mediaType, moduleId, sourceCode } = moduleMetaData; + const { fileName, mediaType, sourceCode } = moduleMetaData; this._scriptFileNames = [fileName]; - console.warn("Compiling", moduleId); let outputCode: string; let sourceMap = ""; // Instead of using TypeScript to transpile JSON modules, we will just do diff --git a/tests/config.ts.out b/tests/config.ts.out index e3ceb52bc8..0f79e03d12 100644 --- a/tests/config.ts.out +++ b/tests/config.ts.out @@ -1,7 +1,7 @@ Unsupported compiler options in "[WILDCARD]tests/config.tsconfig.json" The following options were ignored: module, target -Compiling file://[WILDCARD]tests/config.ts + [WILDCARD]tests/config.ts:3:5 - error TS2532: Object is possibly 'undefined'. 3 if (map.get("bar").foo) { diff --git a/tests/error_004_missing_module.ts.out b/tests/error_004_missing_module.ts.out index ecc8c31869..f6fbf5d9b8 100644 --- a/tests/error_004_missing_module.ts.out +++ b/tests/error_004_missing_module.ts.out @@ -1,5 +1,4 @@ -Compiling [WILDCARD]tests/error_004_missing_module.ts -Uncaught NotFound: Cannot resolve module "bad-module.ts" from "[WILDCARD]/tests/error_004_missing_module.ts" +[WILDCARD]Uncaught NotFound: Cannot resolve module "bad-module.ts" from "[WILDCARD]/tests/error_004_missing_module.ts" at DenoError (js/errors.ts:[WILDCARD]) at maybeError (js/errors.ts:[WILDCARD]) at maybeThrowError (js/errors.ts:[WILDCARD]) diff --git a/tests/error_006_import_ext_failure.ts.out b/tests/error_006_import_ext_failure.ts.out index 9c17d19258..7fe154aa0b 100644 --- a/tests/error_006_import_ext_failure.ts.out +++ b/tests/error_006_import_ext_failure.ts.out @@ -1,5 +1,4 @@ -Compiling [WILDCARD]tests/error_006_import_ext_failure.ts -Uncaught NotFound: Cannot resolve module "./non-existent" from "[WILDCARD]/tests/error_006_import_ext_failure.ts" +[WILDCARD]Uncaught NotFound: Cannot resolve module "./non-existent" from "[WILDCARD]/tests/error_006_import_ext_failure.ts" at DenoError (js/errors.ts:[WILDCARD]) at maybeError (js/errors.ts:[WILDCARD]) at maybeThrowError (js/errors.ts:[WILDCARD]) diff --git a/tests/unbuffered_stderr.ts.out b/tests/unbuffered_stderr.ts.out index 907698acc5..5000197382 100644 --- a/tests/unbuffered_stderr.ts.out +++ b/tests/unbuffered_stderr.ts.out @@ -1,2 +1,2 @@ -Compiling [WILDCARD].ts +[WILDCARD] x \ No newline at end of file