From 31f32ed8c4082d36ad2a4ea460366c0d57a5ddbc Mon Sep 17 00:00:00 2001 From: Valentin Anger Date: Fri, 28 Aug 2020 15:03:50 +0200 Subject: [PATCH] Move benchmarks to Rust (#7134) All benchmarks are done in Rust and can be invoked with `cargo bench`. Currently this has it's own "harness" that behaves like `./tools/benchmark.py` did. Because of this tests inside `cli/bench` are currently not run. This should be switched to the language provided harness once the `#[bench]` attribute has been stabilized. --- .eslintignore | 1 + .github/workflows/ci.yml | 2 +- Cargo.lock | 12 + Cargo.toml | 5 + cli/Cargo.toml | 7 + {tools => cli/bench}/deno_http_proxy.ts | 0 {tools => cli/bench}/deno_tcp.ts | 2 +- {tools => cli/bench}/deno_tcp_proxy.ts | 2 +- cli/bench/http.rs | 303 ++++++++++++ cli/bench/main.rs | 462 ++++++++++++++++++ {tools => cli/bench}/node_http.js | 0 {tools => cli/bench}/node_http_proxy.js | 0 {tools => cli/bench}/node_tcp.js | 0 {tools => cli/bench}/node_tcp_promise.js | 0 {tools => cli/bench}/node_tcp_proxy.js | 0 cli/bench/throughput.rs | 62 +++ cli/tests/integration_tests.rs | 5 - test_util/src/lib.rs | 378 ++++++++++++-- .../src}/testdata/strace_summary.out | 0 .../src}/testdata/strace_summary2.out | 0 {tools => test_util/src}/testdata/time.out | 0 {tools => test_util/src}/testdata/wrk1.txt | 0 {tools => test_util/src}/testdata/wrk2.txt | 0 {tools => test_util/src}/testdata/wrk3.txt | 0 tools/benchmark.py | 277 ----------- tools/benchmark_test.py | 66 --- tools/http_benchmark.py | 215 -------- tools/lint.py | 1 + tools/throughput_benchmark.py | 64 --- tools/util.py | 14 - tools/util_test.py | 19 +- 31 files changed, 1196 insertions(+), 701 deletions(-) rename {tools => cli/bench}/deno_http_proxy.ts (100%) rename {tools => cli/bench}/deno_tcp.ts (91%) rename {tools => cli/bench}/deno_tcp_proxy.ts (90%) create mode 100644 cli/bench/http.rs create mode 100644 cli/bench/main.rs rename {tools => cli/bench}/node_http.js (100%) rename {tools => cli/bench}/node_http_proxy.js (100%) rename {tools => cli/bench}/node_tcp.js (100%) rename {tools => cli/bench}/node_tcp_promise.js (100%) rename {tools => cli/bench}/node_tcp_proxy.js (100%) create mode 100644 cli/bench/throughput.rs rename {tools => test_util/src}/testdata/strace_summary.out (100%) rename {tools => test_util/src}/testdata/strace_summary2.out (100%) rename {tools => test_util/src}/testdata/time.out (100%) rename {tools => test_util/src}/testdata/wrk1.txt (100%) rename {tools => test_util/src}/testdata/wrk2.txt (100%) rename {tools => test_util/src}/testdata/wrk3.txt (100%) delete mode 100755 tools/benchmark.py delete mode 100755 tools/benchmark_test.py delete mode 100755 tools/http_benchmark.py delete mode 100755 tools/throughput_benchmark.py diff --git a/.eslintignore b/.eslintignore index a1d26dbe9e..a07adeb942 100644 --- a/.eslintignore +++ b/.eslintignore @@ -5,3 +5,4 @@ std/**/testdata/ std/**/node_modules/ cli/tsc/*typescript.js cli/dts/* +cli/bench/node*.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09a0b443d4..27005a0946 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -154,7 +154,7 @@ jobs: - name: Run Benchmarks if: matrix.config.kind == 'bench' - run: python ./tools/benchmark.py --release + run: cargo bench - name: Post Benchmarks if: | diff --git a/Cargo.lock b/Cargo.lock index 3e5e957ee6..085a2a95cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -230,6 +230,17 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +[[package]] +name = "chrono" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c74d84029116787153e02106bf53e66828452a4b325cc8652b788b5967c0a0b6" +dependencies = [ + "num-integer", + "num-traits", + "time", +] + [[package]] name = "clap" version = "2.33.2" @@ -327,6 +338,7 @@ dependencies = [ "base64 0.12.3", "byteorder", "bytes", + "chrono", "clap", "deno_core", "deno_doc", diff --git a/Cargo.toml b/Cargo.toml index 80e6ce4e6a..91e591b734 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,3 +15,8 @@ exclude = [ codegen-units = 1 lto = true opt-level = 'z' # Optimize for size + +[profile.bench] +codegen-units = 1 +lto = true +opt-level = 'z' # Optimize for size diff --git a/cli/Cargo.toml b/cli/Cargo.toml index d7149b79c2..751a7c461c 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -14,6 +14,11 @@ default-run = "deno" name = "deno" path = "main.rs" +[[bench]] +name = "deno_bench" +harness = false +path = "./bench/main.rs" + [build-dependencies] deno_core = { path = "../core", version = "0.54.0" } deno_web = { path = "../op_crates/web", version = "0.4.0" } @@ -78,6 +83,8 @@ fwdansi = "1.1.0" nix = "0.17.0" [dev-dependencies] +# Used in benchmark +chrono = "0.4" os_pipe = "0.9.2" # Used for testing inspector. Keep in-sync with warp. tokio-tungstenite = { version = "0.10.1", features = ["connect"] } diff --git a/tools/deno_http_proxy.ts b/cli/bench/deno_http_proxy.ts similarity index 100% rename from tools/deno_http_proxy.ts rename to cli/bench/deno_http_proxy.ts diff --git a/tools/deno_tcp.ts b/cli/bench/deno_tcp.ts similarity index 91% rename from tools/deno_tcp.ts rename to cli/bench/deno_tcp.ts index 8988697686..a204e7bfb4 100644 --- a/tools/deno_tcp.ts +++ b/cli/bench/deno_tcp.ts @@ -1,4 +1,4 @@ -// Used for benchmarking Deno's networking. See tools/http_benchmark.py +// Used for benchmarking Deno's networking. // TODO Replace this with a real HTTP server once // https://github.com/denoland/deno/issues/726 is completed. // Note: this is a keep-alive server. diff --git a/tools/deno_tcp_proxy.ts b/cli/bench/deno_tcp_proxy.ts similarity index 90% rename from tools/deno_tcp_proxy.ts rename to cli/bench/deno_tcp_proxy.ts index a9832e363c..db693b6901 100644 --- a/tools/deno_tcp_proxy.ts +++ b/cli/bench/deno_tcp_proxy.ts @@ -1,4 +1,4 @@ -// Used for benchmarking Deno's tcp proxy performance. See tools/http_benchmark.py +// Used for benchmarking Deno's tcp proxy performance. const addr = Deno.args[0] || "127.0.0.1:4500"; const originAddr = Deno.args[1] || "127.0.0.1:4501"; diff --git a/cli/bench/http.rs b/cli/bench/http.rs new file mode 100644 index 0000000000..f9acf20da0 --- /dev/null +++ b/cli/bench/http.rs @@ -0,0 +1,303 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +use super::Result; +use std::{ + collections::HashMap, path::PathBuf, process::Command, time::Duration, +}; +pub use test_util::{parse_wrk_output, WrkOutput as HttpBenchmarkResult}; + +// Some of the benchmarks in this file have been renamed. In case the history +// somehow gets messed up: +// "node_http" was once called "node" +// "deno_tcp" was once called "deno" +// "deno_http" was once called "deno_net_http" + +const DURATION: &str = "20s"; + +pub(crate) fn benchmark( + target_path: &PathBuf, +) -> Result> { + let deno_exe = test_util::deno_exe_path(); + let deno_exe = deno_exe.to_str().unwrap(); + + let hyper_hello_exe = target_path.join("test_server"); + let hyper_hello_exe = hyper_hello_exe.to_str().unwrap(); + + let core_http_bin_ops_exe = target_path.join("examples/http_bench_bin_ops"); + let core_http_bin_ops_exe = core_http_bin_ops_exe.to_str().unwrap(); + + let core_http_json_ops_exe = target_path.join("examples/http_bench_json_ops"); + let core_http_json_ops_exe = core_http_json_ops_exe.to_str().unwrap(); + + let mut res = HashMap::new(); + + // "deno_tcp" was once called "deno" + res.insert("deno_tcp".to_string(), deno_tcp(deno_exe)?); + // res.insert("deno_udp".to_string(), deno_udp(deno_exe)?); + res.insert("deno_http".to_string(), deno_http(deno_exe)?); + // TODO(ry) deno_proxy disabled to make fetch() standards compliant. + // res.insert("deno_proxy".to_string(), deno_http_proxy(deno_exe) hyper_hello_exe)) + res.insert( + "deno_proxy_tcp".to_string(), + deno_tcp_proxy(deno_exe, hyper_hello_exe)?, + ); + // "core_http_bin_ops" was once called "deno_core_single" + // "core_http_bin_ops" was once called "deno_core_http_bench" + res.insert( + "core_http_bin_ops".to_string(), + core_http_bin_ops(core_http_bin_ops_exe)?, + ); + res.insert( + "core_http_json_ops".to_string(), + core_http_json_ops(core_http_json_ops_exe)?, + ); + // "node_http" was once called "node" + res.insert("node_http".to_string(), node_http()?); + res.insert("node_proxy".to_string(), node_http_proxy(hyper_hello_exe)?); + res.insert( + "node_proxy_tcp".to_string(), + node_tcp_proxy(hyper_hello_exe)?, + ); + res.insert("node_tcp".to_string(), node_tcp()?); + res.insert("hyper".to_string(), hyper_http(hyper_hello_exe)?); + + Ok(res) +} + +fn run( + server_cmd: &[&str], + port: u16, + env: Option>, + origin_cmd: Option<&[&str]>, +) -> Result { + // Wait for port 4544 to become available. + // TODO Need to use SO_REUSEPORT with tokio::net::TcpListener. + std::thread::sleep(Duration::from_secs(5)); + + let mut origin = None; + if let Some(cmd) = origin_cmd { + let mut com = Command::new(cmd[0]); + com.args(&cmd[1..]); + if let Some(env) = env.clone() { + com.envs(env); + } + origin = Some(com.spawn()?); + }; + + println!("{}", server_cmd.join(" ")); + let mut server = { + let mut com = Command::new(server_cmd[0]); + com.args(&server_cmd[1..]); + if let Some(env) = env { + com.envs(env); + } + com.spawn()? + }; + + std::thread::sleep(Duration::from_secs(5)); // wait for server to wake up. TODO racy. + + let wrk = test_util::prebuilt_tool_path("wrk"); + assert!(wrk.is_file()); + + let wrk_cmd = &[ + wrk.to_str().unwrap(), + "-d", + DURATION, + "--latency", + &format!("http://127.0.0.1:{}/", port), + ]; + println!("{}", wrk_cmd.join(" ")); + let output = test_util::run_collect(wrk_cmd, None, None, None, true).0; + + println!("{}", output); + assert!( + server.try_wait()?.map_or(true, |s| s.success()), + "server ended with error" + ); + + server.kill()?; + if let Some(mut origin) = origin { + origin.kill()?; + } + + Ok(parse_wrk_output(&output)) +} + +fn get_port() -> u16 { + static mut NEXT_PORT: u16 = 4544; + + let port = unsafe { NEXT_PORT }; + + unsafe { + NEXT_PORT += 1; + } + + port +} + +fn server_addr(port: u16) -> String { + format!("0.0.0.0:{}", port) +} + +fn deno_tcp(deno_exe: &str) -> Result { + let port = get_port(); + println!("http_benchmark testing DENO tcp."); + run( + &[ + deno_exe, + "run", + "--allow-net", + "cli/bench/deno_tcp.ts", + &server_addr(port), + ], + port, + None, + None, + ) +} + +fn deno_tcp_proxy( + deno_exe: &str, + hyper_exe: &str, +) -> Result { + let port = get_port(); + let origin_port = get_port(); + + println!("http_proxy_benchmark testing DENO using net/tcp."); + run( + &[ + deno_exe, + "run", + "--allow-net", + "--reload", + "--unstable", + "cli/bench/deno_tcp_proxy.ts", + &server_addr(port), + &server_addr(origin_port), + ], + port, + None, + Some(&[hyper_exe, &origin_port.to_string()]), + ) +} + +fn deno_http(deno_exe: &str) -> Result { + let port = get_port(); + println!("http_benchmark testing DENO using net/http."); + run( + &[ + deno_exe, + "run", + "--allow-net", + "--reload", + "--unstable", + "std/http/http_bench.ts", + &server_addr(port), + ], + port, + None, + None, + ) +} + +#[allow(dead_code)] +fn deno_http_proxy( + deno_exe: &str, + hyper_exe: &str, +) -> Result { + let port = get_port(); + let origin_port = get_port(); + + println!("http_proxy_benchmark testing DENO using net/http."); + run( + &[ + deno_exe, + "run", + "--allow-net", + "--reload", + "--unstable", + "cli/bench/deno_http_proxy.ts", + &server_addr(port), + &server_addr(origin_port), + ], + port, + None, + Some(&[hyper_exe, &origin_port.to_string()]), + ) +} + +fn core_http_bin_ops(exe: &str) -> Result { + println!("http_benchmark testing CORE http_bench_bin_ops"); + run(&[exe], 4544, None, None) +} + +fn core_http_json_ops(exe: &str) -> Result { + println!("http_benchmark testing CORE http_bench_json_ops"); + run(&[exe], 4544, None, None) +} + +fn node_http() -> Result { + let port = get_port(); + println!("http_benchmark testing NODE."); + run( + &["node", "cli/bench/node_http.js", &port.to_string()], + port, + None, + None, + ) +} + +fn node_http_proxy(hyper_exe: &str) -> Result { + let port = get_port(); + let origin_port = get_port(); + let origin_port = origin_port.to_string(); + + println!("http_proxy_benchmark testing NODE."); + run( + &[ + "node", + "cli/bench/node_http_proxy.js", + &port.to_string(), + &origin_port, + ], + port, + None, + Some(&[hyper_exe, &origin_port]), + ) +} + +fn node_tcp_proxy(exe: &str) -> Result { + let port = get_port(); + let origin_port = get_port(); + let origin_port = origin_port.to_string(); + + println!("http_proxy_benchmark testing NODE tcp."); + run( + &[ + "node", + "cli/bench/node_tcp_proxy.js", + &port.to_string(), + &origin_port, + ], + port, + None, + Some(&[exe, &origin_port]), + ) +} + +fn node_tcp() -> Result { + let port = get_port(); + println!("http_benchmark testing node_tcp.js"); + run( + &["node", "cli/bench/node_tcp.js", &port.to_string()], + port, + None, + None, + ) +} + +fn hyper_http(exe: &str) -> Result { + let port = get_port(); + println!("http_benchmark testing RUST hyper"); + run(&[exe, &format!("{}", port)], port, None, None) +} diff --git a/cli/bench/main.rs b/cli/bench/main.rs new file mode 100644 index 0000000000..b2ae59d4f4 --- /dev/null +++ b/cli/bench/main.rs @@ -0,0 +1,462 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +use serde_json::{self, map::Map, Number, Value}; +use std::{ + convert::From, + env, fs, + path::PathBuf, + process::{Command, Stdio}, +}; + +mod http; +mod throughput; + +fn read_json(filename: &str) -> Result { + let f = fs::File::open(filename)?; + Ok(serde_json::from_reader(f)?) +} + +fn write_json(filename: &str, value: &serde_json::Value) -> Result<()> { + let f = fs::File::create(filename)?; + serde_json::to_writer(f, value)?; + Ok(()) +} + +/// The list of the tuples of the benchmark name, arguments and return code +const EXEC_TIME_BENCHMARKS: &[(&str, &[&str], Option)] = &[ + ("hello", &["run", "cli/tests/002_hello.ts"], None), + ( + "relative_import", + &["run", "cli/tests/003_relative_import.ts"], + None, + ), + ("error_001", &["run", "cli/tests/error_001.ts"], Some(1)), + ( + "cold_hello", + &["run", "--reload", "cli/tests/002_hello.ts"], + None, + ), + ( + "cold_relative_import", + &["run", "--reload", "cli/tests/003_relative_import.ts"], + None, + ), + ( + "workers_startup", + &["run", "--allow-read", "cli/tests/workers_startup_bench.ts"], + None, + ), + ( + "workers_round_robin", + &[ + "run", + "--allow-read", + "cli/tests/workers_round_robin_bench.ts", + ], + None, + ), + ( + "text_decoder", + &["run", "cli/tests/text_decoder_perf.js"], + None, + ), + ( + "text_encoder", + &["run", "cli/tests/text_encoder_perf.js"], + None, + ), + ( + "check", + &["cache", "--reload", "std/examples/chat/server_test.ts"], + None, + ), + ( + "no_check", + &[ + "cache", + "--reload", + "--no-check", + "std/examples/chat/server_test.ts", + ], + None, + ), +]; + +const RESULT_KEYS: &[&str] = + &["mean", "stddev", "user", "system", "min", "max"]; +fn run_exec_time(deno_exe: &PathBuf, target_dir: &PathBuf) -> Result { + let hyperfine_exe = test_util::prebuilt_tool_path("hyperfine"); + + let benchmark_file = target_dir.join("hyperfine_results.json"); + let benchmark_file = benchmark_file.to_str().unwrap(); + + let mut command = [ + hyperfine_exe.to_str().unwrap(), + "--export-json", + benchmark_file, + "--warmup", + "3", + ] + .iter() + .map(|s| s.to_string()) + .collect::>(); + + for (_, args, return_code) in EXEC_TIME_BENCHMARKS { + let ret_code_test = if let Some(code) = return_code { + // Bash test which asserts the return code value of the previous command + // $? contains the return code of the previous command + format!("; test $? -eq {}", code) + } else { + "".to_string() + }; + command.push(format!( + "{} {} {}", + deno_exe.to_str().unwrap(), + args.join(" "), + ret_code_test + )); + } + + test_util::run( + &command.iter().map(|s| s.as_ref()).collect::>(), + None, + None, + None, + true, + ); + + let mut results = Map::new(); + let hyperfine_results = read_json(benchmark_file)?; + for ((name, _, _), data) in EXEC_TIME_BENCHMARKS.iter().zip( + hyperfine_results + .as_object() + .unwrap() + .get("results") + .unwrap() + .as_array() + .unwrap(), + ) { + let data = data.as_object().unwrap().clone(); + results.insert( + name.to_string(), + Value::Object( + data + .into_iter() + .filter(|(key, _)| RESULT_KEYS.contains(&key.as_str())) + .collect::>(), + ), + ); + } + + Ok(Value::Object(results)) +} + +const BINARY_TARGET_FILES: &[&str] = + &["CLI_SNAPSHOT.bin", "COMPILER_SNAPSHOT.bin"]; +fn get_binary_sizes(target_dir: &PathBuf) -> Result { + let mut sizes = Map::new(); + let mut mtimes = std::collections::HashMap::new(); + + sizes.insert( + "deno".to_string(), + Value::Number(Number::from(test_util::deno_exe_path().metadata()?.len())), + ); + + // Because cargo's OUT_DIR is not predictable, search the build tree for + // snapshot related files. + for file in walkdir::WalkDir::new(target_dir) { + if file.is_err() { + continue; + } + let file = file.unwrap(); + let filename = file.file_name().to_str().unwrap().to_string(); + + if !BINARY_TARGET_FILES.contains(&filename.as_str()) { + continue; + } + + let meta = file.metadata()?; + let file_mtime = meta.modified()?; + + // If multiple copies of a file are found, use the most recent one. + if let Some(stored_mtime) = mtimes.get(&filename) { + if *stored_mtime > file_mtime { + continue; + } + } + + mtimes.insert(filename.clone(), file_mtime); + sizes.insert(filename, Value::Number(Number::from(meta.len()))); + } + + Ok(Value::Object(sizes)) +} + +const BUNDLES: &[(&str, &str)] = &[ + ("file_server", "./std/http/file_server.ts"), + ("gist", "./std/examples/gist.ts"), +]; +fn bundle_benchmark(deno_exe: &PathBuf) -> Result { + let mut sizes = Map::new(); + + for (name, url) in BUNDLES { + let path = format!("{}.bundle.js", name); + test_util::run( + &[ + deno_exe.to_str().unwrap(), + "bundle", + "--unstable", + url, + &path, + ], + None, + None, + None, + true, + ); + + let file = PathBuf::from(path); + assert!(file.is_file()); + sizes.insert( + name.to_string(), + Value::Number(Number::from(file.metadata()?.len())), + ); + let _ = fs::remove_file(file); + } + + Ok(Value::Object(sizes)) +} + +fn run_throughput(deno_exe: &PathBuf) -> Result { + let mut m = Map::new(); + + m.insert("100M_tcp".to_string(), throughput::tcp(deno_exe, 100)?); + m.insert("100M_cat".to_string(), throughput::cat(deno_exe, 100)?); + m.insert("10M_tcp".to_string(), throughput::tcp(deno_exe, 10)?); + m.insert("10M_cat".to_string(), throughput::cat(deno_exe, 10)?); + + Ok(Value::Object(m)) +} + +fn run_http( + target_dir: &PathBuf, + new_data: &mut Map, +) -> Result<()> { + let stats = http::benchmark(target_dir)?; + + new_data.insert( + "req_per_sec".to_string(), + Value::Object( + stats + .iter() + .map(|(name, result)| { + (name.clone(), Value::Number(Number::from(result.requests))) + }) + .collect::>(), + ), + ); + + new_data.insert( + "max_latency".to_string(), + Value::Object( + stats + .iter() + .map(|(name, result)| { + ( + name.clone(), + Value::Number(Number::from_f64(result.latency).unwrap()), + ) + }) + .collect::>(), + ), + ); + + Ok(()) +} + +fn run_strace_benchmarks( + deno_exe: &PathBuf, + new_data: &mut Map, +) -> Result<()> { + use std::io::Read; + + let mut thread_count = Map::new(); + let mut syscall_count = Map::new(); + + for (name, args, _) in EXEC_TIME_BENCHMARKS { + let mut file = tempfile::NamedTempFile::new()?; + + Command::new("strace") + .args(&[ + "-c", + "-f", + "-o", + file.path().to_str().unwrap(), + deno_exe.to_str().unwrap(), + ]) + .args(args.iter()) + .stdout(Stdio::null()) + .spawn()? + .wait()?; + + let mut output = String::new(); + file.as_file_mut().read_to_string(&mut output)?; + + let strace_result = test_util::parse_strace_output(&output); + thread_count.insert( + name.to_string(), + Value::Number(Number::from( + strace_result.get("clone").unwrap().calls + 1, + )), + ); + syscall_count.insert( + name.to_string(), + Value::Number(Number::from(strace_result.get("total").unwrap().calls)), + ); + } + + new_data.insert("thread_count".to_string(), Value::Object(thread_count)); + new_data.insert("syscall_count".to_string(), Value::Object(syscall_count)); + + Ok(()) +} + +fn run_max_mem_benchmark(deno_exe: &PathBuf) -> Result { + let mut results = Map::new(); + + for (name, args, return_code) in EXEC_TIME_BENCHMARKS { + let proc = Command::new("time") + .args(&["-v", deno_exe.to_str().unwrap()]) + .args(args.iter()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn()?; + + let proc_result = proc.wait_with_output()?; + if let Some(code) = return_code { + assert_eq!(proc_result.status.code().unwrap(), *code); + } + let out = String::from_utf8(proc_result.stderr)?; + + results.insert( + name.to_string(), + Value::Number(Number::from(test_util::parse_max_mem(&out).unwrap())), + ); + } + + Ok(Value::Object(results)) +} + +/* + TODO(SyrupThinker) + Switch to the #[bench] attribute once + it is stabilized. + Before that the #[test] tests won't be run because + we replace the harness with our own runner here. +*/ +fn main() -> Result<()> { + if env::args().find(|s| s == "--bench").is_none() { + return Ok(()); + } + + println!("Starting Deno benchmark"); + + let target_dir = test_util::target_dir(); + let deno_exe = test_util::deno_exe_path(); + + env::set_current_dir(&test_util::root_path())?; + + let mut new_data: Map = Map::new(); + + new_data.insert( + "created_at".to_string(), + Value::String( + chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true), + ), + ); + new_data.insert( + "sha1".to_string(), + Value::String( + test_util::run_collect( + &["git", "rev-parse", "HEAD"], + None, + None, + None, + true, + ) + .0 + .trim() + .to_string(), + ), + ); + + // TODO(ry) The "benchmark" benchmark should actually be called "exec_time". + // When this is changed, the historical data in gh-pages branch needs to be + // changed too. + new_data.insert( + "benchmark".to_string(), + run_exec_time(&deno_exe, &target_dir)?, + ); + + new_data.insert("binary_size".to_string(), get_binary_sizes(&target_dir)?); + new_data.insert("bundle_size".to_string(), bundle_benchmark(&deno_exe)?); + + // Cannot run throughput benchmark on windows because they don't have nc or + // pipe. + if cfg!(not(target_os = "windows")) { + new_data.insert("throughput".to_string(), run_throughput(&deno_exe)?); + run_http(&target_dir, &mut new_data)?; + } + + if cfg!(target_os = "linux") { + run_strace_benchmarks(&deno_exe, &mut new_data)?; + new_data + .insert("max_memory".to_string(), run_max_mem_benchmark(&deno_exe)?); + } + + println!("===== "); + serde_json::to_writer_pretty(std::io::stdout(), &new_data)?; + println!("\n===== "); + + if let Some(filename) = target_dir.join("bench.json").to_str() { + write_json(filename, &Value::Object(new_data))?; + } else { + eprintln!("Cannot write bench.json, path is invalid"); + } + + Ok(()) +} + +#[derive(Debug)] +enum Error { + Io(std::io::Error), + Serde(serde_json::error::Error), + FromUtf8(std::string::FromUtf8Error), + Walkdir(walkdir::Error), +} + +impl From for Error { + fn from(ioe: std::io::Error) -> Self { + Error::Io(ioe) + } +} + +impl From for Error { + fn from(sje: serde_json::error::Error) -> Self { + Error::Serde(sje) + } +} + +impl From for Error { + fn from(fue: std::string::FromUtf8Error) -> Self { + Error::FromUtf8(fue) + } +} + +impl From for Error { + fn from(wde: walkdir::Error) -> Self { + Error::Walkdir(wde) + } +} + +pub(crate) type Result = std::result::Result; diff --git a/tools/node_http.js b/cli/bench/node_http.js similarity index 100% rename from tools/node_http.js rename to cli/bench/node_http.js diff --git a/tools/node_http_proxy.js b/cli/bench/node_http_proxy.js similarity index 100% rename from tools/node_http_proxy.js rename to cli/bench/node_http_proxy.js diff --git a/tools/node_tcp.js b/cli/bench/node_tcp.js similarity index 100% rename from tools/node_tcp.js rename to cli/bench/node_tcp.js diff --git a/tools/node_tcp_promise.js b/cli/bench/node_tcp_promise.js similarity index 100% rename from tools/node_tcp_promise.js rename to cli/bench/node_tcp_promise.js diff --git a/tools/node_tcp_proxy.js b/cli/bench/node_tcp_proxy.js similarity index 100% rename from tools/node_tcp_proxy.js rename to cli/bench/node_tcp_proxy.js diff --git a/cli/bench/throughput.rs b/cli/bench/throughput.rs new file mode 100644 index 0000000000..0be46f142f --- /dev/null +++ b/cli/bench/throughput.rs @@ -0,0 +1,62 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +use super::Result; +use serde_json::{Number, Value}; +use std::{ + path::PathBuf, + process::Command, + time::{Duration, Instant}, +}; + +const MB: usize = 1024 * 1024; +const SERVER_ADDR: &str = "0.0.0.0:4544"; +const CLIENT_ADDR: &str = "127.0.0.1 4544"; + +pub(crate) fn cat(deno_exe: &PathBuf, megs: usize) -> Result { + let size = megs * MB; + let shell_cmd = format!( + "{} run --allow-read cli/tests/cat.ts /dev/zero | head -c {}", + deno_exe.to_str().unwrap(), + size + ); + println!("{}", shell_cmd); + let cmd = &["sh", "-c", &shell_cmd]; + + let start = Instant::now(); + let _ = test_util::run_collect(cmd, None, None, None, true); + let end = Instant::now(); + + Ok(Value::Number( + Number::from_f64((end - start).as_secs_f64()).unwrap(), + )) +} + +pub(crate) fn tcp(deno_exe: &PathBuf, megs: usize) -> Result { + let size = megs * MB; + + let shell_cmd = format!("head -c {} /dev/zero | nc {}", size, CLIENT_ADDR); + println!("{}", shell_cmd); + let cmd = &["sh", "-c", &shell_cmd]; + + // Run deno echo server in the background. + let mut echo_server = Command::new(deno_exe.to_str().unwrap()) + .args(&[ + "run", + "--allow-net", + "cli/tests/echo_server.ts", + SERVER_ADDR, + ]) + .spawn()?; + + std::thread::sleep(Duration::from_secs(5)); // wait for deno to wake up. TODO racy. + + let start = Instant::now(); + let _ = test_util::run_collect(cmd, None, None, None, true); + let end = Instant::now(); + + echo_server.kill()?; + + Ok(Value::Number( + Number::from_f64((end - start).as_secs_f64()).unwrap(), + )) +} diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs index 4184b0b0a7..c0dcdb606b 100644 --- a/cli/tests/integration_tests.rs +++ b/cli/tests/integration_tests.rs @@ -272,11 +272,6 @@ grault", )); } -#[test] -fn benchmark_test() { - util::run_python_script("tools/benchmark_test.py") -} - #[test] fn deno_dir_test() { use std::fs::remove_dir_all; diff --git a/test_util/src/lib.rs b/test_util/src/lib.rs index 9b029c3556..84e353a8b5 100644 --- a/test_util/src/lib.rs +++ b/test_util/src/lib.rs @@ -10,6 +10,7 @@ use os_pipe::pipe; #[cfg(unix)] pub use pty; use regex::Regex; +use std::collections::HashMap; use std::env; use std::io::Read; use std::io::Write; @@ -57,10 +58,18 @@ pub fn root_path() -> PathBuf { PathBuf::from(concat!(env!("CARGO_MANIFEST_DIR"), "/..")) } +pub fn prebuilt_path() -> PathBuf { + third_party_path().join("prebuilt") +} + pub fn tests_path() -> PathBuf { root_path().join("cli").join("tests") } +pub fn third_party_path() -> PathBuf { + root_path().join("third_party") +} + pub fn target_dir() -> PathBuf { let current_exe = std::env::current_exe().unwrap(); let target_dir = current_exe.parent().unwrap().parent().unwrap(); @@ -77,6 +86,24 @@ pub fn deno_exe_path() -> PathBuf { p } +pub fn prebuilt_tool_path(tool: &str) -> PathBuf { + let mut exe = tool.to_string(); + exe.push_str(if cfg!(windows) { ".exe" } else { "" }); + prebuilt_path().join(platform_dir_name()).join(exe) +} + +fn platform_dir_name() -> &'static str { + if cfg!(target_os = "linux") { + "linux64" + } else if cfg!(target_os = "macos") { + "mac" + } else if cfg!(target_os = "windows") { + "win" + } else { + unreachable!() + } +} + pub fn test_server_path() -> PathBuf { let mut p = target_dir().join("test_server"); if cfg!(windows) { @@ -578,6 +605,76 @@ pub fn strip_ansi_codes(s: &str) -> std::borrow::Cow { STRIP_ANSI_RE.replace_all(s, "") } +pub fn run( + cmd: &[&str], + input: Option<&[&str]>, + envs: Option>, + current_dir: Option<&str>, + expect_success: bool, +) { + let mut process_builder = Command::new(cmd[0]); + process_builder.args(&cmd[1..]).stdin(Stdio::piped()); + + if let Some(dir) = current_dir { + process_builder.current_dir(dir); + } + if let Some(envs) = envs { + process_builder.envs(envs); + } + let mut prog = process_builder.spawn().expect("failed to spawn script"); + if let Some(lines) = input { + let stdin = prog.stdin.as_mut().expect("failed to get stdin"); + stdin + .write_all(lines.join("\n").as_bytes()) + .expect("failed to write to stdin"); + } + let status = prog.wait().expect("failed to wait on child"); + if expect_success != status.success() { + panic!("Unexpected exit code: {:?}", status.code()); + } +} + +pub fn run_collect( + cmd: &[&str], + input: Option<&[&str]>, + envs: Option>, + current_dir: Option<&str>, + expect_success: bool, +) -> (String, String) { + let mut process_builder = Command::new(cmd[0]); + process_builder + .args(&cmd[1..]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + if let Some(dir) = current_dir { + process_builder.current_dir(dir); + } + if let Some(envs) = envs { + process_builder.envs(envs); + } + let mut prog = process_builder.spawn().expect("failed to spawn script"); + if let Some(lines) = input { + let stdin = prog.stdin.as_mut().expect("failed to get stdin"); + stdin + .write_all(lines.join("\n").as_bytes()) + .expect("failed to write to stdin"); + } + let Output { + stdout, + stderr, + status, + } = prog.wait_with_output().expect("failed to wait on child"); + let stdout = String::from_utf8(stdout).unwrap(); + let stderr = String::from_utf8(stderr).unwrap(); + if expect_success != status.success() { + eprintln!("stdout: <<<{}>>>", stdout); + eprintln!("stderr: <<<{}>>>", stderr); + panic!("Unexpected exit code: {:?}", status.code()); + } + (stdout, stderr) +} + pub fn run_and_collect_output( expect_success: bool, args: &str, @@ -855,45 +952,248 @@ pub fn test_pty(args: &str, output_path: &str, input: &[u8]) { } } -#[test] -fn test_wildcard_match() { - let fixtures = vec![ - ("foobarbaz", "foobarbaz", true), - ("[WILDCARD]", "foobarbaz", true), - ("foobar", "foobarbaz", false), - ("foo[WILDCARD]baz", "foobarbaz", true), - ("foo[WILDCARD]baz", "foobazbar", false), - ("foo[WILDCARD]baz[WILDCARD]qux", "foobarbazqatqux", true), - ("foo[WILDCARD]", "foobar", true), - ("foo[WILDCARD]baz[WILDCARD]", "foobarbazqat", true), - // check with different line endings - ("foo[WILDCARD]\nbaz[WILDCARD]\n", "foobar\nbazqat\n", true), - ( - "foo[WILDCARD]\nbaz[WILDCARD]\n", - "foobar\r\nbazqat\r\n", - true, - ), - ( - "foo[WILDCARD]\r\nbaz[WILDCARD]\n", - "foobar\nbazqat\r\n", - true, - ), - ( - "foo[WILDCARD]\r\nbaz[WILDCARD]\r\n", - "foobar\nbazqat\n", - true, - ), - ( - "foo[WILDCARD]\r\nbaz[WILDCARD]\r\n", - "foobar\r\nbazqat\r\n", - true, - ), - ]; +pub struct WrkOutput { + pub latency: f64, + pub requests: u64, +} - // Iterate through the fixture lists, testing each one - for (pattern, string, expected) in fixtures { - let actual = wildcard_match(pattern, string); - dbg!(pattern, string, expected); - assert_eq!(actual, expected); +pub fn parse_wrk_output(output: &str) -> WrkOutput { + lazy_static! { + static ref REQUESTS_RX: Regex = + Regex::new(r"Requests/sec:\s+(\d+)").unwrap(); + static ref LATENCY_RX: Regex = + Regex::new(r"\s+99%(?:\s+(\d+.\d+)([a-z]+))").unwrap(); + } + + let mut requests = None; + let mut latency = None; + + for line in output.lines() { + if requests == None { + if let Some(cap) = REQUESTS_RX.captures(line) { + requests = + Some(str::parse::(cap.get(1).unwrap().as_str()).unwrap()); + } + } + if latency == None { + if let Some(cap) = LATENCY_RX.captures(line) { + let time = cap.get(1).unwrap(); + let unit = cap.get(2).unwrap(); + + latency = Some( + str::parse::(time.as_str()).unwrap() + * match unit.as_str() { + "ms" => 1.0, + "us" => 0.001, + "s" => 1000.0, + _ => unreachable!(), + }, + ); + } + } + } + + WrkOutput { + requests: requests.unwrap(), + latency: latency.unwrap(), + } +} + +pub struct StraceOutput { + pub percent_time: f64, + pub seconds: f64, + pub usecs_per_call: Option, + pub calls: u64, + pub errors: u64, +} + +pub fn parse_strace_output(output: &str) -> HashMap { + let mut summary = HashMap::new(); + + // Filter out non-relevant lines. See the error log at + // https://github.com/denoland/deno/pull/3715/checks?check_run_id=397365887 + // This is checked in testdata/strace_summary2.out + let mut lines = output + .lines() + .filter(|line| !line.is_empty() && !line.contains("detached ...")); + let count = lines.clone().count(); + + if count < 4 { + return summary; + } + + let total_line = lines.next_back().unwrap(); + lines.next_back(); // Drop separator + let data_lines = lines.skip(2); + + for line in data_lines { + let syscall_fields = line.split_whitespace().collect::>(); + let len = syscall_fields.len(); + let syscall_name = syscall_fields.last().unwrap(); + + if 5 <= len && len <= 6 { + summary.insert( + syscall_name.to_string(), + StraceOutput { + percent_time: str::parse::(syscall_fields[0]).unwrap(), + seconds: str::parse::(syscall_fields[1]).unwrap(), + usecs_per_call: Some(str::parse::(syscall_fields[2]).unwrap()), + calls: str::parse::(syscall_fields[3]).unwrap(), + errors: if syscall_fields.len() < 6 { + 0 + } else { + str::parse::(syscall_fields[4]).unwrap() + }, + }, + ); + } + } + + let total_fields = total_line.split_whitespace().collect::>(); + summary.insert( + "total".to_string(), + StraceOutput { + percent_time: str::parse::(total_fields[0]).unwrap(), + seconds: str::parse::(total_fields[1]).unwrap(), + usecs_per_call: None, + calls: str::parse::(total_fields[2]).unwrap(), + errors: str::parse::(total_fields[3]).unwrap(), + }, + ); + + summary +} + +pub fn parse_max_mem(output: &str) -> Option { + // Takes the output from "time -v" as input and extracts the 'maximum + // resident set size' and returns it in bytes. + for line in output.lines() { + if line + .to_lowercase() + .contains("maximum resident set size (kbytes)") + { + let value = line.split(": ").nth(1).unwrap(); + return Some(str::parse::(value).unwrap() * 1024); + } + } + + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_wrk_output_1() { + const TEXT: &str = include_str!("./testdata/wrk1.txt"); + let wrk = parse_wrk_output(TEXT); + assert_eq!(wrk.requests, 1837); + assert!((wrk.latency - 6.25).abs() < f64::EPSILON); + } + + #[test] + fn parse_wrk_output_2() { + const TEXT: &str = include_str!("./testdata/wrk2.txt"); + let wrk = parse_wrk_output(TEXT); + assert_eq!(wrk.requests, 53435); + assert!((wrk.latency - 6.22).abs() < f64::EPSILON); + } + + #[test] + fn parse_wrk_output_3() { + const TEXT: &str = include_str!("./testdata/wrk3.txt"); + let wrk = parse_wrk_output(TEXT); + assert_eq!(wrk.requests, 96037); + assert!((wrk.latency - 6.36).abs() < f64::EPSILON); + } + + #[test] + fn strace_parse_1() { + const TEXT: &str = include_str!("./testdata/strace_summary.out"); + let strace = parse_strace_output(TEXT); + + // first syscall line + let munmap = strace.get("munmap").unwrap(); + assert_eq!(munmap.calls, 60); + assert_eq!(munmap.errors, 0); + + // line with errors + assert_eq!(strace.get("mkdir").unwrap().errors, 2); + + // last syscall line + let prlimit = strace.get("prlimit64").unwrap(); + assert_eq!(prlimit.calls, 2); + assert!((prlimit.percent_time - 0.0).abs() < f64::EPSILON); + + // summary line + assert_eq!(strace.get("total").unwrap().calls, 704); + assert_eq!(strace.get("total").unwrap().errors, 5); + } + + #[test] + fn strace_parse_2() { + const TEXT: &str = include_str!("./testdata/strace_summary2.out"); + let strace = parse_strace_output(TEXT); + + // first syscall line + let futex = strace.get("futex").unwrap(); + assert_eq!(futex.calls, 449); + assert_eq!(futex.errors, 94); + + // summary line + assert_eq!(strace.get("total").unwrap().calls, 821); + assert_eq!(strace.get("total").unwrap().errors, 107); + } + + #[test] + fn test_wildcard_match() { + let fixtures = vec![ + ("foobarbaz", "foobarbaz", true), + ("[WILDCARD]", "foobarbaz", true), + ("foobar", "foobarbaz", false), + ("foo[WILDCARD]baz", "foobarbaz", true), + ("foo[WILDCARD]baz", "foobazbar", false), + ("foo[WILDCARD]baz[WILDCARD]qux", "foobarbazqatqux", true), + ("foo[WILDCARD]", "foobar", true), + ("foo[WILDCARD]baz[WILDCARD]", "foobarbazqat", true), + // check with different line endings + ("foo[WILDCARD]\nbaz[WILDCARD]\n", "foobar\nbazqat\n", true), + ( + "foo[WILDCARD]\nbaz[WILDCARD]\n", + "foobar\r\nbazqat\r\n", + true, + ), + ( + "foo[WILDCARD]\r\nbaz[WILDCARD]\n", + "foobar\nbazqat\r\n", + true, + ), + ( + "foo[WILDCARD]\r\nbaz[WILDCARD]\r\n", + "foobar\nbazqat\n", + true, + ), + ( + "foo[WILDCARD]\r\nbaz[WILDCARD]\r\n", + "foobar\r\nbazqat\r\n", + true, + ), + ]; + + // Iterate through the fixture lists, testing each one + for (pattern, string, expected) in fixtures { + let actual = wildcard_match(pattern, string); + dbg!(pattern, string, expected); + assert_eq!(actual, expected); + } + } + + #[test] + fn max_mem_parse() { + const TEXT: &str = include_str!("./testdata/time.out"); + let size = parse_max_mem(TEXT); + + assert_eq!(size, Some(120380 * 1024)); } } diff --git a/tools/testdata/strace_summary.out b/test_util/src/testdata/strace_summary.out similarity index 100% rename from tools/testdata/strace_summary.out rename to test_util/src/testdata/strace_summary.out diff --git a/tools/testdata/strace_summary2.out b/test_util/src/testdata/strace_summary2.out similarity index 100% rename from tools/testdata/strace_summary2.out rename to test_util/src/testdata/strace_summary2.out diff --git a/tools/testdata/time.out b/test_util/src/testdata/time.out similarity index 100% rename from tools/testdata/time.out rename to test_util/src/testdata/time.out diff --git a/tools/testdata/wrk1.txt b/test_util/src/testdata/wrk1.txt similarity index 100% rename from tools/testdata/wrk1.txt rename to test_util/src/testdata/wrk1.txt diff --git a/tools/testdata/wrk2.txt b/test_util/src/testdata/wrk2.txt similarity index 100% rename from tools/testdata/wrk2.txt rename to test_util/src/testdata/wrk2.txt diff --git a/tools/testdata/wrk3.txt b/test_util/src/testdata/wrk3.txt similarity index 100% rename from tools/testdata/wrk3.txt rename to test_util/src/testdata/wrk3.txt diff --git a/tools/benchmark.py b/tools/benchmark.py deleted file mode 100755 index 74a5fb6655..0000000000 --- a/tools/benchmark.py +++ /dev/null @@ -1,277 +0,0 @@ -#!/usr/bin/env python -# Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -# Performs benchmark and append data to //website/data.json. -# If //website/data.json doesn't exist, this script tries to import it from -# gh-pages branch. -# To view the results locally run target/debug/test_server and visit -# http://localhost:4545/website - -import os -import sys -import json -import time -import tempfile -import subprocess -from util import build_path, executable_suffix, root_path, run, run_output -import third_party -from http_benchmark import http_benchmark -import throughput_benchmark - -# The list of the tuples of the benchmark name, arguments and return code -exec_time_benchmarks = [ - ("hello", ["run", "cli/tests/002_hello.ts"], None), - ("relative_import", ["run", "cli/tests/003_relative_import.ts"], None), - ("error_001", ["run", "cli/tests/error_001.ts"], 1), - ("cold_hello", ["run", "--reload", "cli/tests/002_hello.ts"], None), - ("cold_relative_import", - ["run", "--reload", "cli/tests/003_relative_import.ts"], None), - ("workers_startup", - ["run", "--allow-read", "cli/tests/workers_startup_bench.ts"], None), - ("workers_round_robin", - ["run", "--allow-read", "cli/tests/workers_round_robin_bench.ts"], None), - ("text_decoder", ["run", "cli/tests/text_decoder_perf.js"], None), - ("text_encoder", ["run", "cli/tests/text_encoder_perf.js"], None), - ("check", ["cache", "--reload", "std/examples/chat/server_test.ts"], None), - ("no_check", - ["cache", "--reload", "--no-check", - "std/examples/chat/server_test.ts"], None), -] - - -def read_json(filename): - with open(filename) as json_file: - return json.load(json_file) - - -def write_json(filename, data): - with open(filename, 'w') as outfile: - json.dump(data, outfile) - - -def get_binary_sizes(build_dir): - sizes = {} - mtimes = {} - # The deno executable should be located at the root of the build tree. - deno_exe = os.path.join(build_dir, "deno" + executable_suffix) - sizes["deno"] = os.path.getsize(deno_exe) - # Because cargo's OUT_DIR is not predictable, search the build tree for - # snapshot related files. - for parent_dir, _, file_names in os.walk(build_dir): - for file_name in file_names: - if not file_name in [ - "CLI_SNAPSHOT.bin", - "COMPILER_SNAPSHOT.bin", - ]: - continue - file_path = os.path.join(parent_dir, file_name) - file_mtime = os.path.getmtime(file_path) - # If multiple copies of a file are found, use the most recent one. - if file_name in mtimes and mtimes[file_name] > file_mtime: - continue - mtimes[file_name] = file_mtime - sizes[file_name] = os.path.getsize(file_path) - return sizes - - -def get_strace_summary_text(test_args): - f = tempfile.NamedTemporaryFile() - cmd = ["strace", "-c", "-f", "-o", f.name] + test_args - try: - subprocess.check_output(cmd) - except subprocess.CalledProcessError: - pass - return f.read() - - -def strace_parse(summary_text): - summary = {} - # clear empty lines - lines = list(filter(lambda x: x and x != "\n", summary_text.split("\n"))) - # Filter out non-relevant lines. See the error log at - # https://github.com/denoland/deno/pull/3715/checks?check_run_id=397365887 - # This is checked in tools/testdata/strace_summary2.out - lines = [x for x in lines if x.find("detached ...") == -1] - if len(lines) < 4: - return {} # malformed summary - lines, total_line = lines[2:-2], lines[-1] - # data to dict for each line - for line in lines: - syscall_fields = line.split() - syscall_name = syscall_fields[-1] - syscall_dict = {} - if 5 <= len(syscall_fields) <= 6: - syscall_dict = { - "% time": float(syscall_fields[0]), - "seconds": float(syscall_fields[1]), - "usecs/call": int(syscall_fields[2]), - "calls": int(syscall_fields[3]) - } - syscall_dict["errors"] = 0 if len(syscall_fields) < 6 else int( - syscall_fields[4]) - summary[syscall_name] = syscall_dict - # record overall (total) data - total_fields = total_line.split() - summary["total"] = { - "% time": float(total_fields[0]), - "seconds": float(total_fields[1]), - "calls": int(total_fields[2]), - "errors": int(total_fields[3]) - } - return summary - - -def get_strace_summary(test_args): - s = get_strace_summary_text(test_args) - try: - return strace_parse(s) - except ValueError: - print "error parsing strace" - print "----- -------" - print s - print "----- ------" - - -def run_throughput(deno_exe): - m = {} - m["100M_tcp"] = throughput_benchmark.tcp(deno_exe, 100) - m["100M_cat"] = throughput_benchmark.cat(deno_exe, 100) - m["10M_tcp"] = throughput_benchmark.tcp(deno_exe, 10) - m["10M_cat"] = throughput_benchmark.cat(deno_exe, 10) - return m - - -# "thread_count" and "syscall_count" are both calculated here. -def run_strace_benchmarks(deno_exe, new_data): - thread_count = {} - syscall_count = {} - for (name, args, _) in exec_time_benchmarks: - s = get_strace_summary([deno_exe] + args) - thread_count[name] = s["clone"]["calls"] + 1 - syscall_count[name] = s["total"]["calls"] - new_data["thread_count"] = thread_count - new_data["syscall_count"] = syscall_count - - -# Takes the output from "/usr/bin/time -v" as input and extracts the 'maximum -# resident set size' and returns it in bytes. -def find_max_mem_in_bytes(time_v_output): - for line in time_v_output.split('\n'): - if 'maximum resident set size (kbytes)' in line.lower(): - _, value = line.split(': ') - return int(value) * 1024 - - -def run_max_mem_benchmark(deno_exe): - results = {} - for (name, args, return_code) in exec_time_benchmarks: - cmd = ["/usr/bin/time", "-v", deno_exe] + args - try: - out = subprocess.check_output(cmd, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: - if (return_code is e.returncode): - pass - else: - raise e - mem = find_max_mem_in_bytes(out) - results[name] = mem - return results - - -def run_exec_time(deno_exe, build_dir): - hyperfine_exe = third_party.get_prebuilt_tool_path("hyperfine") - benchmark_file = os.path.join(build_dir, "hyperfine_results.json") - - def benchmark_command(deno_exe, args, return_code): - # Bash test which asserts the return code value of the previous command - # $? contains the return code of the previous command - return_code_test = "; test $? -eq {}".format( - return_code) if return_code is not None else "" - return "{} {}{}".format(deno_exe, " ".join(args), return_code_test) - - run([hyperfine_exe, "--export-json", benchmark_file, "--warmup", "3"] + [ - benchmark_command(deno_exe, args, return_code) - for (_, args, return_code) in exec_time_benchmarks - ]) - hyperfine_results = read_json(benchmark_file) - results = {} - for [[name, _, _], data] in zip(exec_time_benchmarks, - hyperfine_results["results"]): - results[name] = { - "mean": data["mean"], - "stddev": data["stddev"], - "user": data["user"], - "system": data["system"], - "min": data["min"], - "max": data["max"] - } - return results - - -def run_http(build_dir, new_data): - stats = http_benchmark(build_dir) - new_data["req_per_sec"] = {k: v["req_per_sec"] for k, v in stats.items()} - new_data["max_latency"] = {k: v["max_latency"] for k, v in stats.items()} - - -def bundle_benchmark(deno_exe): - bundles = { - "file_server": "./std/http/file_server.ts", - "gist": "./std/examples/gist.ts", - } - - sizes = {} - - for name, url in bundles.items(): - # bundle - path = name + ".bundle.js" - run([deno_exe, "bundle", "--unstable", url, path]) - # get size of bundle - assert os.path.exists(path) - sizes[name] = os.path.getsize(path) - # remove bundle - os.remove(path) - - return sizes - - -def main(): - build_dir = build_path() - sha1 = run_output(["git", "rev-parse", "HEAD"], - exit_on_fail=True).out.strip() - - deno_exe = os.path.join(build_dir, "deno") - - os.chdir(root_path) - - new_data = { - "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ"), - "sha1": sha1, - } - - # TODO(ry) The "benchmark" benchmark should actually be called "exec_time". - # When this is changed, the historical data in gh-pages branch needs to be - # changed too. - new_data["benchmark"] = run_exec_time(deno_exe, build_dir) - - new_data["binary_size"] = get_binary_sizes(build_dir) - new_data["bundle_size"] = bundle_benchmark(deno_exe) - - # Cannot run throughput benchmark on windows because they don't have nc or - # pipe. - if os.name != 'nt': - new_data["throughput"] = run_throughput(deno_exe) - run_http(build_dir, new_data) - - if "linux" in sys.platform: - run_strace_benchmarks(deno_exe, new_data) - new_data["max_memory"] = run_max_mem_benchmark(deno_exe) - - print "===== " - print json.dumps(new_data, indent=2) - print "===== " - - write_json(os.path.join(build_dir, "bench.json"), new_data) - - -if __name__ == '__main__': - main() diff --git a/tools/benchmark_test.py b/tools/benchmark_test.py deleted file mode 100755 index efd4594b64..0000000000 --- a/tools/benchmark_test.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python -# Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -import sys -import os -import unittest -import benchmark -from test_util import DenoTestCase, run_tests - - -class TestBenchmark(DenoTestCase): - def test_strace_parse(self): - with open( - os.path.join(sys.path[0], "testdata/strace_summary.out"), - "r") as f: - summary = benchmark.strace_parse(f.read()) - # first syscall line - assert summary["munmap"]["calls"] == 60 - assert summary["munmap"]["errors"] == 0 - # line with errors - assert summary["mkdir"]["errors"] == 2 - # last syscall line - assert summary["prlimit64"]["calls"] == 2 - assert summary["prlimit64"]["% time"] == 0 - # summary line - assert summary["total"]["calls"] == 704 - - def test_strace_parse2(self): - with open( - os.path.join(sys.path[0], "testdata/strace_summary2.out"), - "r") as f: - summary = benchmark.strace_parse(f.read()) - # first syscall line - assert summary["futex"]["calls"] == 449 - assert summary["futex"]["errors"] == 94 - # summary line - assert summary["total"]["calls"] == 821 - - def test_max_mem_parse(self): - with open(os.path.join(sys.path[0], "testdata/time.out"), "r") as f: - data = f.read() - assert benchmark.find_max_mem_in_bytes(data) == 120380 * 1024 - - def test_binary_size(self): - binary_size_dict = benchmark.get_binary_sizes(self.build_dir) - assert binary_size_dict["deno"] > 0 - assert binary_size_dict["CLI_SNAPSHOT.bin"] > 0 - - @unittest.skipIf("linux" not in sys.platform, - "strace only supported on linux") - def test_strace(self): - new_data = {} - benchmark.run_strace_benchmarks(self.deno_exe, new_data) - assert "thread_count" in new_data - assert "syscall_count" in new_data - - s = new_data["thread_count"] - assert "hello" in s - assert s["hello"] > 1 - - s = new_data["syscall_count"] - assert "hello" in s - assert s["hello"] > 1 - - -if __name__ == '__main__': - run_tests() diff --git a/tools/http_benchmark.py b/tools/http_benchmark.py deleted file mode 100755 index 17420fe84b..0000000000 --- a/tools/http_benchmark.py +++ /dev/null @@ -1,215 +0,0 @@ -#!/usr/bin/env python -# Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -import os -import sys -import time -import subprocess -import util -import third_party - -# Some of the benchmarks in this file have been renamed. In case the history -# somehow gets messed up: -# "node_http" was once called "node" -# "deno_tcp" was once called "deno" -# "deno_http" was once called "deno_net_http" - -DURATION = "20s" -NEXT_PORT = 4544 - - -def server_addr(port): - return "0.0.0.0:%s" % port - - -def get_port(): - global NEXT_PORT - port = NEXT_PORT - NEXT_PORT += 1 - # Return port as str because all usages below are as a str and having it an - # integer just adds complexity. - return str(port) - - -def deno_tcp(deno_exe): - port = get_port() - deno_cmd = [ - # TODO(lucacasonato): remove unstable when stabilized - deno_exe, - "run", - "--allow-net", - "tools/deno_tcp.ts", - server_addr(port) - ] - print "http_benchmark testing DENO tcp." - return run(deno_cmd, port) - - -def deno_http(deno_exe): - port = get_port() - deno_cmd = [ - deno_exe, "run", "--allow-net", "--reload", "--unstable", - "std/http/http_bench.ts", - server_addr(port) - ] - print "http_benchmark testing DENO using net/http." - return run(deno_cmd, port) - - -def deno_tcp_proxy(deno_exe, hyper_hello_exe): - port = get_port() - origin_port = get_port() - deno_cmd = [ - deno_exe, "run", "--allow-net", "tools/deno_tcp_proxy.ts", - server_addr(port), - server_addr(origin_port) - ] - print "http_proxy_benchmark testing DENO using net/tcp." - return run( - deno_cmd, - port, - origin_cmd=http_proxy_origin(hyper_hello_exe, origin_port)) - - -def deno_http_proxy(deno_exe, hyper_hello_exe): - port = get_port() - origin_port = get_port() - deno_cmd = [ - deno_exe, "run", "--allow-net", "tools/deno_http_proxy.ts", - server_addr(port), - server_addr(origin_port) - ] - print "http_proxy_benchmark testing DENO using net/http." - return run( - deno_cmd, - port, - origin_cmd=http_proxy_origin(hyper_hello_exe, origin_port)) - - -def core_http_bin_ops(exe): - print "http_benchmark testing CORE http_bench_bin_ops" - return run([exe], 4544) - - -def core_http_json_ops(exe): - print "http_benchmark testing CORE http_bench_json_ops" - return run([exe], 4544) - - -def node_http(): - port = get_port() - node_cmd = ["node", "tools/node_http.js", port] - print "http_benchmark testing NODE." - return run(node_cmd, port) - - -def node_http_proxy(hyper_hello_exe): - port = get_port() - origin_port = get_port() - node_cmd = ["node", "tools/node_http_proxy.js", port, origin_port] - print "http_proxy_benchmark testing NODE." - return run(node_cmd, port, None, - http_proxy_origin(hyper_hello_exe, origin_port)) - - -def node_tcp_proxy(hyper_hello_exe): - port = get_port() - origin_port = get_port() - node_cmd = ["node", "tools/node_tcp_proxy.js", port, origin_port] - print "http_proxy_benchmark testing NODE tcp." - return run(node_cmd, port, None, - http_proxy_origin(hyper_hello_exe, origin_port)) - - -def node_tcp(): - port = get_port() - node_cmd = ["node", "tools/node_tcp.js", port] - print "http_benchmark testing node_tcp.js" - return run(node_cmd, port) - - -def http_proxy_origin(hyper_hello_exe, port): - return [hyper_hello_exe, port] - - -def hyper_http(hyper_hello_exe): - port = get_port() - hyper_cmd = [hyper_hello_exe, port] - print "http_benchmark testing RUST hyper." - return run(hyper_cmd, port) - - -def http_benchmark(build_dir): - deno_exe = os.path.join(build_dir, "deno") - hyper_hello_exe = os.path.join(build_dir, "test_server") - core_http_bin_ops_exe = os.path.join(build_dir, - "examples/http_bench_bin_ops") - core_http_json_ops_exe = os.path.join(build_dir, - "examples/http_bench_json_ops") - return { - # "deno_tcp" was once called "deno" - "deno_tcp": deno_tcp(deno_exe), - # "deno_udp": deno_udp(deno_exe), - "deno_http": deno_http(deno_exe), - # TODO(ry) deno_proxy disabled to make fetch() standards compliant. - # "deno_proxy": deno_http_proxy(deno_exe, hyper_hello_exe), - "deno_proxy_tcp": deno_tcp_proxy(deno_exe, hyper_hello_exe), - # "core_http_bin_ops" was once called "deno_core_single" - # "core_http_bin_ops" was once called "deno_core_http_bench" - "core_http_bin_ops": core_http_bin_ops(core_http_bin_ops_exe), - "core_http_json_ops": core_http_json_ops(core_http_json_ops_exe), - # "node_http" was once called "node" - "node_http": node_http(), - "node_proxy": node_http_proxy(hyper_hello_exe), - "node_proxy_tcp": node_tcp_proxy(hyper_hello_exe), - "node_tcp": node_tcp(), - "hyper": hyper_http(hyper_hello_exe) - } - - -def run(server_cmd, port, merge_env=None, origin_cmd=None): - - # Run deno echo server in the background. - if merge_env is None: - env = None - else: - env = os.environ.copy() - for key, value in merge_env.iteritems(): - env[key] = value - - # Wait for port 4544 to become available. - # TODO Need to use SO_REUSEPORT with tokio::net::TcpListener. - time.sleep(5) - - origin = None - if origin_cmd is not None: - origin = subprocess.Popen(origin_cmd, env=env) - - print server_cmd - server = subprocess.Popen(server_cmd, env=env) - - time.sleep(5) # wait for server to wake up. TODO racy. - - try: - wrk = third_party.get_prebuilt_tool_path("wrk") - assert os.path.exists(wrk) - cmd = "%s -d %s --latency http://127.0.0.1:%s/" % (wrk, DURATION, port) - print cmd - output = subprocess.check_output(cmd, shell=True) - stats = util.parse_wrk_output(output) - print output - return stats - finally: - server_retcode = server.poll() - if server_retcode is not None and server_retcode != 0: - print "server ended with error" - sys.exit(1) - server.kill() - if origin is not None: - origin.kill() - - -if __name__ == '__main__': - if len(sys.argv) < 2: - print "Usage ./tools/http_benchmark.py target/debug/deno" - sys.exit(1) - deno_http(sys.argv[1]) diff --git a/tools/lint.py b/tools/lint.py index 5891ba9f9a..e7f967140d 100755 --- a/tools/lint.py +++ b/tools/lint.py @@ -72,6 +72,7 @@ def eslint(): ":!:cli/tests/encoding/**", ":!:cli/dts/**", ":!:cli/tsc/*typescript.js", + ":!:cli/bench/node*.js", ]) if source_files: max_command_len = 30000 diff --git a/tools/throughput_benchmark.py b/tools/throughput_benchmark.py deleted file mode 100755 index f5b9f2db23..0000000000 --- a/tools/throughput_benchmark.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python -# Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -# Performs benchmark and append data to //website/data.json. -# If //website/data.json doesn't exist, this script tries to import it from -# gh-pages branch. -# To view the results locally run target/debug/test_server and visit -# http://localhost:4545/website - -import os -import sys -import time -import subprocess -import util - -MB = 1024 * 1024 -SERVER_ADDR = "0.0.0.0:4544" -CLIENT_ADDR = "127.0.0.1 4544" - - -def cat(deno_exe, megs): - size = megs * MB - start = time.time() - cmd = deno_exe + " run --allow-read " - cmd += "cli/tests/cat.ts /dev/zero | head -c %s " % size - print cmd - subprocess.check_output(cmd, shell=True) - end = time.time() - return end - start - - -def tcp(deno_exe, megs): - size = megs * MB - # Run deno echo server in the background. - args = [ - deno_exe, "run", "--allow-net", "cli/tests/echo_server.ts", SERVER_ADDR - ] - print args - echo_server = subprocess.Popen(args) - - time.sleep(5) # wait for deno to wake up. TODO racy. - try: - start = time.time() - nc_cmd = "nc " + CLIENT_ADDR - cmd = ("head -c %s /dev/zero " % size) + " | " + nc_cmd - print cmd - subprocess.check_output(cmd, shell=True) - end = time.time() - return end - start - finally: - echo_server.kill() - - -def main(): - deno_exe = sys.argv[1] - megs = int(sys.argv[2]) - if not deno_exe or not megs: - print "Usage ./tools/throughput_benchmark.py target/debug/deno 100" - sys.exit(1) - secs = tcp(sys.argv[1], megs) - print secs, "seconds" - - -if __name__ == '__main__': - main() diff --git a/tools/util.py b/tools/util.py index 889afae82e..7cae4f84c2 100644 --- a/tools/util.py +++ b/tools/util.py @@ -361,20 +361,6 @@ def extract_max_latency_in_milliseconds(pattern, string): return num * 1000 -def parse_wrk_output(output): - stats = {} - stats['req_per_sec'] = None - stats['max_latency'] = None - for line in output.split("\n"): - if stats['req_per_sec'] is None: - stats['req_per_sec'] = extract_number(r'Requests/sec:\s+(\d+)', - line) - if stats['max_latency'] is None: - stats['max_latency'] = extract_max_latency_in_milliseconds( - r'\s+99%(?:\s+(\d+.\d+)([a-z]+))', line) - return stats - - def platform(): return {"linux2": "linux", "darwin": "mac", "win32": "win"}[sys.platform] diff --git a/tools/util_test.py b/tools/util_test.py index 8c25b10ed2..7ad9b415e5 100755 --- a/tools/util_test.py +++ b/tools/util_test.py @@ -2,8 +2,7 @@ import os from test_util import DenoTestCase, run_tests -from util import (parse_exit_code, shell_quote_win, parse_wrk_output, - root_path) +from util import (parse_exit_code, shell_quote_win, root_path) class TestUtil(DenoTestCase): @@ -21,22 +20,6 @@ class TestUtil(DenoTestCase): assert shell_quote_win( 'a"b""c\\d\\"e\\\\') == '"a""b""""c\\d\\\\""e\\\\\\\\"' - def test_parse_wrk_output(self): - f = open(os.path.join(root_path, "tools/testdata/wrk1.txt")) - stats = parse_wrk_output(f.read()) - assert stats['req_per_sec'] == 1837 - assert stats['max_latency'] == 6.25 - - f2 = open(os.path.join(root_path, "tools/testdata/wrk2.txt")) - stats2 = parse_wrk_output(f2.read()) - assert stats2['req_per_sec'] == 53435 - assert stats2['max_latency'] == 6.22 - - f3 = open(os.path.join(root_path, "tools/testdata/wrk3.txt")) - stats3 = parse_wrk_output(f3.read()) - assert stats3['req_per_sec'] == 96037 - assert stats3['max_latency'] == 6.36 - def test_executable_exists(self): assert os.path.exists(self.deno_exe)