opensourcemirror
/
deno
mirror of https://github.com/denoland/deno.git
1
0
Fork 0
Code Issues Releases Wiki Activity

refactor: move redirect handling into deno_graph (#23444)

This commit is contained in:
David Sherret 2024-04-18 21:43:28 -04:00 committed by GitHub
parent 8e77f091ad
commit c497e766f1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
23 changed files with 377 additions and 263 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.toml
View File

@ -48,7 +48,7 @@ deno_core = { version = "0.275.0" }
deno_bench_util = { version = "0.141.0", path = "./bench_util" }
deno_lockfile = "0.19.0"
deno_media_type = { version = "0.1.3", features = ["module_specifier"] }
deno_media_type = { version = "0.1.4", features = ["module_specifier"] }
deno_permissions = { version = "0.7.0", path = "./runtime/permissions" }
deno_runtime = { version = "0.155.0", path = "./runtime" }
deno_terminal = "0.1.1"

4
cli/args/import_map.rs
View File

@ -28,7 +28,7 @@ pub async fn resolve_import_map(
let specifier = specifier.clone();
async move {
let file = file_fetcher
.fetch(&specifier, PermissionsContainer::allow_all())
.fetch(&specifier, &PermissionsContainer::allow_all())
.await?
.into_text_decoded()?;
Ok(file.source.to_string())
@ -62,7 +62,7 @@ async fn resolve_import_map_from_specifier(
serde_json::from_str(&data_url_text)?
} else {
let file = file_fetcher
.fetch(&specifier, PermissionsContainer::allow_all())
.fetch(&specifier, &PermissionsContainer::allow_all())
.await?
.into_text_decoded()?;
serde_json::from_str(&file.source)?

55
cli/cache/mod.rs vendored
View File

@ -2,8 +2,10 @@
use crate::args::CacheSetting;
use crate::errors::get_error_class_name;
use crate::file_fetcher::FetchNoFollowOptions;
use crate::file_fetcher::FetchOptions;
use crate::file_fetcher::FileFetcher;
use crate::file_fetcher::FileOrRedirect;
use crate::npm::CliNpmResolver;
use crate::util::fs::atomic_write_file;
@ -234,29 +236,40 @@ impl Loader for FetchCacher {
LoaderCacheSetting::Only => Some(CacheSetting::Only),
};
file_fetcher
.fetch_with_options(FetchOptions {
specifier: &specifier,
permissions,
maybe_accept: None,
maybe_cache_setting: maybe_cache_setting.as_ref(),
maybe_checksum: options.maybe_checksum,
.fetch_no_follow_with_options(FetchNoFollowOptions {
fetch_options: FetchOptions {
specifier: &specifier,
permissions: &permissions,
maybe_accept: None,
maybe_cache_setting: maybe_cache_setting.as_ref(),
},
maybe_checksum: options.maybe_checksum.as_ref(),
})
.await
.map(|file| {
let maybe_headers =
match (file.maybe_headers, file_header_overrides.get(&specifier)) {
(Some(headers), Some(overrides)) => {
Some(headers.into_iter().chain(overrides.clone()).collect())
}
(Some(headers), None) => Some(headers),
(None, Some(overrides)) => Some(overrides.clone()),
(None, None) => None,
};
Ok(Some(LoadResponse::Module {
specifier: file.specifier,
maybe_headers,
content: file.source,
}))
.map(|file_or_redirect| {
match file_or_redirect {
FileOrRedirect::File(file) => {
let maybe_headers =
match (file.maybe_headers, file_header_overrides.get(&specifier)) {
(Some(headers), Some(overrides)) => {
Some(headers.into_iter().chain(overrides.clone()).collect())
}
(Some(headers), None) => Some(headers),
(None, Some(overrides)) => Some(overrides.clone()),
(None, None) => None,
};
Ok(Some(LoadResponse::Module {
specifier: file.specifier,
maybe_headers,
content: file.source,
}))
},
FileOrRedirect::Redirect(redirect_specifier) => {
Ok(Some(LoadResponse::Redirect {
specifier: redirect_specifier,
}))
},
}
})
.unwrap_or_else(|err| {
if let Some(io_err) = err.downcast_ref::<std::io::Error>() {

451
cli/file_fetcher.rs
View File

@ -20,8 +20,6 @@ use deno_core::error::custom_error;
use deno_core::error::generic_error;
use deno_core::error::uri_error;
use deno_core::error::AnyError;
use deno_core::futures;
use deno_core::futures::future::FutureExt;
use deno_core::parking_lot::Mutex;
use deno_core::url::Url;
use deno_core::ModuleSpecifier;
@ -34,12 +32,11 @@ use deno_runtime::deno_fetch::reqwest::StatusCode;
use deno_runtime::deno_web::BlobStore;
use deno_runtime::permissions::PermissionsContainer;
use log::debug;
use std::borrow::Cow;
use std::collections::HashMap;
use std::env;
use std::fs;
use std::future::Future;
use std::path::PathBuf;
use std::pin::Pin;
use std::sync::Arc;
use std::time::SystemTime;
@ -56,6 +53,12 @@ pub struct TextDecodedFile {
pub source: Arc<str>,
}
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum FileOrRedirect {
File(File),
Redirect(ModuleSpecifier),
}
/// A structure representing a source file.
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct File {
@ -145,10 +148,16 @@ fn get_validated_scheme(
pub struct FetchOptions<'a> {
pub specifier: &'a ModuleSpecifier,
pub permissions: PermissionsContainer,
pub permissions: &'a PermissionsContainer,
pub maybe_accept: Option<&'a str>,
pub maybe_cache_setting: Option<&'a CacheSetting>,
pub maybe_checksum: Option<LoaderChecksum>,
}
pub struct FetchNoFollowOptions<'a> {
pub fetch_options: FetchOptions<'a>,
/// This setting doesn't make sense to provide for `FetchOptions`
/// since the required checksum may change for a redirect.
pub maybe_checksum: Option<&'a LoaderChecksum>,
}
/// A structure for resolving, fetching and caching source files.
@ -202,13 +211,34 @@ impl FileFetcher {
pub fn fetch_cached(
&self,
specifier: &ModuleSpecifier,
maybe_checksum: Option<LoaderChecksum>,
redirect_limit: i64,
) -> Result<Option<File>, AnyError> {
debug!("FileFetcher::fetch_cached - specifier: {}", specifier);
if redirect_limit < 0 {
return Err(custom_error("Http", "Too many redirects."));
let mut specifier = Cow::Borrowed(specifier);
for _ in 0..=redirect_limit {
match self.fetch_cached_no_follow(&specifier, None)? {
Some(FileOrRedirect::File(file)) => {
return Ok(Some(file));
}
Some(FileOrRedirect::Redirect(redirect_specifier)) => {
specifier = Cow::Owned(redirect_specifier);
}
None => {
return Ok(None);
}
}
}
Err(custom_error("Http", "Too many redirects."))
}
fn fetch_cached_no_follow(
&self,
specifier: &ModuleSpecifier,
maybe_checksum: Option<&LoaderChecksum>,
) -> Result<Option<FileOrRedirect>, AnyError> {
debug!(
"FileFetcher::fetch_cached_no_follow - specifier: {}",
specifier
);
let cache_key = self.http_cache.cache_item_key(specifier)?; // compute this once
let Some(headers) = self.http_cache.read_headers(&cache_key)? else {
@ -217,7 +247,7 @@ impl FileFetcher {
if let Some(redirect_to) = headers.get("location") {
let redirect =
deno_core::resolve_import(redirect_to, specifier.as_str())?;
return self.fetch_cached(&redirect, maybe_checksum, redirect_limit - 1);
return Ok(Some(FileOrRedirect::Redirect(redirect)));
}
let Some(bytes) = self.http_cache.read_file_bytes(
&cache_key,
@ -230,11 +260,11 @@ impl FileFetcher {
return Ok(None);
};
Ok(Some(File {
Ok(Some(FileOrRedirect::File(File {
specifier: specifier.clone(),
maybe_headers: Some(headers),
source: Arc::from(bytes),
}))
})))
}
/// Convert a data URL into a file, resulting in an error if the URL is
@ -280,51 +310,33 @@ impl FileFetcher {
})
}
/// Asynchronously fetch remote source file specified by the URL following
/// redirects.
///
/// **Note** this is a recursive method so it can't be "async", but needs to
/// return a `Pin<Box<..>>`.
fn fetch_remote(
async fn fetch_remote_no_follow(
&self,
specifier: &ModuleSpecifier,
permissions: PermissionsContainer,
redirect_limit: i64,
maybe_accept: Option<String>,
maybe_accept: Option<&str>,
cache_setting: &CacheSetting,
maybe_checksum: Option<LoaderChecksum>,
) -> Pin<Box<dyn Future<Output = Result<File, AnyError>> + Send>> {
debug!("FileFetcher::fetch_remote() - specifier: {}", specifier);
if redirect_limit < 0 {
return futures::future::err(custom_error("Http", "Too many redirects."))
.boxed();
}
if let Err(err) = permissions.check_specifier(specifier) {
return futures::future::err(err).boxed();
}
maybe_checksum: Option<&LoaderChecksum>,
) -> Result<FileOrRedirect, AnyError> {
debug!(
"FileFetcher::fetch_remote_no_follow - specifier: {}",
specifier
);
if self.should_use_cache(specifier, cache_setting) {
match self.fetch_cached(specifier, maybe_checksum.clone(), redirect_limit)
if let Some(file_or_redirect) =
self.fetch_cached_no_follow(specifier, maybe_checksum)?
{
Ok(Some(file)) => {
return futures::future::ok(file).boxed();
}
Ok(None) => {}
Err(err) => {
return futures::future::err(err).boxed();
}
return Ok(file_or_redirect);
}
}
if *cache_setting == CacheSetting::Only {
return futures::future::err(custom_error(
return Err(custom_error(
"NotCached",
format!(
"Specifier not found in cache: \"{specifier}\", --cached-only is specified."
),
))
.boxed();
));
}
let mut maybe_progress_guard = None;
@ -346,12 +358,6 @@ impl FileFetcher {
.and_then(|key| self.http_cache.read_headers(&key).ok().flatten())
.and_then(|headers| headers.get("etag").cloned());
let maybe_auth_token = self.auth_tokens.get(specifier);
let specifier = specifier.clone();
let client = self.http_client.clone();
let file_fetcher = self.clone();
let cache_setting = cache_setting.clone();
// A single pass of fetch either yields code or yields a redirect, server
// error causes a single retry to avoid crashing hard on intermittent failures.
async fn handle_request_or_server_error(
retried: &mut bool,
@ -372,90 +378,75 @@ impl FileFetcher {
}
}
async move {
let mut maybe_etag = maybe_etag;
let mut retried = false;
let result = loop {
let result = match fetch_once(
&client,
FetchOnceArgs {
url: specifier.clone(),
maybe_accept: maybe_accept.clone(),
maybe_etag: maybe_etag.clone(),
maybe_auth_token: maybe_auth_token.clone(),
maybe_progress_guard: maybe_progress_guard.as_ref(),
},
)
.await?
{
FetchOnceResult::NotModified => {
let file = file_fetcher
.fetch_cached(&specifier, maybe_checksum.clone(), 10)?;
match file {
Some(file) => Ok(file),
None => {
// Someone may have deleted the body from the cache since
// it's currently stored in a separate file from the headers,
// so delete the etag and try again
if maybe_etag.is_some() {
debug!("Cache body not found. Trying again without etag.");
maybe_etag = None;
continue;
} else {
// should never happen
bail!("Your deno cache directory is in an unrecoverable state. Please delete it and try again.")
}
let mut maybe_etag = maybe_etag;
let mut retried = false; // retry intermittent failures
let result = loop {
let result = match fetch_no_follow(
&self.http_client,
FetchOnceArgs {
url: specifier.clone(),
maybe_accept: maybe_accept.map(ToOwned::to_owned),
maybe_etag: maybe_etag.clone(),
maybe_auth_token: maybe_auth_token.clone(),
maybe_progress_guard: maybe_progress_guard.as_ref(),
},
)
.await?
{
FetchOnceResult::NotModified => {
let file_or_redirect =
self.fetch_cached_no_follow(specifier, maybe_checksum)?;
match file_or_redirect {
Some(file_or_redirect) => Ok(file_or_redirect),
None => {
// Someone may have deleted the body from the cache since
// it's currently stored in a separate file from the headers,
// so delete the etag and try again
if maybe_etag.is_some() {
debug!("Cache body not found. Trying again without etag.");
maybe_etag = None;
continue;
} else {
// should never happen
bail!("Your deno cache directory is in an unrecoverable state. Please delete it and try again.")
}
}
}
FetchOnceResult::Redirect(redirect_url, headers) => {
file_fetcher.http_cache.set(&specifier, headers, &[])?;
file_fetcher
.fetch_remote(
&redirect_url,
permissions,
redirect_limit - 1,
maybe_accept,
&cache_setting,
maybe_checksum,
)
.await
}
FetchOnceResult::Redirect(redirect_url, headers) => {
self.http_cache.set(specifier, headers, &[])?;
Ok(FileOrRedirect::Redirect(redirect_url))
}
FetchOnceResult::Code(bytes, headers) => {
self.http_cache.set(specifier, headers.clone(), &bytes)?;
if let Some(checksum) = &maybe_checksum {
checksum.check_source(&bytes)?;
}
FetchOnceResult::Code(bytes, headers) => {
file_fetcher
.http_cache
.set(&specifier, headers.clone(), &bytes)?;
if let Some(checksum) = &maybe_checksum {
checksum.check_source(&bytes)?;
}
Ok(File {
specifier,
maybe_headers: Some(headers),
source: Arc::from(bytes),
})
}
FetchOnceResult::RequestError(err) => {
handle_request_or_server_error(&mut retried, &specifier, err)
.await?;
continue;
}
FetchOnceResult::ServerError(status) => {
handle_request_or_server_error(
&mut retried,
&specifier,
status.to_string(),
)
.await?;
continue;
}
};
break result;
Ok(FileOrRedirect::File(File {
specifier: specifier.clone(),
maybe_headers: Some(headers),
source: Arc::from(bytes),
}))
}
FetchOnceResult::RequestError(err) => {
handle_request_or_server_error(&mut retried, specifier, err).await?;
continue;
}
FetchOnceResult::ServerError(status) => {
handle_request_or_server_error(
&mut retried,
specifier,
status.to_string(),
)
.await?;
continue;
}
};
break result;
};
drop(maybe_progress_guard);
result
}
.boxed()
drop(maybe_progress_guard);
result
}
/// Returns if the cache should be used for a given specifier.
@ -508,7 +499,7 @@ impl FileFetcher {
pub async fn fetch(
&self,
specifier: &ModuleSpecifier,
permissions: PermissionsContainer,
permissions: &PermissionsContainer,
) -> Result<File, AnyError> {
self
.fetch_with_options(FetchOptions {
@ -516,7 +507,6 @@ impl FileFetcher {
permissions,
maybe_accept: None,
maybe_cache_setting: None,
maybe_checksum: None,
})
.await
}
@ -525,20 +515,68 @@ impl FileFetcher {
&self,
options: FetchOptions<'_>,
) -> Result<File, AnyError> {
self.fetch_with_options_and_max_redirect(options, 10).await
}
async fn fetch_with_options_and_max_redirect(
&self,
options: FetchOptions<'_>,
max_redirect: usize,
) -> Result<File, AnyError> {
let mut specifier = Cow::Borrowed(options.specifier);
for _ in 0..=max_redirect {
match self
.fetch_no_follow_with_options(FetchNoFollowOptions {
fetch_options: FetchOptions {
specifier: &specifier,
permissions: options.permissions,
maybe_accept: options.maybe_accept,
maybe_cache_setting: options.maybe_cache_setting,
},
maybe_checksum: None,
})
.await?
{
FileOrRedirect::File(file) => {
return Ok(file);
}
FileOrRedirect::Redirect(redirect_specifier) => {
specifier = Cow::Owned(redirect_specifier);
}
}
}
Err(custom_error("Http", "Too many redirects."))
}
/// Fetches without following redirects.
pub async fn fetch_no_follow_with_options(
&self,
options: FetchNoFollowOptions<'_>,
) -> Result<FileOrRedirect, AnyError> {
let maybe_checksum = options.maybe_checksum;
let options = options.fetch_options;
let specifier = options.specifier;
debug!("FileFetcher::fetch() - specifier: {}", specifier);
// note: this debug output is used by the tests
debug!(
"FileFetcher::fetch_no_follow_with_options - specifier: {}",
specifier
);
let scheme = get_validated_scheme(specifier)?;
options.permissions.check_specifier(specifier)?;
if let Some(file) = self.memory_files.get(specifier) {
Ok(file)
Ok(FileOrRedirect::File(file))
} else if scheme == "file" {
// we do not in memory cache files, as this would prevent files on the
// disk changing effecting things like workers and dynamic imports.
fetch_local(specifier)
fetch_local(specifier).map(FileOrRedirect::File)
} else if scheme == "data" {
self.fetch_data_url(specifier)
self.fetch_data_url(specifier).map(FileOrRedirect::File)
} else if scheme == "blob" {
self.fetch_blob_url(specifier).await
self
.fetch_blob_url(specifier)
.await
.map(FileOrRedirect::File)
} else if !self.allow_remote {
Err(custom_error(
"NoRemote",
@ -546,13 +584,11 @@ impl FileFetcher {
))
} else {
self
.fetch_remote(
.fetch_remote_no_follow(
specifier,
options.permissions,
10,
options.maybe_accept.map(String::from),
options.maybe_accept,
options.maybe_cache_setting.unwrap_or(&self.cache_setting),
options.maybe_checksum,
maybe_checksum,
)
.await
}
@ -605,7 +641,7 @@ struct FetchOnceArgs<'a> {
/// yields Code(ResultPayload).
/// If redirect occurs, does not follow and
/// yields Redirect(url).
async fn fetch_once<'a>(
async fn fetch_no_follow<'a>(
http_client: &HttpClient,
args: FetchOnceArgs<'a>,
) -> Result<FetchOnceResult, AnyError> {
@ -746,25 +782,26 @@ mod tests {
async fn test_fetch(specifier: &ModuleSpecifier) -> (File, FileFetcher) {
let (file_fetcher, _) = setup(CacheSetting::ReloadAll, None);
let result = file_fetcher
.fetch(specifier, PermissionsContainer::allow_all())
.fetch(specifier, &PermissionsContainer::allow_all())
.await;
assert!(result.is_ok());
(result.unwrap(), file_fetcher)
}
async fn test_fetch_remote(
async fn test_fetch_options_remote(
specifier: &ModuleSpecifier,
) -> (File, HashMap<String, String>) {
let _http_server_guard = test_util::http_server();
let (file_fetcher, _) = setup(CacheSetting::ReloadAll, None);
let result: Result<File, AnyError> = file_fetcher
.fetch_remote(
specifier,
PermissionsContainer::allow_all(),
.fetch_with_options_and_max_redirect(
FetchOptions {
specifier,
permissions: &PermissionsContainer::allow_all(),
maybe_accept: None,
maybe_cache_setting: Some(&file_fetcher.cache_setting),
},
1,
None,
&file_fetcher.cache_setting,
None,
)
.await;
let cache_key = file_fetcher.http_cache.cache_item_key(specifier).unwrap();
@ -788,7 +825,7 @@ mod tests {
) {
let url_str = format!("http://127.0.0.1:4545/encoding/{fixture}");
let specifier = resolve_url(&url_str).unwrap();
let (file, headers) = test_fetch_remote(&specifier).await;
let (file, headers) = test_fetch_options_remote(&specifier).await;
let (media_type, maybe_charset) =
deno_graph::source::resolve_media_type_and_charset_from_headers(
&specifier,
@ -857,7 +894,7 @@ mod tests {
file_fetcher.insert_memory_files(file.clone());
let result = file_fetcher
.fetch(&specifier, PermissionsContainer::allow_all())
.fetch(&specifier, &PermissionsContainer::allow_all())
.await;
assert!(result.is_ok());
let result_file = result.unwrap();
@ -870,7 +907,7 @@ mod tests {
let specifier = resolve_url("data:application/typescript;base64,ZXhwb3J0IGNvbnN0IGEgPSAiYSI7CgpleHBvcnQgZW51bSBBIHsKICBBLAogIEIsCiAgQywKfQo=").unwrap();
let result = file_fetcher
.fetch(&specifier, PermissionsContainer::allow_all())
.fetch(&specifier, &PermissionsContainer::allow_all())
.await;
assert!(result.is_ok());
let file = result.unwrap().into_text_decoded().unwrap();
@ -901,7 +938,7 @@ mod tests {
);
let result = file_fetcher
.fetch(&specifier, PermissionsContainer::allow_all())
.fetch(&specifier, &PermissionsContainer::allow_all())
.await;
assert!(result.is_ok());
let file = result.unwrap().into_text_decoded().unwrap();
@ -923,7 +960,7 @@ mod tests {
ModuleSpecifier::parse("http://localhost:4545/subdir/mod2.ts").unwrap();
let result = file_fetcher
.fetch(&specifier, PermissionsContainer::allow_all())
.fetch(&specifier, &PermissionsContainer::allow_all())
.await;
assert!(result.is_ok());
let file = result.unwrap().into_text_decoded().unwrap();
@ -943,7 +980,7 @@ mod tests {
.unwrap();
let result = file_fetcher_01
.fetch(&specifier, PermissionsContainer::allow_all())
.fetch(&specifier, &PermissionsContainer::allow_all())
.await;
assert!(result.is_ok());
let file = result.unwrap().into_text_decoded().unwrap();
@ -969,7 +1006,7 @@ mod tests {
.unwrap();
let result = file_fetcher_02
.fetch(&specifier, PermissionsContainer::allow_all())
.fetch(&specifier, &PermissionsContainer::allow_all())
.await;
assert!(result.is_ok());
let file = result.unwrap().into_text_decoded().unwrap();
@ -994,7 +1031,7 @@ mod tests {
None,
);
let result = file_fetcher
.fetch(&specifier, PermissionsContainer::allow_all())
.fetch(&specifier, &PermissionsContainer::allow_all())
.await;
assert!(result.is_ok());
let file = result.unwrap().into_text_decoded().unwrap();
@ -1027,7 +1064,7 @@ mod tests {
);
let result = file_fetcher
.fetch(&specifier, PermissionsContainer::allow_all())
.fetch(&specifier, &PermissionsContainer::allow_all())
.await;
assert!(result.is_ok());
let cache_key =
@ -1063,7 +1100,7 @@ mod tests {
None,
);
let result = file_fetcher
.fetch(&specifier, PermissionsContainer::allow_all())
.fetch(&specifier, &PermissionsContainer::allow_all())
.await;
assert!(result.is_ok());
@ -1102,7 +1139,7 @@ mod tests {
.unwrap();
let result = file_fetcher
.fetch(&specifier, PermissionsContainer::allow_all())
.fetch(&specifier, &PermissionsContainer::allow_all())
.await;
assert!(result.is_ok());
let file = result.unwrap();
@ -1143,7 +1180,7 @@ mod tests {
.unwrap();
let result = file_fetcher
.fetch(&specifier, PermissionsContainer::allow_all())
.fetch(&specifier, &PermissionsContainer::allow_all())
.await;
assert!(result.is_ok());
let file = result.unwrap();
@ -1203,7 +1240,7 @@ mod tests {
);
let result = file_fetcher
.fetch(&specifier, PermissionsContainer::allow_all())
.fetch(&specifier, &PermissionsContainer::allow_all())
.await;
assert!(result.is_ok());
@ -1242,7 +1279,7 @@ mod tests {
None,
);
let result = file_fetcher
.fetch(&redirected_specifier, PermissionsContainer::allow_all())
.fetch(&redirected_specifier, &PermissionsContainer::allow_all())
.await;
assert!(result.is_ok());
@ -1280,33 +1317,35 @@ mod tests {
.unwrap();
let result = file_fetcher
.fetch_remote(
&specifier,
PermissionsContainer::allow_all(),
.fetch_with_options_and_max_redirect(
FetchOptions {
specifier: &specifier,
permissions: &PermissionsContainer::allow_all(),
maybe_accept: None,
maybe_cache_setting: Some(&file_fetcher.cache_setting),
},
2,
None,
&file_fetcher.cache_setting,
None,
)
.await;
assert!(result.is_ok());
let result = file_fetcher
.fetch_remote(
&specifier,
PermissionsContainer::allow_all(),
.fetch_with_options_and_max_redirect(
FetchOptions {
specifier: &specifier,
permissions: &PermissionsContainer::allow_all(),
maybe_accept: None,
maybe_cache_setting: Some(&file_fetcher.cache_setting),
},
1,
None,
&file_fetcher.cache_setting,
None,
)
.await;
assert!(result.is_err());
let result = file_fetcher.fetch_cached(&specifier, None, 2);
let result = file_fetcher.fetch_cached(&specifier, 2);
assert!(result.is_ok());
let result = file_fetcher.fetch_cached(&specifier, None, 1);
let result = file_fetcher.fetch_cached(&specifier, 1);
assert!(result.is_err());
}
@ -1323,7 +1362,7 @@ mod tests {
.unwrap();
let result = file_fetcher
.fetch(&specifier, PermissionsContainer::allow_all())
.fetch(&specifier, &PermissionsContainer::allow_all())
.await;
assert!(result.is_ok());
let file = result.unwrap();
@ -1369,7 +1408,7 @@ mod tests {
resolve_url("http://localhost:4545/run/002_hello.ts").unwrap();
let result = file_fetcher
.fetch(&specifier, PermissionsContainer::allow_all())
.fetch(&specifier, &PermissionsContainer::allow_all())
.await;
assert!(result.is_err());
let err = result.unwrap_err();
@ -1402,7 +1441,7 @@ mod tests {
resolve_url("http://localhost:4545/run/002_hello.ts").unwrap();
let result = file_fetcher_01
.fetch(&specifier, PermissionsContainer::allow_all())
.fetch(&specifier, &PermissionsContainer::allow_all())
.await;
assert!(result.is_err());
let err = result.unwrap_err();
@ -1410,12 +1449,12 @@ mod tests {
assert_eq!(get_custom_error_class(&err), Some("NotCached"));
let result = file_fetcher_02
.fetch(&specifier, PermissionsContainer::allow_all())
.fetch(&specifier, &PermissionsContainer::allow_all())
.await;
assert!(result.is_ok());
let result = file_fetcher_01
.fetch(&specifier, PermissionsContainer::allow_all())
.fetch(&specifier, &PermissionsContainer::allow_all())
.await;
assert!(result.is_ok());
}
@ -1427,7 +1466,7 @@ mod tests {
let specifier = ModuleSpecifier::from_file_path(&fixture_path).unwrap();
fs::write(fixture_path.clone(), r#"console.log("hello deno");"#).unwrap();
let result = file_fetcher
.fetch(&specifier, PermissionsContainer::allow_all())
.fetch(&specifier, &PermissionsContainer::allow_all())
.await;
assert!(result.is_ok());
let file = result.unwrap().into_text_decoded().unwrap();
@ -1435,7 +1474,7 @@ mod tests {
fs::write(fixture_path, r#"console.log("goodbye deno");"#).unwrap();
let result = file_fetcher
.fetch(&specifier, PermissionsContainer::allow_all())
.fetch(&specifier, &PermissionsContainer::allow_all())
.await;
assert!(result.is_ok());
let file = result.unwrap().into_text_decoded().unwrap();
@ -1451,7 +1490,7 @@ mod tests {
let specifier =
ModuleSpecifier::parse("http://localhost:4545/dynamic").unwrap();
let result = file_fetcher
.fetch(&specifier, PermissionsContainer::allow_all())
.fetch(&specifier, &PermissionsContainer::allow_all())
.await;
assert!(result.is_ok());
let file = result.unwrap();
@ -1460,7 +1499,7 @@ mod tests {
let (file_fetcher, _) =
setup(CacheSetting::RespectHeaders, Some(temp_dir.clone()));
let result = file_fetcher
.fetch(&specifier, PermissionsContainer::allow_all())
.fetch(&specifier, &PermissionsContainer::allow_all())
.await;
assert!(result.is_ok());
let file = result.unwrap();
@ -1478,7 +1517,7 @@ mod tests {
let specifier =
ModuleSpecifier::parse("http://localhost:4545/dynamic_cache").unwrap();
let result = file_fetcher
.fetch(&specifier, PermissionsContainer::allow_all())
.fetch(&specifier, &PermissionsContainer::allow_all())
.await;
assert!(result.is_ok());
let file = result.unwrap();
@ -1487,7 +1526,7 @@ mod tests {
let (file_fetcher, _) =
setup(CacheSetting::RespectHeaders, Some(temp_dir.clone()));
let result = file_fetcher
.fetch(&specifier, PermissionsContainer::allow_all())
.fetch(&specifier, &PermissionsContainer::allow_all())
.await;
assert!(result.is_ok());
let file = result.unwrap();
@ -1551,7 +1590,7 @@ mod tests {
// Relies on external http server. See target/debug/test_server
let url = Url::parse("http://127.0.0.1:4545/assets/fixture.json").unwrap();
let client = create_test_client();
let result = fetch_once(
let result = fetch_no_follow(
&client,
FetchOnceArgs {
url,
@ -1579,7 +1618,7 @@ mod tests {
let url = Url::parse("http://127.0.0.1:4545/run/import_compression/gziped")
.unwrap();
let client = create_test_client();
let result = fetch_once(
let result = fetch_no_follow(
&client,
FetchOnceArgs {
url,
@ -1608,7 +1647,7 @@ mod tests {
let _http_server_guard = test_util::http_server();
let url = Url::parse("http://127.0.0.1:4545/etag_script.ts").unwrap();
let client = create_test_client();
let result = fetch_once(
let result = fetch_no_follow(
&client,
FetchOnceArgs {
url: url.clone(),
@ -1631,7 +1670,7 @@ mod tests {
panic!();
}
let res = fetch_once(
let res = fetch_no_follow(
&client,
FetchOnceArgs {
url,
@ -1652,7 +1691,7 @@ mod tests {
let url = Url::parse("http://127.0.0.1:4545/run/import_compression/brotli")
.unwrap();
let client = create_test_client();
let result = fetch_once(
let result = fetch_no_follow(
&client,
FetchOnceArgs {
url,
@ -1683,7 +1722,7 @@ mod tests {
// Relies on external http server. See target/debug/test_server
let url = Url::parse("http://127.0.0.1:4545/echo_accept").unwrap();
let client = create_test_client();
let result = fetch_once(
let result = fetch_no_follow(
&client,
FetchOnceArgs {
url,
@ -1702,7 +1741,7 @@ mod tests {
}
#[tokio::test]
async fn test_fetch_once_with_redirect() {
async fn test_fetch_no_follow_with_redirect() {
let _http_server_guard = test_util::http_server();
// Relies on external http server. See target/debug/test_server
let url = Url::parse("http://127.0.0.1:4546/assets/fixture.json").unwrap();
@ -1710,7 +1749,7 @@ mod tests {
let target_url =
Url::parse("http://localhost:4545/assets/fixture.json").unwrap();
let client = create_test_client();
let result = fetch_once(
let result = fetch_no_follow(
&client,
FetchOnceArgs {
url,
@ -1747,7 +1786,7 @@ mod tests {
)
.unwrap(),
);
let result = fetch_once(
let result = fetch_no_follow(
&client,
FetchOnceArgs {
url,
@ -1796,7 +1835,7 @@ mod tests {
.unwrap(),
);
let result = fetch_once(
let result = fetch_no_follow(
&client,
FetchOnceArgs {
url,
@ -1860,7 +1899,7 @@ mod tests {
.unwrap(),
);
let result = fetch_once(
let result = fetch_no_follow(
&client,
FetchOnceArgs {
url,
@ -1915,7 +1954,7 @@ mod tests {
)
.unwrap(),
);
let result = fetch_once(
let result = fetch_no_follow(
&client,
FetchOnceArgs {
url,
@ -1958,7 +1997,7 @@ mod tests {
)
.unwrap(),
);
let result = fetch_once(
let result = fetch_no_follow(
&client,
FetchOnceArgs {
url: url.clone(),
@ -1982,7 +2021,7 @@ mod tests {
panic!();
}
let res = fetch_once(
let res = fetch_no_follow(
&client,
FetchOnceArgs {
url,
@ -2018,7 +2057,7 @@ mod tests {
)
.unwrap(),
);
let result = fetch_once(
let result = fetch_no_follow(
&client,
FetchOnceArgs {
url,
@ -2049,7 +2088,7 @@ mod tests {
let url_str = "http://127.0.0.1:4545/bad_redirect";
let url = Url::parse(url_str).unwrap();
let client = create_test_client();
let result = fetch_once(
let result = fetch_no_follow(
&client,
FetchOnceArgs {
url,
@ -2072,7 +2111,7 @@ mod tests {
let url_str = "http://127.0.0.1:4545/server_error";
let url = Url::parse(url_str).unwrap();
let client = create_test_client();
let result = fetch_once(
let result = fetch_no_follow(
&client,
FetchOnceArgs {
url,
@ -2097,7 +2136,7 @@ mod tests {
let url_str = "http://127.0.0.1:9999/";
let url = Url::parse(url_str).unwrap();
let client = create_test_client();
let result = fetch_once(
let result = fetch_no_follow(
&client,
FetchOnceArgs {
url,

12
cli/graph_util.rs
View File

@ -451,17 +451,17 @@ impl ModuleGraphBuilder {
options.roots,
loader.as_mut_loader(),
deno_graph::BuildOptions {
is_dynamic: options.is_dynamic,
jsr_url_provider: &CliJsrUrlProvider,
passthrough_jsr_specifiers: false,
executor: Default::default(),
imports: maybe_imports,
resolver: Some(graph_resolver),
is_dynamic: options.is_dynamic,
passthrough_jsr_specifiers: false,
workspace_members: &workspace_members,
executor: Default::default(),
file_system: &DenoGraphFsAdapter(self.fs.as_ref()),
jsr_url_provider: &CliJsrUrlProvider,
npm_resolver: Some(graph_npm_resolver),
module_analyzer: &analyzer,
reporter: maybe_file_watcher_reporter,
workspace_members: &workspace_members,
resolver: Some(graph_resolver),
},
)
.await

4
cli/jsr.rs
View File

@ -234,7 +234,7 @@ impl JsrFetchResolver {
let meta_url = jsr_url().join(&format!("{}/meta.json", name)).ok()?;
let file = self
.file_fetcher
.fetch(&meta_url, PermissionsContainer::allow_all())
.fetch(&meta_url, &PermissionsContainer::allow_all())
.await
.ok()?;
serde_json::from_slice::<JsrPackageInfo>(&file.source).ok()
@ -257,7 +257,7 @@ impl JsrFetchResolver {
.ok()?;
let file = self
.file_fetcher
.fetch(&meta_url, PermissionsContainer::allow_all())
.fetch(&meta_url, &PermissionsContainer::allow_all())
.await
.ok()?;
partial_jsr_package_version_info_from_slice(&file.source).ok()

2
cli/lsp/config.rs
View File

@ -1393,7 +1393,7 @@ impl ConfigData {
if import_map_value.is_none() {
if let Some(file_fetcher) = file_fetcher {
let fetch_result = file_fetcher
.fetch(specifier, PermissionsContainer::allow_all())
.fetch(specifier, &PermissionsContainer::allow_all())
.await;
let value_result = fetch_result.and_then(|f| {
serde_json::from_slice::<Value>(&f.source).map_err(|e| e.into())

2
cli/lsp/jsr.rs
View File

@ -51,7 +51,7 @@ impl PackageSearchApi for CliJsrSearchApi {
search_url.query_pairs_mut().append_pair("query", query);
let file = self
.file_fetcher
.fetch(&search_url, PermissionsContainer::allow_all())
.fetch(&search_url, &PermissionsContainer::allow_all())
.await?
.into_text_decoded()?;
let names = Arc::new(parse_jsr_search_response(&file.source)?);

2
cli/lsp/npm.rs
View File

@ -48,7 +48,7 @@ impl PackageSearchApi for CliNpmSearchApi {
.append_pair("text", &format!("{} boost-exact:false", query));
let file = self
.file_fetcher
.fetch(&search_url, PermissionsContainer::allow_all())
.fetch(&search_url, &PermissionsContainer::allow_all())
.await?
.into_text_decoded()?;
let names = Arc::new(parse_npm_search_response(&file.source)?);

11
cli/lsp/registries.rs
View File

@ -512,10 +512,9 @@ impl ModuleRegistry {
.file_fetcher
.fetch_with_options(FetchOptions {
specifier,
permissions: PermissionsContainer::allow_all(),
permissions: &PermissionsContainer::allow_all(),
maybe_accept: Some("application/vnd.deno.reg.v2+json, application/vnd.deno.reg.v1+json;q=0.9, application/json;q=0.8"),
maybe_cache_setting: None,
maybe_checksum: None,
})
.await;
// if there is an error fetching, we will cache an empty file, so that
@ -609,7 +608,7 @@ impl ModuleRegistry {
.ok()?;
let file = self
.file_fetcher
.fetch(&endpoint, PermissionsContainer::allow_all())
.fetch(&endpoint, &PermissionsContainer::allow_all())
.await
.ok()?
.into_text_decoded()
@ -975,7 +974,7 @@ impl ModuleRegistry {
let specifier = Url::parse(url).ok()?;
let file = self
.file_fetcher
.fetch(&specifier, PermissionsContainer::allow_all())
.fetch(&specifier, &PermissionsContainer::allow_all())
.await
.ok()?
.into_text_decoded()
@ -1034,7 +1033,7 @@ impl ModuleRegistry {
let specifier = ModuleSpecifier::parse(url).ok()?;
let file = self
.file_fetcher
.fetch(&specifier, PermissionsContainer::allow_all())
.fetch(&specifier, &PermissionsContainer::allow_all())
.await
.map_err(|err| {
error!(
@ -1072,7 +1071,7 @@ impl ModuleRegistry {
.ok()?;
let file = self
.file_fetcher
.fetch(&specifier, PermissionsContainer::allow_all())
.fetch(&specifier, &PermissionsContainer::allow_all())
.await
.map_err(|err| {
error!(

2
cli/npm/mod.rs
View File

@ -142,7 +142,7 @@ impl NpmFetchResolver {
let info_url = npm_registry_url().join(name).ok()?;
let file = self
.file_fetcher
.fetch(&info_url, PermissionsContainer::allow_all())
.fetch(&info_url, &PermissionsContainer::allow_all())
.await
.ok()?;
serde_json::from_slice::<NpmPackageInfo>(&file.source).ok()

2
cli/tools/coverage/mod.rs
View File

@ -529,7 +529,7 @@ pub async fn cover_files(
file_fetcher.get_source(&module_specifier)
} else {
file_fetcher
.fetch_cached(&module_specifier, None, 10)
.fetch_cached(&module_specifier, 10)
.with_context(|| {
format!("Failed to fetch \"{module_specifier}\" from cache.")
})?

12
cli/tools/doc.rs
View File

@ -54,17 +54,17 @@ async fn generate_doc_nodes_for_builtin_types(
vec![source_file_specifier.clone()],
&loader,
deno_graph::BuildOptions {
module_analyzer: analyzer,
file_system: &NullFileSystem,
is_dynamic: false,
imports: Vec::new(),
executor: Default::default(),
jsr_url_provider: Default::default(),
is_dynamic: false,
passthrough_jsr_specifiers: false,
workspace_members: &[],
executor: Default::default(),
file_system: &NullFileSystem,
jsr_url_provider: Default::default(),
module_analyzer: analyzer,
npm_resolver: None,
reporter: None,
resolver: None,
workspace_members: &[],
},
)
.await;

2
cli/tools/repl/mod.rs
View File

@ -145,7 +145,7 @@ async fn read_eval_file(
deno_core::resolve_url_or_path(eval_file, cli_options.initial_cwd())?;
let file = file_fetcher
.fetch(&specifier, PermissionsContainer::allow_all())
.fetch(&specifier, &PermissionsContainer::allow_all())
.await?;
Ok(file.into_text_decoded()?.source)

4
cli/tools/test/mod.rs
View File

@ -1277,7 +1277,7 @@ async fn fetch_inline_files(
for specifier in specifiers {
let fetch_permissions = PermissionsContainer::allow_all();
let file = file_fetcher
.fetch(&specifier, fetch_permissions)
.fetch(&specifier, &fetch_permissions)
.await?
.into_text_decoded()?;
@ -1688,7 +1688,7 @@ async fn fetch_specifiers_with_test_mode(
for (specifier, mode) in &mut specifiers_with_mode {
let file = file_fetcher
.fetch(specifier, PermissionsContainer::allow_all())
.fetch(specifier, &PermissionsContainer::allow_all())
.await?;
let (media_type, _) = file.resolve_media_type_and_charset();

8
cli/tools/vendor/test.rs vendored
View File

@ -116,7 +116,13 @@ impl Loader for TestLoader {
specifier: &ModuleSpecifier,
_options: deno_graph::source::LoadOptions,
) -> LoadFuture {
let specifier = self.redirects.get(specifier).unwrap_or(specifier);
if let Some(redirect) = self.redirects.get(specifier) {
return Box::pin(futures::future::ready(Ok(Some(
LoadResponse::Redirect {
specifier: redirect.clone(),
},
))));
}
let result = self.files.get(specifier).map(|result| match result {
Ok(result) => Ok(LoadResponse::Module {
specifier: specifier.clone(),

2
tests/integration/bench_tests.rs
View File

@ -280,7 +280,7 @@ fn conditionally_loads_type_graph() {
.new_command()
.args("bench --reload -L debug run/type_directives_js_main.js")
.run();
output.assert_matches_text("[WILDCARD] - FileFetcher::fetch() - specifier: file:///[WILDCARD]/subdir/type_reference.d.ts[WILDCARD]");
output.assert_matches_text("[WILDCARD] - FileFetcher::fetch_no_follow_with_options - specifier: file:///[WILDCARD]/subdir/type_reference.d.ts[WILDCARD]");
let output = context
.new_command()
.args("bench --reload -L debug --no-check run/type_directives_js_main.js")

2
tests/integration/cache_tests.rs
View File

@ -177,5 +177,5 @@ fn loads_type_graph() {
.new_command()
.args("cache --reload -L debug run/type_directives_js_main.js")
.run();
output.assert_matches_text("[WILDCARD] - FileFetcher::fetch() - specifier: file:///[WILDCARD]/subdir/type_reference.d.ts[WILDCARD]");
output.assert_matches_text("[WILDCARD] - FileFetcher::fetch_no_follow_with_options - specifier: file:///[WILDCARD]/subdir/type_reference.d.ts[WILDCARD]");
}

2
tests/integration/run_tests.rs
View File

@ -1686,7 +1686,7 @@ fn type_directives_js_main() {
.new_command()
.args("run --reload -L debug --check run/type_directives_js_main.js")
.run();
output.assert_matches_text("[WILDCARD] - FileFetcher::fetch() - specifier: file:///[WILDCARD]/subdir/type_reference.d.ts[WILDCARD]");
output.assert_matches_text("[WILDCARD] - FileFetcher::fetch_no_follow_with_options - specifier: file:///[WILDCARD]/subdir/type_reference.d.ts[WILDCARD]");
let output = context
.new_command()
.args("run --reload -L debug run/type_directives_js_main.js")

2
tests/integration/test_tests.rs
View File

@ -668,7 +668,7 @@ fn conditionally_loads_type_graph() {
.new_command()
.args("test --reload -L debug run/type_directives_js_main.js")
.run();
output.assert_matches_text("[WILDCARD] - FileFetcher::fetch() - specifier: file:///[WILDCARD]/subdir/type_reference.d.ts[WILDCARD]");
output.assert_matches_text("[WILDCARD] - FileFetcher::fetch_no_follow_with_options - specifier: file:///[WILDCARD]/subdir/type_reference.d.ts[WILDCARD]");
let output = context
.new_command()
.args("test --reload -L debug --no-check run/type_directives_js_main.js")

4
tests/specs/info/multiple_redirects/__test__.jsonc Normal file
View File

@ -0,0 +1,4 @@
{
"args": "info --json main.ts",
"output": "main.out"
}

51
tests/specs/info/multiple_redirects/main.out Normal file
View File

@ -0,0 +1,51 @@
Download http://localhost:4548/subdir/redirects/redirect1.js
Download http://localhost:4546/subdir/redirects/redirect1.js
Download http://localhost:4545/subdir/redirects/redirect1.js
{
"roots": [
"file:///[WILDLINE]/multiple_redirects/main.ts"
],
"modules": [
{
"kind": "esm",
"dependencies": [
{
"specifier": "http://localhost:4548/subdir/redirects/redirect1.js",
"code": {
"specifier": "http://localhost:4548/subdir/redirects/redirect1.js",
"span": {
"start": {
"line": 0,
"character": 22
},
"end": {
"line": 0,
"character": 75
}
}
}
}
],
"local": "[WILDLINE]main.ts",
"emit": null,
"map": null,
"size": 97,
"mediaType": "TypeScript",
"specifier": "file:///[WILDLINE]/multiple_redirects/main.ts"
},
{
"kind": "esm",
"local": "[WILDLINE]",
"emit": null,
"map": null,
"size": 27,
"mediaType": "JavaScript",
"specifier": "http://localhost:4545/subdir/redirects/redirect1.js"
}
],
"redirects": {
"http://localhost:4546/subdir/redirects/redirect1.js": "http://localhost:4545/subdir/redirects/redirect1.js",
"http://localhost:4548/subdir/redirects/redirect1.js": "http://localhost:4546/subdir/redirects/redirect1.js"
},
"npmPackages": {}
}

2
tests/specs/info/multiple_redirects/main.ts Normal file
View File

@ -0,0 +1,2 @@
import { value } from "http://localhost:4548/subdir/redirects/redirect1.js";
console.log(value);