Attempt at bypassing FA's Cloudflare page.
This commit is contained in:
parent
0acc9cedd0
commit
cabb286f2f
|
@ -1 +1 @@
|
|||
target
|
||||
**/target/
|
||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -36,6 +36,7 @@ pickledb = "0.4"
|
|||
|
||||
telegram = { path = "./telegram", features = ["trace"] }
|
||||
fautil = { path = "./fautil", features = ["trace"] }
|
||||
cfscrape = { git = "https://git.huefox.com/syfaro/cfscrape-rust.git" }
|
||||
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = "0.2"
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
FROM rustlang/rust:nightly-slim AS builder
|
||||
WORKDIR /src
|
||||
RUN apt-get update -y && apt-get install pkg-config libssl-dev -y
|
||||
RUN apt-get update -y && apt-get install pkg-config libssl-dev python3 python3-dev -y
|
||||
COPY . .
|
||||
RUN cargo install --root / --path .
|
||||
|
||||
|
@ -10,5 +10,5 @@ EXPOSE 8080
|
|||
WORKDIR /app
|
||||
COPY --from=builder /src/langs ./langs
|
||||
COPY --from=builder /bin/foxbot /bin/foxbot
|
||||
RUN apt-get update -y && apt-get install libssl-dev ca-certificates -y
|
||||
RUN apt-get update -y && apt-get install libssl-dev ca-certificates python3 python3-pip -y && pip3 install cfscrape
|
||||
CMD ["/bin/foxbot"]
|
||||
|
|
61
src/sites.rs
61
src/sites.rs
|
@ -46,6 +46,8 @@ pub enum SiteError {
|
|||
Missing(std::option::NoneError),
|
||||
#[fail(display = "twitter error")]
|
||||
Twitter(egg_mode::error::Error),
|
||||
#[fail(display = "python error")]
|
||||
Python(String),
|
||||
}
|
||||
|
||||
impl From<reqwest::Error> for SiteError {
|
||||
|
@ -72,6 +74,14 @@ impl From<std::option::NoneError> for SiteError {
|
|||
}
|
||||
}
|
||||
|
||||
impl From<cfscrape::Error> for SiteError {
|
||||
fn from(e: cfscrape::Error) -> Self {
|
||||
SiteError::Python(match e {
|
||||
cfscrape::Error::Python(err) => format!("{:?}", err),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait Site {
|
||||
fn name(&self) -> &'static str;
|
||||
|
@ -394,7 +404,7 @@ fn get_best_video(media: &egg_mode::entities::MediaEntity) -> Option<&str> {
|
|||
}
|
||||
|
||||
pub struct FurAffinity {
|
||||
cookies: (String, String),
|
||||
cookies: std::collections::HashMap<String, String>,
|
||||
fapi: fautil::FAUtil,
|
||||
submission: scraper::Selector,
|
||||
client: reqwest::Client,
|
||||
|
@ -402,8 +412,13 @@ pub struct FurAffinity {
|
|||
|
||||
impl FurAffinity {
|
||||
pub fn new(cookies: (String, String), util_api: String) -> Self {
|
||||
let mut c = std::collections::HashMap::new();
|
||||
|
||||
c.insert("a".into(), cookies.0);
|
||||
c.insert("b".into(), cookies.1);
|
||||
|
||||
Self {
|
||||
cookies,
|
||||
cookies: c,
|
||||
fapi: fautil::FAUtil::new(util_api),
|
||||
submission: scraper::Selector::parse("#submissionImg").unwrap(),
|
||||
client: reqwest::Client::new(),
|
||||
|
@ -436,23 +451,47 @@ impl FurAffinity {
|
|||
}))
|
||||
}
|
||||
|
||||
async fn load_submission(&self, url: &str) -> Result<Option<PostInfo>, SiteError> {
|
||||
let cookies = vec![
|
||||
format!("a={}", self.cookies.0),
|
||||
format!("b={}", self.cookies.1),
|
||||
]
|
||||
.join("; ");
|
||||
fn stringify_cookies(&self) -> String {
|
||||
let mut cookies = vec![];
|
||||
for (name, value) in &self.cookies {
|
||||
cookies.push(format!("{}={}", name, value));
|
||||
}
|
||||
cookies.join("; ")
|
||||
}
|
||||
|
||||
async fn load_submission(&mut self, url: &str) -> Result<Option<PostInfo>, SiteError> {
|
||||
let resp = self
|
||||
.client
|
||||
.get(url)
|
||||
.header(header::COOKIE, cookies)
|
||||
.header(header::COOKIE, self.stringify_cookies())
|
||||
.header(header::USER_AGENT, USER_AGENT)
|
||||
.send()
|
||||
.await?
|
||||
.text()
|
||||
.await?;
|
||||
|
||||
let resp = if resp.status() == 429 || resp.status() == 503 {
|
||||
let cfscrape::CfscrapeData { cookies, .. } =
|
||||
cfscrape::get_cookie_string(url, Some(USER_AGENT))?;
|
||||
let cookies = cookies.split("; ");
|
||||
for cookie in cookies {
|
||||
let mut parts = cookie.split("=");
|
||||
let name = parts.next().expect("Missing cookie name");
|
||||
let value = parts.next().expect("Missing cookie value");
|
||||
|
||||
self.cookies.insert(name.into(), value.into());
|
||||
}
|
||||
|
||||
self.client
|
||||
.get(url)
|
||||
.header(header::COOKIE, self.stringify_cookies())
|
||||
.header(header::USER_AGENT, USER_AGENT)
|
||||
.send()
|
||||
.await?
|
||||
.text()
|
||||
.await?
|
||||
} else {
|
||||
resp.text().await?
|
||||
};
|
||||
|
||||
let body = scraper::Html::parse_document(&resp);
|
||||
let img = match body.select(&self.submission).next() {
|
||||
Some(img) => img,
|
||||
|
|
Loading…
Reference in New Issue