Attempt at bypassing FA's Cloudflare page.
This commit is contained in:
parent
0acc9cedd0
commit
cabb286f2f
|
@ -1 +1 @@
|
||||||
target
|
**/target/
|
||||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -36,6 +36,7 @@ pickledb = "0.4"
|
||||||
|
|
||||||
telegram = { path = "./telegram", features = ["trace"] }
|
telegram = { path = "./telegram", features = ["trace"] }
|
||||||
fautil = { path = "./fautil", features = ["trace"] }
|
fautil = { path = "./fautil", features = ["trace"] }
|
||||||
|
cfscrape = { git = "https://git.huefox.com/syfaro/cfscrape-rust.git" }
|
||||||
|
|
||||||
tracing = "0.1"
|
tracing = "0.1"
|
||||||
tracing-subscriber = "0.2"
|
tracing-subscriber = "0.2"
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
FROM rustlang/rust:nightly-slim AS builder
|
FROM rustlang/rust:nightly-slim AS builder
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
RUN apt-get update -y && apt-get install pkg-config libssl-dev -y
|
RUN apt-get update -y && apt-get install pkg-config libssl-dev python3 python3-dev -y
|
||||||
COPY . .
|
COPY . .
|
||||||
RUN cargo install --root / --path .
|
RUN cargo install --root / --path .
|
||||||
|
|
||||||
|
@ -10,5 +10,5 @@ EXPOSE 8080
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
COPY --from=builder /src/langs ./langs
|
COPY --from=builder /src/langs ./langs
|
||||||
COPY --from=builder /bin/foxbot /bin/foxbot
|
COPY --from=builder /bin/foxbot /bin/foxbot
|
||||||
RUN apt-get update -y && apt-get install libssl-dev ca-certificates -y
|
RUN apt-get update -y && apt-get install libssl-dev ca-certificates python3 python3-pip -y && pip3 install cfscrape
|
||||||
CMD ["/bin/foxbot"]
|
CMD ["/bin/foxbot"]
|
||||||
|
|
61
src/sites.rs
61
src/sites.rs
|
@ -46,6 +46,8 @@ pub enum SiteError {
|
||||||
Missing(std::option::NoneError),
|
Missing(std::option::NoneError),
|
||||||
#[fail(display = "twitter error")]
|
#[fail(display = "twitter error")]
|
||||||
Twitter(egg_mode::error::Error),
|
Twitter(egg_mode::error::Error),
|
||||||
|
#[fail(display = "python error")]
|
||||||
|
Python(String),
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<reqwest::Error> for SiteError {
|
impl From<reqwest::Error> for SiteError {
|
||||||
|
@ -72,6 +74,14 @@ impl From<std::option::NoneError> for SiteError {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl From<cfscrape::Error> for SiteError {
|
||||||
|
fn from(e: cfscrape::Error) -> Self {
|
||||||
|
SiteError::Python(match e {
|
||||||
|
cfscrape::Error::Python(err) => format!("{:?}", err),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
pub trait Site {
|
pub trait Site {
|
||||||
fn name(&self) -> &'static str;
|
fn name(&self) -> &'static str;
|
||||||
|
@ -394,7 +404,7 @@ fn get_best_video(media: &egg_mode::entities::MediaEntity) -> Option<&str> {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct FurAffinity {
|
pub struct FurAffinity {
|
||||||
cookies: (String, String),
|
cookies: std::collections::HashMap<String, String>,
|
||||||
fapi: fautil::FAUtil,
|
fapi: fautil::FAUtil,
|
||||||
submission: scraper::Selector,
|
submission: scraper::Selector,
|
||||||
client: reqwest::Client,
|
client: reqwest::Client,
|
||||||
|
@ -402,8 +412,13 @@ pub struct FurAffinity {
|
||||||
|
|
||||||
impl FurAffinity {
|
impl FurAffinity {
|
||||||
pub fn new(cookies: (String, String), util_api: String) -> Self {
|
pub fn new(cookies: (String, String), util_api: String) -> Self {
|
||||||
|
let mut c = std::collections::HashMap::new();
|
||||||
|
|
||||||
|
c.insert("a".into(), cookies.0);
|
||||||
|
c.insert("b".into(), cookies.1);
|
||||||
|
|
||||||
Self {
|
Self {
|
||||||
cookies,
|
cookies: c,
|
||||||
fapi: fautil::FAUtil::new(util_api),
|
fapi: fautil::FAUtil::new(util_api),
|
||||||
submission: scraper::Selector::parse("#submissionImg").unwrap(),
|
submission: scraper::Selector::parse("#submissionImg").unwrap(),
|
||||||
client: reqwest::Client::new(),
|
client: reqwest::Client::new(),
|
||||||
|
@ -436,23 +451,47 @@ impl FurAffinity {
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn load_submission(&self, url: &str) -> Result<Option<PostInfo>, SiteError> {
|
fn stringify_cookies(&self) -> String {
|
||||||
let cookies = vec![
|
let mut cookies = vec![];
|
||||||
format!("a={}", self.cookies.0),
|
for (name, value) in &self.cookies {
|
||||||
format!("b={}", self.cookies.1),
|
cookies.push(format!("{}={}", name, value));
|
||||||
]
|
}
|
||||||
.join("; ");
|
cookies.join("; ")
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn load_submission(&mut self, url: &str) -> Result<Option<PostInfo>, SiteError> {
|
||||||
let resp = self
|
let resp = self
|
||||||
.client
|
.client
|
||||||
.get(url)
|
.get(url)
|
||||||
.header(header::COOKIE, cookies)
|
.header(header::COOKIE, self.stringify_cookies())
|
||||||
.header(header::USER_AGENT, USER_AGENT)
|
.header(header::USER_AGENT, USER_AGENT)
|
||||||
.send()
|
.send()
|
||||||
.await?
|
|
||||||
.text()
|
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
|
let resp = if resp.status() == 429 || resp.status() == 503 {
|
||||||
|
let cfscrape::CfscrapeData { cookies, .. } =
|
||||||
|
cfscrape::get_cookie_string(url, Some(USER_AGENT))?;
|
||||||
|
let cookies = cookies.split("; ");
|
||||||
|
for cookie in cookies {
|
||||||
|
let mut parts = cookie.split("=");
|
||||||
|
let name = parts.next().expect("Missing cookie name");
|
||||||
|
let value = parts.next().expect("Missing cookie value");
|
||||||
|
|
||||||
|
self.cookies.insert(name.into(), value.into());
|
||||||
|
}
|
||||||
|
|
||||||
|
self.client
|
||||||
|
.get(url)
|
||||||
|
.header(header::COOKIE, self.stringify_cookies())
|
||||||
|
.header(header::USER_AGENT, USER_AGENT)
|
||||||
|
.send()
|
||||||
|
.await?
|
||||||
|
.text()
|
||||||
|
.await?
|
||||||
|
} else {
|
||||||
|
resp.text().await?
|
||||||
|
};
|
||||||
|
|
||||||
let body = scraper::Html::parse_document(&resp);
|
let body = scraper::Html::parse_document(&resp);
|
||||||
let img = match body.select(&self.submission).next() {
|
let img = match body.select(&self.submission).next() {
|
||||||
Some(img) => img,
|
Some(img) => img,
|
||||||
|
|
Loading…
Reference in New Issue