diff --git a/src/agenda_cultural/api.rs b/src/agenda_cultural/api.rs index 8049e95..878a7d9 100644 --- a/src/agenda_cultural/api.rs +++ b/src/agenda_cultural/api.rs @@ -10,7 +10,7 @@ use reqwest_retry::Jitter::Bounded; use reqwest_retry::RetryTransientMiddleware; use scraper::{Html, Selector}; use std::cmp::Ordering; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashSet}; use std::ops::Add; use std::time::Duration; use strum::Display; @@ -22,7 +22,14 @@ const EVENT_TYPE: &str = "event"; const DATE_PRINT_FORMAT: &str = "%Y-%m-%d"; lazy_static! { - static ref REST_CLIENT: ClientWithMiddleware = ClientBuilder::new(Client::new()) + static ref REST_CLIENT: ClientWithMiddleware = ClientBuilder::new( + Client::builder() + .timeout(Duration::from_secs(60)) + .connect_timeout(Duration::from_secs(10)) + .read_timeout(Duration::from_secs(30)) + .build() + .expect("Failed building the HTTP client") + ) .with(RetryTransientMiddleware::new_with_policy( ExponentialBackoff::builder() .jitter(Bounded) @@ -60,7 +67,7 @@ impl AgendaCulturalAPI { } let category: &'static str = category.into(); - let parsed_response = Self::get_events_by_category(amount_per_page, category) + let mut parsed_response = Self::get_events_by_category(amount_per_page, category) .await .inspect_err(|err| { error!("Failed to get events by category '{}': {}", category, err) @@ -68,11 +75,29 @@ impl AgendaCulturalAPI { info!("Fetched {} events", parsed_response.len()); + // the agenda cultural API has returned the same event more than once before + + Self::dedupe_events(&mut parsed_response); + let events = Self::parse_events_by_date(parsed_response).await; Ok(events) } + fn dedupe_events(parsed_response: &mut Vec) { + let mut seen_links = HashSet::new(); + let fetched_count = parsed_response.len(); + + parsed_response.retain(|event| seen_links.insert(event.link.clone())); + + if parsed_response.len() != fetched_count { + warn!( + "Dropped {} duplicated events from the API response", + fetched_count - parsed_response.len() + ); + } + } + async fn parse_events_by_date(response: Vec) -> BTreeMap> { let mut events_by_date: BTreeMap> = BTreeMap::new(); @@ -134,21 +159,12 @@ impl AgendaCulturalAPI { } } + /// Full description is pulled only for the events that survive filtering + /// (see `get_full_description`). async fn convert_response_to_model(response: &EventResponse) -> Event { - let description = Self::get_full_description(&response.link) + response + .to_model(Self::clean_description(&response.description.concat())) .await - .unwrap_or_else(|| { - let preview_description = Self::clean_description(&response.description.concat()); - - warn!( - "Unable to get full description. Using only preview description ({})", - preview_description - ); - - preview_description - }); - - response.to_model(description).await } #[instrument] @@ -176,7 +192,7 @@ impl AgendaCulturalAPI { serde_json::from_str::>(&json_response).map_err(APIError::ParseError) } - async fn get_full_description(link: &str) -> Option { + pub async fn get_full_description(link: &str) -> Option { let full_page: Result = REST_CLIENT.get(link).send().await; match full_page { diff --git a/src/main.rs b/src/main.rs index 074496e..19a6f3b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -386,11 +386,21 @@ async fn send_new_events( .thread_id .name(&discord.client.http) .await - .unwrap_or_default() + .unwrap_or_else(|_| "no name".to_string()) ); - for event in events { + for mut event in events { async { + match AgendaCulturalAPI::get_full_description(&event.link).await { + Some(description) => event.details.description = description, + None => warn!( + event = event.link, + preview_description = event.details.description, + "Unable to get full description of '{}'. Using only preview description", + event.link + ), + } + let ticket_url = config.venue_ticket_shop_url.get(&event.venue).cloned(); let send_started_at = Instant::now(); let message = match discord