From 44d7d84fb30420336b91557b98b501d6886241e3 Mon Sep 17 00:00:00 2001 From: OwenOrcan Date: Sat, 2 Mar 2024 20:19:35 -0500 Subject: [PATCH] added timeout parameters to every request to ensure stability --- yirabot/data_extraction_functions.py | 2 +- yirabot/seo_functions.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/yirabot/data_extraction_functions.py b/yirabot/data_extraction_functions.py index 826521b..70ef9dc 100644 --- a/yirabot/data_extraction_functions.py +++ b/yirabot/data_extraction_functions.py @@ -140,7 +140,7 @@ def parse_sitemap(url, script=False): # Attempt to parse standard sitemaps for sitemap_url in sitemap_urls: try: - response = requests.get(sitemap_url) + response = requests.get(sitemap_url, timeout=10) if response.status_code == 200: soup = BeautifulSoup(response.content, 'xml') return [element.text for element in soup.find_all("loc")] diff --git a/yirabot/seo_functions.py b/yirabot/seo_functions.py index 05bd3c6..9375d7c 100644 --- a/yirabot/seo_functions.py +++ b/yirabot/seo_functions.py @@ -19,7 +19,7 @@ def check_website_language(url): try: - response = requests.get(url) + response = requests.get(url, timeout=10) soup = BeautifulSoup(response.content, 'html.parser') html_tag = soup.find('html') @@ -34,7 +34,7 @@ def check_website_language(url): def check_social_media_integration(url): try: - response = requests.get(url) + response = requests.get(url, timeout=10) soup = BeautifulSoup(response.content, 'html.parser') social_media = { @@ -83,7 +83,7 @@ def check_link_status(url, session=None): Returns a tuple of (is_broken, status_code, reason). """ try: - response = session.head(url, allow_redirects=True) if session else requests.head(url, allow_redirects=True) + response = session.head(url, allow_redirects=True, timeout=10) if session else requests.head(url, allow_redirects=True, timeout=10) if response.status_code == 404: return True, 404, "Not Found" elif 300 <= response.status_code < 400: @@ -172,7 +172,7 @@ def analyze_images_for_alt_text(soup): def seo_error_analysis(url, session=None): try: print("YiraBot: Starting SEO Analysis") - response = session.get(url) if session else requests.get(url) + response = session.get(url, timeout=10) if session else requests.get(url, timeout=10) soup = BeautifulSoup(response.content, 'html.parser') title_length, title_status = analyze_title(soup)