From a7b8506c6ee0929e549d2185975dd67065005565 Mon Sep 17 00:00:00 2001 From: khaled-wsa <102720886+khaled-wsa@users.noreply.github.com> Date: Sat, 4 Feb 2023 01:55:06 +0300 Subject: [PATCH 1/2] Fixed UnicodeEncodeError when writing to file This pull request fixes the UnicodeEncodeError that was occurring when writing to a file. Updated the multiple open() calls to specify UTF-8 encoding. --- solutions/web_crawl_Q&A/web-qa.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/solutions/web_crawl_Q&A/web-qa.py b/solutions/web_crawl_Q&A/web-qa.py index 7fc7e723..45c823c1 100644 --- a/solutions/web_crawl_Q&A/web-qa.py +++ b/solutions/web_crawl_Q&A/web-qa.py @@ -135,7 +135,7 @@ def crawl(url): print(url) # for debugging and to see the progress # Save text from the url to a .txt file - with open('text/'+local_domain+'/'+url[8:].replace("/", "_") + ".txt", "w") as f: + with open('text/'+local_domain+'/'+url[8:].replace("/", "_") + ".txt", "w", encoding="utf-8") as f: # Get the text from the URL using BeautifulSoup soup = BeautifulSoup(requests.get(url).text, "html.parser") @@ -181,7 +181,7 @@ texts=[] for file in os.listdir("text/" + domain + "/"): # Open the file and read the text - with open("text/" + domain + "/" + file, "r") as f: + with open("text/" + domain + "/" + file, "r", encoding="utf-8") as f: text = f.read() # Omit the first 11 lines and the last 4 lines, then replace -, _, and #update with spaces. @@ -379,4 +379,4 @@ def answer_question( print(answer_question(df, question="What day is it?", debug=False)) -print(answer_question(df, question="What is our newest embeddings model?")) \ No newline at end of file +print(answer_question(df, question="What is our newest embeddings model?")) From a2837aa5cdefd56fccace2b5fcf6a93b207826d0 Mon Sep 17 00:00:00 2001 From: Logan Kilpatrick <23kilpatrick23@gmail.com> Date: Mon, 6 Feb 2023 11:28:31 -0600 Subject: [PATCH 2/2] Update web-qa.py --- solutions/web_crawl_Q&A/web-qa.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/solutions/web_crawl_Q&A/web-qa.py b/solutions/web_crawl_Q&A/web-qa.py index 45c823c1..b4c05c01 100644 --- a/solutions/web_crawl_Q&A/web-qa.py +++ b/solutions/web_crawl_Q&A/web-qa.py @@ -135,7 +135,7 @@ def crawl(url): print(url) # for debugging and to see the progress # Save text from the url to a .txt file - with open('text/'+local_domain+'/'+url[8:].replace("/", "_") + ".txt", "w", encoding="utf-8") as f: + with open('text/'+local_domain+'/'+url[8:].replace("/", "_") + ".txt", "w", encoding="UTF-8") as f: # Get the text from the URL using BeautifulSoup soup = BeautifulSoup(requests.get(url).text, "html.parser") @@ -181,7 +181,7 @@ texts=[] for file in os.listdir("text/" + domain + "/"): # Open the file and read the text - with open("text/" + domain + "/" + file, "r", encoding="utf-8") as f: + with open("text/" + domain + "/" + file, "r", encoding="UTF-8") as f: text = f.read() # Omit the first 11 lines and the last 4 lines, then replace -, _, and #update with spaces.