PyCharm实现唐诗三百首数据采集、特征向量化与统计分析

下面是一个使用PyCharm进行数据采集、文本特征向量化和数据统计分析的示例代码。请确保在运行代码之前已安装所需的库（如beautifulsoup4、jieba、wordcloud、pandas等）。 首先，我们需要使用爬虫技术从网站上获取唐诗三百首的内容，并保存到txt文件中。以下是实现此功能的代码： ```python import requests from bs4 import BeautifulSoup # 爬取唐诗三百首的内容 def scrape_data(): url = "https://so.gushiwen.cn/gushi/tangshi.aspx" headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36" } response = requests.get(url, headers=headers) if response.status_code == 200: soup = BeautifulSoup(response.text, "html.parser") content_divs = soup.find_all('div', class_='cont') poems = [] for div in content_divs: poem_type = div.p.get_text()[:-1] # 诗类型 poem_title = div.a.get_text() # 诗题目 poem_author = div.p.find_all('a')[1].get_text() # 诗作者 poem_content = div.find('div', class_='contson').get_text().replace('\n', '') # 诗内容 poem = f"{poem_type}\t{poem_title}\t{poem_content}\t{poem_author}" poems.append(poem) return poems else: print("请求失败") return None # 保存爬取的数据到txt文件 def save_to_txt(data, filename): with open(filename, 'w', encoding='utf-8') as f: for item in data: f.write(item + '\n') # 执行爬取数据并保存到txt文件 if __name__ == '__main__': poems = scrape_data() if poems: save_to_txt(poems, "poems.txt") print("数据采集完成并保存为txt文件！") ``` 接下来，我们需要对任务一中保存的txt文件进行文本特征向量化（使用tf-idf值）。以下是实现此功能的代码： ```python import pandas as pd from sklearn.feature_extraction.text import TfidfVectorizer # 读取txt文件中的数据 def read_data(filename): with open(filename, 'r', encoding='utf-8') as f: lines = f.readlines() return lines # 对文本进行特征向量化 def tfidf_vectorization(data): vectorizer = TfidfVectorizer() features = vectorizer.fit_transform(data) feature_names = vectorizer.get_feature_names() df = pd.DataFrame(features.toarray(), columns=feature_names) return df # 保存特征向量化后的数据到txt文件 def save_vectorization_to_txt(df, filename): with open(filename, 'w', encoding='utf-8') as f: for _, row in df.iterrows(): line = '\t'.join(map(str, row.values.tolist())) f.write(line + '\n') # 执行文本特征向量化并保存到txt文件 if __name__ == '__main__': data = read_data("poems.txt") df = tfidf_vectorization(data) save_vectorization_to_txt(df, "vectorization.txt") print("文本特征向量化完成并保存为txt文件！") ``` 我们需要对任务一中保存的txt文件进行数据统计分析。以下是实现此功能的代码： ```python import os import jieba import pandas as pd import matplotlib.pyplot as plt from collections import Counter from wordcloud import WordCloud # 统计每种诗类型的诗数量，并保存为excel文件 def count_poem_type(data): poem_types = [item.split('\t')[0] for item in data] counter = Counter(poem_types) df = pd.DataFrame(counter.most_common(), columns=['诗类型', '数量']) df.to_excel("poem_type_count.xlsx", index=False) # 统计每位作者的诗数量，并保存为excel文件 def count_poem_author(data): poem_authors = [item.split('\t')[-1].strip() for item in data] counter = Counter(poem_authors) df = pd.DataFrame(counter.most_common(), columns=['作者', '数量']) df.to_excel("poem_author_count.xlsx", index=False) # 对诗内容进行中文分词，并生成词云图 def create_wordcloud(data): poem_contents = [item.split('\t')[2] for item in data] text = '\n'.join(poem_contents) words = ' '.join(jieba.lcut(text)) wordcloud = WordCloud(width=800, height=400, background_color='white').generate(words) plt.imshow(wordcloud, interpolation='bilinear') plt.axis("off") plt.show() # 执行数据统计分析 if __name__ == '__main__': with open("poems.txt", 'r', encoding='utf-8') as f: data = f.readlines() count_poem_type(data) count_poem_author(data) create_wordcloud(data) print("数据统计分析完成！") ``` 上述代码中的爬虫部分仅针对指定的网站进行了简单处理，可能不适用于其他网站的爬取。在实际运行前，请先检查代码中的请求头部参数和页面解析方式，以确保代码正确运行。 ######[AI写代码神器 | 2961点数解答 | 2024-06-24 01:04:38]