diff --git a/pachong.py b/pachong.py index 4cc365b..dc1c6f8 100644 --- a/pachong.py +++ b/pachong.py @@ -2,8 +2,25 @@ import requests from bs4 import BeautifulSoup import time import random +import re +import logging +from urllib.parse import urlparse +import os # 导入 os 模块用于文件操作 + +# 配置日志 +logging.basicConfig(filename="Records.log", level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") + +def is_valid_url(url): + #验证 URL 是否有效 + parsed = urlparse(url) + return bool(parsed.scheme in ["http", "https"] and parsed.netloc) + +def sanitize_filename(filename): + #清理文件名中的非法字符 + return re.sub(r"[^\w\s]", "_", filename) def pachong(url): + #爬取网页中的所有链接 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36" } @@ -14,38 +31,56 @@ def pachong(url): soup = BeautifulSoup(response.text, "html.parser") # 提取所有链接 - links = [] + links = set() # 使用集合去重 for a_tag in soup.find_all("a", href=True): link = a_tag["href"] - links.append(link) + links.add(link) # 添加到集合中 - print(f"从 {url} 爬取到 {len(links)} 个链接。") + logging.info(f"从 {url} 爬取到 {len(links)} 个链接") + print(f"从 {url} 爬取到 {len(links)} 个链接") return links except requests.RequestException as e: + logging.error(f"无法访问 {url},错误:{e}") print(f"无法访问 {url},错误:{e}") return [] def main(): - # 用户输入多个网站的 URL - urls = input("请输入多个网站的 URL,用逗号分隔:").strip().split(",") + #主函数 + urls = input("请输入网站的 URL,超过两个用逗号分隔:").strip().split(",") for url in urls: url = url.strip() # 去除多余的空格 - if url: # 确保 URL 不为空 - print(f"正在爬取 {url}...") - links = pachong(url) + if not is_valid_url(url): + logging.warning(f"无效的 URL:{url},已跳过") + print(f"无效的 URL:{url},已跳过") + continue + print(f"正在爬取 {url}...") + logging.info(f"正在爬取 {url}...") + links = pachong(url) + + if links: # 为每个网站创建一个独立的文件 - filename = f"{url.replace('https://', '').replace('www.', '').replace('/', '_')}.txt" + filename = sanitize_filename(f"{url.replace('https://', '').replace('www.', '').replace('/', '_')}.txt") + + # 如果文件已存在,删除旧文件 + if os.path.exists(filename): + logging.info(f"检测到旧文件 {filename},将用新内容更新。") + print(f"检测到旧文件 {filename},将用新内容更新") + os.remove(filename) + + # 写入新的爬取内容 with open(filename, "w", encoding="utf-8") as file: for link in links: file.write(link + "\n") + logging.info(f"链接已保存到 {filename}") print(f"链接已保存到 {filename}") time.sleep(random.uniform(1, 3)) # 随机延迟,避免对服务器造成过大压力 - print("所有网站的链接已分别保存到对应的文件中。") + print("所有网站的链接已分别保存到对应的文件中") + logging.info("所有网站的链接已分别保存到对应的文件中") if __name__ == "__main__": - main() + main() \ No newline at end of file