Python BeautifulSoup 使用教程
·
BeautifulSoup 是 Python 中最流行的 HTML/XML 解析库，可以轻松地从网页中提取数据。
安装 BeautifulSoup
# 安装 BeautifulSoup 和解析器
pip install beautifulsoup4 lxml html5lib
# 如果需要请求网页，还需要安装 requests
pip install requests基本用法
1. 快速开始
from bs4 import BeautifulSoup
import requests
# 从字符串创建 BeautifulSoup 对象
html_doc = """
<html>
<head><title>The Dormouse's story</title></head>
<body>
<p class="title"><b>The Dormouse's story</b></p>
<p class="story">Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" id="link1">Elsie</a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>
<p class="story">...</p>
"""
soup = BeautifulSoup(html_doc, 'lxml')
print(soup.prettify()) # 格式化输出 HTML2. 对象种类
BeautifulSoup 将 HTML 文档转换为树形结构，包含四种主要对象：
from bs4 import BeautifulSoup
soup = BeautifulSoup('<b class="boldest">Extremely bold</b>', 'lxml')
# Tag 对象
tag = soup.b
print(type(tag)) # <class 'bs4.element.Tag'>
print(tag.name) # b
print(tag.attrs) # {'class': ['boldest']}
# NavigableString 对象
print(tag.string) # Extremely bold
print(type(tag.string)) # <class 'bs4.element.NavigableString'>
# BeautifulSoup 对象
print(type(soup)) # <class 'bs4.BeautifulSoup'>
# Comment 对象（注释）
comment_soup = BeautifulSoup("<p><!--This is a comment--></p>", 'lxml')
comment = comment_soup.p.string
print(type(comment)) # <class 'bs4.element.Comment'>文档导航
1. 使用标签名查找
from bs4 import BeautifulSoup
html_doc = """
<html>
<head><title>The Dormouse's story</title></head>
<body>
<p class="title"><b>The Dormouse's story</b></p>
<p class="story">Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" id="link1">Elsie</a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>
<p class="story">...</p>
"""
soup = BeautifulSoup(html_doc, 'lxml')
# 获取第一个 title 标签
print(soup.title) # <title>The Dormouse's story</title>
# 获取 title 标签的文本内容
print(soup.title.string) # The Dormouse's story
# 获取第一个 p 标签
print(soup.p) # <p class="title"><b>The Dormouse's story</b></p>
# 获取第一个 a 标签
print(soup.a) # <a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>2. 遍历文档树
# 子节点
print(soup.head.contents) # 返回列表 [<title>The Dormouse's story</title>]
print(soup.head.children) # 返回迭代器
for child in soup.head.children:
print(child)
# 所有子孙节点
for descendant in soup.head.descendants:
print(descendant)
# 父节点
print(soup.title.parent.name) # head
# 所有父节点
for parent in soup.a.parents:
print(parent.name)
# 兄弟节点
print(soup.a.next_sibling) # 下一个兄弟节点
print(soup.a.previous_sibling) # 上一个兄弟节点
# 所有兄弟节点
for sibling in soup.a.next_siblings:
print(sibling)搜索文档树
1. find() 和 find_all()
# find_all() 查找所有匹配的标签
all_links = soup.find_all('a')
print(all_links) # 返回所有 <a> 标签的列表
# find() 查找第一个匹配的标签
first_link = soup.find('a')
print(first_link)
# 按属性查找
links = soup.find_all('a', class_='sister')
links = soup.find_all('a', {'class': 'sister'})
links = soup.find_all('a', id='link1')
# 按文本内容查找
links = soup.find_all(string='Elsie')
links = soup.find_all(string=['Elsie', 'Lacie', 'Tillie'])
# 使用正则表达式
import re
links = soup.find_all(string=re.compile('Dormouse'))
links = soup.find_all('a', href=re.compile('example.com'))2. CSS 选择器
# 使用 select() 方法（CSS 选择器）
# 通过标签名
print(soup.select('title'))
# 通过类名
print(soup.select('.sister'))
# 通过 ID
print(soup.select('#link1'))
# 组合选择
print(soup.select('p .sister'))
print(soup.select('body a'))
# 属性选择器
print(soup.select('a[href="http://example.com/elsie"]'))
print(soup.select('a[href^="http://example.com/"]'))
print(soup.select('a[href$="tillie"]'))
# 获取属性值
for link in soup.select('a'):
print(link.get('href'))
print(link['href']) # 另一种方式修改文档
1. 修改标签和属性
# 修改标签名
tag = soup.b
tag.name = "blockquote"
print(tag) # <blockquote class="boldest">Extremely bold</blockquote>
# 修改属性
tag['class'] = 'newclass'
tag['id'] = 'newid'
print(tag)
# 删除属性
del tag['class']
print(tag)
# 添加新属性
tag['newattr'] = 'value'
print(tag)2. 修改字符串内容
# 修改字符串
tag.string = "New text content"
print(tag)
# 添加新标签
new_tag = soup.new_tag("p")
new_tag.string = "This is a new paragraph"
soup.body.append(new_tag)
# 插入标签
another_tag = soup.new_tag("div")
another_tag.string = "Inserted div"
soup.body.insert(0, another_tag)实际应用示例
1. 爬取网页数据
import requests
from bs4 import BeautifulSoup
import re
def scrape_quotes():
url = 'http://quotes.toscrape.com'
response = requests.get(url)
soup = BeautifulSoup(response.text, 'lxml')
quotes = []
# 提取所有引用
for quote_div in soup.select('.quote'):
text = quote_div.select_one('.text').get_text()
author = quote_div.select_one('.author').get_text()
tags = [tag.get_text() for tag in quote_div.select('.tag')]
quotes.append({
'text': text,
'author': author,
'tags': tags
})
return quotes
# 使用示例
quotes = scrape_quotes()
for quote in quotes[:3]:
print(f"作者: {quote['author']}")
print(f"引用: {quote['text']}")
print(f"标签: {', '.join(quote['tags'])}")
print('-' * 50)2. 提取新闻信息
import requests
from bs4 import BeautifulSoup
def scrape_news():
# 示例：模拟新闻网站结构
html_content = """
<div class="news-container">
<article class="news-item">
<h2><a href="/news/1">Python 3.11 发布，性能大幅提升</a></h2>
<div class="meta">
<span class="date">2023-10-01</span>
<span class="author">技术小编</span>
</div>
<div class="summary">Python 3.11 正式发布，相比 3.10 性能提升 25%</div>
</article>
<article class="news-item">
<h2><a href="/news/2">人工智能助力科学研究</a></h2>
<div class="meta">
<span class="date">2023-09-28</span>
<span class="author">AI观察员</span>
</div>
<div class="summary">AI技术在多个科学领域取得突破性进展</div>
</article>
</div>
"""
soup = BeautifulSoup(html_content, 'lxml')
news_items = []
for article in soup.select('.news-item'):
title_elem = article.select_one('h2 a')
title = title_elem.get_text(strip=True)
link = title_elem['href']
date = article.select_one('.date').get_text(strip=True)
author = article.select_one('.author').get_text(strip=True)
summary = article.select_one('.summary').get_text(strip=True)
news_items.append({
'title': title,
'link': link,
'date': date,
'author': author,
'summary': summary
})
return news_items
# 使用示例
news = scrape_news()
for item in news:
print(f"标题: {item['title']}")
print(f"日期: {item['date']}")
print(f"作者: {item['author']}")
print(f"摘要: {item['summary']}")
print('-' * 50)3. 处理表格数据
from bs4 import BeautifulSoup
def parse_table():
html_content = """
<table class="data-table">
<thead>
<tr>
<th>姓名</th>
<th>年龄</th>
<th>城市</th>
<th>职业</th>
</tr>
</thead>
<tbody>
<tr>
<td>张三</td>
<td>25</td>
<td>北京</td>
<td>工程师</td>
</tr>
<tr>
<td>李四</td>
<td>30</td>
<td>上海</td>
<td>设计师</td>
</tr>
<tr>
<td>王五</td>
<td>28</td>
<td>广州</td>
<td>产品经理</td>
</tr>
</tbody>
</table>
"""
soup = BeautifulSoup(html_content, 'lxml')
table_data = []
# 获取表头
headers = [th.get_text(strip=True) for th in soup.select('thead th')]
# 获取表格数据
for row in soup.select('tbody tr'):
cells = [td.get_text(strip=True) for td in row.select('td')]
table_data.append(dict(zip(headers, cells)))
return table_data
# 使用示例
data = parse_table()
for row in data:
print(row)高级技巧
1. 处理编码问题
# 指定编码
html_content = "<p>中文内容</p>".encode('gbk')
soup = BeautifulSoup(html_content, 'lxml', from_encoding='gbk')
print(soup.p.get_text())2. 使用不同的解析器
# lxml (推荐，速度快)
soup1 = BeautifulSoup(html_doc, 'lxml')
# html.parser (Python 内置)
soup2 = BeautifulSoup(html_doc, 'html.parser')
# html5lib (最兼容)
soup3 = BeautifulSoup(html_doc, 'html5lib')3. 提取特定属性的元素
# 提取所有链接
links = soup.find_all('a')
for link in links:
print(link.get('href'))
# 提取所有图片
images = soup.find_all('img')
for img in images:
print(img.get('src'))
print(img.get('alt', 'No alt text'))注意事项
-
尊重网站规则：检查 robots.txt，不要过度请求
-
设置请求头：模拟真实浏览器行为
-
处理异常：网络请求可能失败
-
使用延时：避免对服务器造成压力
-
遵守法律法规：只爬取允许公开访问的数据
总结
BeautifulSoup 提供了强大而灵活的方式来解析和提取 HTML/XML 数据。关键点包括：
-
使用
find_all()和find()搜索元素 -
使用 CSS 选择器
select()精确查找 -
熟练使用标签导航方法
-
掌握字符串和属性的提取技巧
-
结合实际项目练习提高技能
通过本教程，你应该能够使用 BeautifulSoup 处理大多数网页解析任务。
Agent 垂直技术社区，欢迎活跃、内容共建。
更多推荐
5
0- 0
已为社区贡献24条内容
所有评论(0)