1.table 是表格，tr是行，td是列 ul li是无序列标签用的较多，ol li是有序列标签
最基本的结构
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title> Title </title>
</head>
<body>
</body>
</html><!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Title</title>
</head>
<body>
<!-- table 表格
tr 行
td 列
-->
<table width="200px" height="200px" border="1px">
<tr>
<td>
姓名
</td>
<td>
年龄
</td>
<td>
性别
</td>
</tr>
<tr>
<td>
张三
</td>
<td>
18
</td>
<td>
男
</td>
</tr>
</table>
<!-- ul li 无序列表 爬虫的使用场景非常之多-->
<ul>
<li>铁锅炖大鹅</li>
<li>小鸡炖蘑菇</li>
<li>锅包肉</li>
</ul>
<!-- ol li-->
<ol>
<li>穿上衣服</li>
<li>下床</li>
<li>洗漱</li>
</ol>
<a href="http://www.atguigu.com/">尚硅谷</a>
</body>
</html>2.urllib库的基本使用
请求对象定制，现在国际统一编码-unicode
下面是代码的练习
#使用urllib来获取百度首页的源码
#1.定义链接，要访问的网址
import urllib.request
url='http://www.baidu.com'
#2.模拟浏览器发送请求
respone=urllib.request.urlopen(url)
#3.content是内容的意思,read方法，返回的是字节形式的二进制数据，需要将二进制格式的数据转化为字符串
content=respone.read().decode('utf-8')
#打印字符串
print(content)
#1.练习
#respone是Httprespone的类型，print(type(respone))
#content=respone.read() 一个一个字节的去读，括号里面加一个参数是返回多少字节
#content=respone.readline() 是读取一行
#print(response.geturl()) 返回状态码，如果状态码是200就证明逻辑没有错
#print(response.geturl()) 返回的是url地址
#print(response.getheaders()) 获取的是一个状态信息
import urllib.request
url='http://www.baidu.com'
response=urllib.request.urlopen(url)
print(response.getheaders())
#2.下载网页，下载图片，下载视频
import urllib.request
#下载网页
#url_page='http://www.baidu.com'
#urllib.request.urlretrieve(url_page,'baidu.html')
#下载图片
#url_img='https://img1.baidu.com/it/u=3004965690,4089234593&fm=26&fmt=auto&gp=0.jpg'
url_video='https://vd3.bdstatic.com/mda-mhkku4ndaka5etk3/1080p/cae_h264/1629557146541497769/mda-mhkku4ndaka5etk3.mp4?v_from_s=hkapp-haokan-tucheng&auth_key=1629687514-0-0-7ed57ed7d1168bb1f06d18a4ea214300&bcevod_channel=searchbox_feed&pd=1&pt=3&abtest='
urllib.request.urlretrieve(url_video,'1.mp4')
#3.请求对象的定制
import urllib.request
url='https://www.baidu.com'
# url的组成
# https://www.baidu.com/s?wd=周杰伦
#http/https www.baidu.com 80/443 s wd=周杰伦 #
# 协议 主机 端口号 路径 参数 锚点
#http 80 https 443 mysql 3306
headers={
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36'
}
request=urllib.request.Request(url=url,headers=headers)
response =urllib.request.urlopen(request)
content=response.read().decode('utf-8')
print(content)
#4.解决编码问题
# 需求 获取 https://www.baidu.com/s?wd=周杰伦
import urllib.request
import urllib.parse
url='https://www.baidu.com/s?wd='
#请求对象定制为了解决反爬的第一种手段
headers={
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36'
}
#需要将周杰伦变成unicode编码格式
name=urllib.parse.quote('周杰伦')
url=url+name
#请求对象的定制
request=urllib.request.Request(url=url,headers=headers)
#模拟浏览器向服务器发送请求
response=urllib.request.urlopen(request)
#获取相应内容
content=response.read().decode('utf-8')
print(content)
#5.需要多个汉字转编码时用urlencode
#eg:# https://www.baidu.com/s?wd=周杰伦&sex=男
import urllib.parse
data={
'wd':'周杰伦',
'sex':'男'
}
a=urllib.parse.urlencode(data)
print(a)
#6.post请求
# post请求方式的参数 必须编码 data = urllib.parse.urlencode(data)
# 编码之后 必须调用encode方法 data = urllib.parse.urlencode(data).encode('utf-8')
# 参数是放在请求对象定制的方法中
#request =urllib.request.Request(url=url,data=data,headers=headers)
import urllib.request
import urllib.parse
import json
url = 'https://fanyi.baidu.com/sug'
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36'
}
data={
'kw':'spider'
}
data=urllib.parse.urlencode(data).encode('utf-8')
request=urllib.request.Request(url=url,data=data,headers=headers)
response=urllib.request.urlopen(request)
content=response.read().decode('utf-8')
obj=json.loads(content)
print(obj)
#注意有时候请求头需要全部加上，或许只有某一个才是需要的，要具体问题具体分析
'''headers = {
# 'Accept': '*/*',
# 'Accept-Encoding': 'gzip, deflate, br',
# 'Accept-Language': 'zh-CN,zh;q=0.9',
# 'Connection': 'keep-alive',
# 'Content-Length': '135',
# 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'Cookie': 'BIDUPSID=DAA8F9F0BD801A2929D96D69CF7EBF50; PSTM=1597202227; BAIDUID=DAA8F9F0BD801A29B2813502000BF8E9:SL=0:NR=10:FG=1; __yjs_duid=1_c19765bd685fa6fa12c2853fc392f8db1618999058029; REALTIME_TRANS_SWITCH=1; FANYI_WORD_SWITCH=1; HISTORY_SWITCH=1; SOUND_SPD_SWITCH=1; SOUND_PREFER_SWITCH=1; BDUSS=R2bEZvTjFCNHQxdUV-cTZ-MzZrSGxhbUYwSkRkUWk2SkxxS3E2M2lqaFRLUlJoRVFBQUFBJCQAAAAAAAAAAAEAAAA3e~BTveK-9sHLZGF5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFOc7GBTnOxgaW; BDUSS_BFESS=R2bEZvTjFCNHQxdUV-cTZ-MzZrSGxhbUYwSkRkUWk2SkxxS3E2M2lqaFRLUlJoRVFBQUFBJCQAAAAAAAAAAAEAAAA3e~BTveK-9sHLZGF5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFOc7GBTnOxgaW; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; BAIDUID_BFESS=DAA8F9F0BD801A29B2813502000BF8E9:SL=0:NR=10:FG=1; BDRCVFR[feWj1Vr5u3D]=I67x6TjHwwYf0; PSINO=2; H_PS_PSSID=34435_31660_34405_34004_34073_34092_26350_34426_34323_22158_34390; delPer=1; BA_HECTOR=8185a12020018421b61gi6ka20q; BCLID=10943521300863382545; BDSFRCVID=boDOJexroG0YyvRHKn7hh7zlD_weG7bTDYLEOwXPsp3LGJLVJeC6EG0Pts1-dEu-EHtdogKK0mOTHv8F_2uxOjjg8UtVJeC6EG0Ptf8g0M5; H_BDCLCKID_SF=tR3aQ5rtKRTffjrnhPF3-44vXP6-hnjy3bRkX4Q4Wpv_Mnndjn6SQh4Wbttf5q3RymJ42-39LPO2hpRjyxv4y4Ldj4oxJpOJ-bCL0p5aHl51fbbvbURvD-ug3-7qqU5dtjTO2bc_5KnlfMQ_bf--QfbQ0hOhqP-jBRIE3-oJqC8hMIt43f; BCLID_BFESS=10943521300863382545; BDSFRCVID_BFESS=boDOJexroG0YyvRHKn7hh7zlD_weG7bTDYLEOwXPsp3LGJLVJeC6EG0Pts1-dEu-EHtdogKK0mOTHv8F_2uxOjjg8UtVJeC6EG0Ptf8g0M5; H_BDCLCKID_SF_BFESS=tR3aQ5rtKRTffjrnhPF3-44vXP6-hnjy3bRkX4Q4Wpv_Mnndjn6SQh4Wbttf5q3RymJ42-39LPO2hpRjyxv4y4Ldj4oxJpOJ-bCL0p5aHl51fbbvbURvD-ug3-7qqU5dtjTO2bc_5KnlfMQ_bf--QfbQ0hOhqP-jBRIE3-oJqC8hMIt43f; Hm_lvt_64ecd82404c51e03dc91cb9e8c025574=1629701482,1629702031,1629702343,1629704515; Hm_lpvt_64ecd82404c51e03dc91cb9e8c025574=1629704515; __yjs_st=2_MDBkZDdkNzg4YzYyZGU2NTM5NzBjZmQ0OTZiMWRmZGUxM2QwYzkwZTc2NTZmMmIxNDJkYzk4NzU1ZDUzN2U3Yjc4ZTJmYjE1YTUzMTljYWFkMWUwYmVmZGEzNmZjN2FlY2M3NDAzOThhZTY5NzI0MjVkMmQ0NWU3MWE1YTJmNGE5NDBhYjVlOWY3MTFiMWNjYTVhYWI0YThlMDVjODBkNWU2NjMwMzY2MjFhZDNkMzVhNGMzMGZkMWY2NjU5YzkxMDk3NTEzODJiZWUyMjEyYTk5YzY4ODUyYzNjZTJjMGM5MzhhMWE5YjU3NTM3NWZiOWQxNmU3MDVkODExYzFjN183XzliY2RhYjgz; ab_sr=1.0.1_ZTc2ZDFkMTU5ZTM0ZTM4MWVlNDU2MGEzYTM4MzZiY2I2MDIxNzY1Nzc1OWZjZGNiZWRhYjU5ZjYwZmNjMTE2ZjIzNmQxMTdiMzIzYTgzZjVjMTY0ZjM1YjMwZTdjMjhiNDRmN2QzMjMwNWRhZmUxYTJjZjZhNTViMGM2ODFlYjE5YTlmMWRjZDAwZGFmMDY4ZTFlNGJiZjU5YzE1MGIxN2FiYTU3NDgzZmI4MDdhMDM5NTQ0MjQxNDBiNzdhMDdl',
# 'Host': 'fanyi.baidu.com',
# 'Origin': 'https://fanyi.baidu.com',
# 'Referer': 'https://fanyi.baidu.com/?aldtype=16047',
# 'sec-ch-ua': '"Chromium";v="92", " Not A;Brand";v="99", "Google Chrome";v="92"',
# 'sec-ch-ua-mobile': '?0',
# 'Sec-Fetch-Dest': 'empty',
# 'Sec-Fetch-Mode': 'cors',
# 'Sec-Fetch-Site': 'same-origin',
# 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36',
# 'X-Requested-With': 'XMLHttpRequest',
}'''
#7.下载视频
import urllib.request
url='https://movie.douban.com/j/chart/top_list?type=5&interval_id=100%3A90&action=&start=0&limit=20'
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36'
}
request=urllib.request.Request(url=url,headers=headers) #其实有三个参数，写两个必须前面也要写
#三个就可以不写了，为了防止对应不上
response=urllib.request.urlopen(request)
content=response.read().decode('utf-8')
#数据下载到本地，open方法默认是gbk编码 如果想要保存汉字那么需要在open方法中指定编码格式utf-8
with open('douban.json','w',encoding='utf-8') as fp:
fp.write(content)
