Python 官方文档:入门教程 => 点击学习
通过抓取西刺网免费代理ip实现代理爬虫: from bs4 import BeautifulSoup import requests import random import telnetlib requests = requests.se
通过抓取西刺网免费代理ip实现代理爬虫:
from bs4 import BeautifulSoup
import requests
import random
import telnetlib
requests = requests.session()
ip_list = []
proxy_list = []
headers = [
'Mozilla/5.0 (windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0',
'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWEBKit/537.36 (Khtml, like Gecko) Chrome/53.0.2785.143 Safari/537.36'
]
headers = random.choice(headers)
headers = {'User-Agent': headers}
def get_proxy():
url = 'Http://www.xicidaili.com/nn/'
web_data = requests.get(url, headers=headers)
soup = BeautifulSoup(web_data.text, 'lxml')
ips = soup.find_all('tr')
for i in range(1, len(ips)):
ip_info = ips[i]
tds = ip_info.find_all('td')
ip = tds[1].text
port = tds[2].text
try:
telnetlib.Telnet(ip, port, timeout=2)
ip_list.append(ip+":"+port)
except:
pass
#print(ip_list)
for ip in ip_list:
proxy_list.append('http://' + ip)
proxies = get_proxy()
proxy_ip = random.choice(proxy_list)
proxies = {'http': proxy_ip}
print(proxies)
代理IP的使用,把他传到get请求里面就可以了
requests.get(url, headers=headers, proxies=proxies)
--结束END--
本文标题: python: 抓取免费代理ip
本文链接: https://www.lsjlt.com/news/190384.html(转载时请注明来源链接)
有问题或投稿请发送至: 邮箱/279061341@qq.com QQ/279061341
下载Word文档到电脑,方便收藏和打印~
2024-03-01
2024-03-01
2024-03-01
2024-02-29
2024-02-29
2024-02-29
2024-02-29
2024-02-29
2024-02-29
2024-02-29
回答
回答
回答
回答
回答
回答
回答
回答
回答
回答
0