Python中爬虫编程的常见问题及解决方案

import requests

headers = {
    'User-Agent': 'Mozilla/5.0 (windows NT 10.0; Win64; x64) AppleWEBKit/537.36 (Khtml, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
    'Referer': 'Http://www.example.com'
}

response = requests.get(url, headers=headers)

import requests

proxies = {
    'http': 'http://111.11.111.111:8080',
    'https': 'http://111.11.111.111:8080'
}

response = requests.get(url, proxies=proxies)

import requests

cookies = {
    'name': 'value'
}

response = requests.get(url, cookies=cookies)

from selenium import webdriver
from selenium.webdriver.common.by import By

driver = webdriver.Chrome()
driver.get(url)

# 使用WebDriverWait等待数据加载完毕
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

locator = (By.XPATH, '//div[@class="data"]')
data = WebDriverWait(driver, 10).until(EC.presence_of_element_located(locator)).text

import requests

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
    'Referer': 'http://www.example.com',
    'X-Requested-With': 'XMLHttpRequest'
}

response = requests.get(url, headers=headers)

from bs4 import BeautifulSoup

soup = BeautifulSoup(html, 'html.parser')

# 使用选择器提取数据
data = soup.select('.class')

import json

data = json.loads(response.text)

import xml.etree.ElementTree as ET

tree = ET.fromstring(xml)
root = tree.getroot()

# 提取数据
data = root.find('tag').text