自己封装的http.client get请求

由于requests的get请求有各种各样的缺陷，所以我自己用http.client封装了一个 client_get 支持设置代理、 ssl验证、是否重定向

import urllib.parse
import http.client
def client_get(url,headers=None,proxies=None,verify=True,allow_redirect=True):

  url = urllib.parse.urlparse(url)
  hostname = url.hostname
  port = 80
  if proxies!={} or None:
    hostname,port = proxies["http"].split(":")
  conn = http.client.HTTPSConnection(hostname, port)
  if verify == False:
    conn = http.client.HTTPSConnection(hostname,port,context=ssl._create_unverified_context())
  if proxies != {} or None:
    conn.set_tunnel(url.hostname)
  if url.query == "":
    urlPa = url.path
  else:
    urlPa = "%s?%s" % (url.path,url.query)

  if headers is None:
    headers = {}
  conn.request("GET",urlPa,headers=headers)
  response = conn.getresponse()
  if response.code in [301,302] and allow_redirect is True:
    res_headers = response.headers._headers
    location = ""
    for item in res_headers:
      name,value = item
      if name == "location":
        location = value
    return client_get(location,headers=headers,proxies=proxies,verify=verify)
  return response

使用例子

url = "https://pornhub.com/view_video.php?viewkey=ph635d67e1ecd60"

headers = {
  'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.74 Safari/537.36',
  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'en-us,en;q=0.5',
  'Sec-Fetch-Mode': 'navigate', 'Connection': 'close'}
proxies = {
  "http": "http://127.0.0.1:8888", "https": "http://127.0.0.1:8888",
}
res = client_get(url,headers=headers,proxies=proxies,verify=False,allow_redirect=True)
print(res.read().decode('utf-8'))

原文地址：http://www.cnblogs.com/czlnb/p/16857005.html