requests

发表于 | 分类于 ， |

默认请求头

1
2
3
4
5
6
GET / HTTP/1.1
Host: www.baidu.com
Connection: keep-alive
Accept-Encoding: gzip, deflate
Accept: */*
User-Agent: python-requests/2.10.0

发送请求

GET方法

1
2
import requests
r = requests.get(url)

POST方法

1
r = requests.post(url)

其它方法

1
2
3
4
r = requests.put(url)
r = requests.delete(url)
r = requests.head(url)
r = requests.options(url)

将请求方法放在参数中

1
2
import requests
req = requests.request('GET', url)

传递参数

GET传参

1
2
payload = {'key1': 'value1', 'key2': 'value2'}
r = requests.get(url, params=payload)

POST传参

1
2
payload = {'key1': 'value1', 'key2': 'value2'}
r = requests.post(url, data=payload)

JSON传参

1
2
3
import json
payload = {'some': 'data'}
r = requests.post(url, data=json.dumps(payload))

设置cookie

1
2
3
4
5
>>> url = 'http://httpbin.org/cookies'
>>> cookies = dict(cookies_are='working')
>>> r = requests.get(url, cookies=cookies)
>>> r.text
'{"cookies": {"cookies_are": "working"}}'

设置Headers

1
2
3
4
5
6
7
import json
payload = {'some': 'data'}
headers = {
    'content-type': 'application/json',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:44.0) Gecko/20100101 Firefox/44.0'
}
r = requests.post(url, data=json.dumps(payload), headers=headers)

上传

直接写入

1
2
f = {"file": ("upload.php","<?php phpinfo() ?>")}
r = requests.post(url, files=f)

上传文件

{‘name’: file-like-objects}
{‘name’: (‘filename’, fileobj)}

1
2
files = {'file': open('report.xls', 'rb')}
r = requests.post(url, files=files)

也可以明确设置filename, content_type and headers

1
2
3
files = {'file': ('report.xls', open('report.xls', 'rb'),
'application/vnd.ms-excel', {'Expires': '0'})}
r = requests.post(url, files=files)

上传多个文件

1
2
3
multiple_files = [('images', ('foo.png', open('foo.png', 'rb'), 'image/png')),
                      ('images', ('bar.png', open('bar.png', 'rb'), 'image/png'))]
r = requests.post(url, files=multiple_files)

流上传

1
2
with open('massive-body', 'rb') as f:
    requests.post(url, data=f)

Response

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
r.status_code
# 200
r.headers['content-type']
# 'application/json; charset=utf8'
r.encoding
# 'utf-8'
r.text
# u'{"type":"User"...'
r.json()
# {u'private_gists': 419, u'total_private_repos': 77, ...}
r.url
# http://httpbin.org/get?key2=value2&key1=value1
r.text
# u'[{"repository":{"open_issues":0,"url":"https://github.com/...
r.encoding
# 'utf-8'
r.encoding = 'ISO-8859-1'
r.status_code
# 200
r.headers['content-type']
# 'application/json; charset=utf8'
r.encoding
# 'utf-8'
r.text
# u'{"type":"User"...'
r.json()
# {u'private_gists': 419, u'total_private_repos': 77, ...}
r.contentr.content
# b'[{"repository":{"open_issues":0,"url":"https://github.com/...
r.cookies

返回体

获取unicode字符串，会自动根据响应头部的字符编码(r.encoding)进行解码，当然也可以自己设定r.encoding

1
2
3
>>> r = requests.get('https://github.com/timeline.json')
>>> r.text
u'{"message":"Hello there, wayfaring stranger...

获取bytes字符串，会自动解码gzip和deflate数据

1
2
>>> r.content
'{"message":"Hello there, wayfaring stranger. ..

要存储web图片，可以

1
2
3
>>> from PIL import Image
>>> from StringIO import StringIO
>>> i = Image.open(StringIO(r.content))

可以解码json对象

1
2
>>> r.json()
{u'documentation_url': u'https://developer...

返回raw response，需要在requests请求中将stream设为True

1
2
3
4
5
>>> r = requests.get('https://github.com/timeline.json', stream=True)
>>> r.raw
<requests.packages.urllib3.response.HTTPResponse object at 0x101194810>
>>> r.raw.read(10)
'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03'

如果不想一次性处理全部的数据，可以

1
2
3
4
5
tarball_url = 'https://github.com/kennethreitz/requests/tarball/master'
r = requests.get(tarball_url, stream=True)
if int(r.headers['content-length']) < TOO_LONG:
  content = r.content
  ...

也可以迭代的处理数据

1
2
3
with open(filename, 'wb') as fd:
    for chunk in r.iter_content(chunk_size):
        fd.write(chunk)

或者

1
2
3
4
5
6
7
import json
import requests
r = requests.get('http://httpbin.org/stream/20', stream=True)
for line in r.iter_lines():
    # filter out keep-alive new lines
    if line:
        print(json.loads(line))

获取响应代码

1
2
3
>>> r = requests.get('http://httpbin.org/get')
>>> r.status_code
200

获取响应headers

1
2
3
4
5
6
7
8
9
10
>>> r.headers
{
  'content-encoding': 'gzip',
  'transfer-encoding': 'chunked',
  'connection': 'close',
  'server': 'nginx/1.0.4',
  'x-runtime': '148ms',
  'etag': '"e1ca502697e5c9317743dc078f67693f"',
  'content-type': 'application/json'
}

获取发送的headers

1
2
3
>>> r.request.headers
{'Accept-Encoding': 'identity, deflate, compress, gzip',
'Accept': '*/*', 'User-Agent': 'python-requests/1.2.0'}

获取cookie，返回CookieJar对象

1
2
3
>>> url = 'http://www.baidu.com'
>>> r = requests.get(url)
>>> r.cookies

将CookieJar转为字典

1
2
>>> requests.utils.dict_from_cookiejar(r.cookies)
{'BAIDUID': '84722199DF8EDC372D549EC56CA1A0E2:FG=1', 'BD_HOME': '0', 'BDSVRTM': '0'}

将字典转为CookieJar

1
requests.utils.cookiejar_from_dict(cookie_dict, cookiejar=None, overwrite=True)

如果需要在会话中保留cookie，需要用到后面要说的Session。
Redirection and History

追踪重定向

1
2
3
4
5
6
7
>>> r = requests.get('http://github.com')
>>> r.url
'https://github.com/'
>>> r.status_code
200
>>> r.history
[<Response [301]>]

Session

要在会话中保留状态，可以使用request.Session()。
Session对象在请求时允许你保留一定的参数和自动设置cookie

使用get，post等方法

1
2
3
4
s = requests.Session()
s.get(url)   #cookie保留在s中
r = s.get(url) #再次访问时会保留cookie
r = s.post(url) #再次访问时会保留cookie

自己设置headers，cookies

1
2
3
4
s = requests.Session()
s.auth = ('user', 'pass')
s.headers.update({'x-test': 'true'})
s.get('http://httpbin.org/headers', headers={'x-test2': 'true'})    #  'x-test' and 'x-test2' 都会被发送

预设Request

可以在发送request前做些额外的设定

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
from requests import Request, Session

s = Session()
req = Request('GET', url,
  data=data,
  headers=header
)
prepped = req.prepare()

# do something with prepped.body
# do something with prepped.headers

resp = s.send(prepped,
  stream=stream,
  verify=verify,
  proxies=proxies,
  cert=cert,
  timeout=timeout
)

print(resp.status_code)

验证

Basic Authentication

1
2
3
>>> from requests.auth import HTTPBasicAuth
>>> requests.get('https://api.github.com/user', auth=HTTPBasicAuth('user', 'pass'))
<Response [200]>

因为HTTP Basic Auth很常用，所以也可以直接验证

1
2
>>> requests.get('https://api.github.com/user', auth=('user', 'pass'))
<Response [200]>

Digest Authentication

1
2
3
4
>>> from requests.auth import HTTPDigestAuth
>>> url = 'http://httpbin.org/digest-auth/auth/user/pass'
>>> requests.get(url, auth=HTTPDigestAuth('user', 'pass'))
<Response [200]>

OAuth 1 Authentication

1
2
3
4
5
6
7
>>> import requests
>>> from requests_oauthlib import OAuth1
>>> url = 'https://api.twitter.com/1.1/account/verify_credentials.json'
>>> auth = OAuth1('YOUR_APP_KEY', 'YOUR_APP_SECRET',
                  'USER_OAUTH_TOKEN', 'USER_OAUTH_TOKEN_SECRET')
>>> requests.get(url, auth=auth)
<Response [200]>

也可以使用自己写的验证类

比如某个web服务接受将X-Pizza报头设置成密码的验证，可以这样写验证类

1
2
3
4
5
6
7
8
9
10
from requests.auth import AuthBase
class PizzaAuth(AuthBase):
  """Attaches HTTP Pizza Authentication to the given Request object."""
  def __init__(self, username):
    # setup any auth-related data here
    self.username = username
  def __call__(self, r):
    # modify and return the request
    r.headers['X-Pizza'] = self.username
    return r

使用

1
2
>>> requests.get('http://pizzabin.org/admin', auth=PizzaAuth('kenneth'))
<Response [200]>

SSL证书验证

检查主机的ssl证书

1
2
3
>>> requests.get('https://kennethreitz.com', verify=True)
    raise ConnectionError(e)
ConnectionError: HTTPSConnectionPool(host='kennethreitz.com', port=443): Max retries exceeded with url: / (Caused by <class 'socket.error'>: [Errno 10061] )

github是有的

1
2
>>> requests.get('https://github.com', verify=True)
<Response [200]>

如果你设置验证设置为False，也可以忽略验证SSL证书。

可以读取验证文件

1
>>> requests.get('https://kennethreitz.com', cert=('/path/server.crt', '/path/key'))

代理

使用

1
2
3
4
5
6
import requests
proxies = {
  "http": "http://127.0.0.1:8080",
  "https": "socks5://127.0.0.1:1080"
}
requests.get("http://example.org", proxies=proxies)

socket出错

1
pip install pysocks

设置环境变量

1
2
3
4
5
$ export HTTP_PROXY="http://10.10.1.10:3128"
$ export HTTPS_PROXY="http://10.10.1.10:1080"
$ python
>>> import requests
>>> requests.get("http://example.org")

验证

1
2
3
proxies = {
    "http": "http://user:[email protected]:3128/",
}
打赏
0%