python網(wǎng)頁請求urllib2模塊簡單封裝代碼
對python網(wǎng)頁請求模塊urllib2進(jìn)行簡單的封裝。
例子:
#!/usr/bin/python
#coding: utf-8
import base64
import urllib
import urllib2
import time
class SendRequest:
'''
This class use to set and request the http, and get the info of response.
e.g. set Authorization Type, request tyep..
e.g. get html content, state code, cookie..
SendRequest('http://10.75.0.103:8850/2/photos/square/type.json',
data='source=216274069', type='POST', auth='base',
user='zl2010', password='111111')
'''
def __init__(self, url, data=None, type='GET', auth=None, user=None, password=None, cookie = None, **header):
'''
url:request, raise error if none
date: data for post or get, must be dict type
type: GET, POST
auth: option, if has the value must be 'base' or 'cookie'
user: user for auth
password: password for auth
cookie: if request with cookie
other header info:
e.g. referer='www.sina.com.cn'
'''
self.url = url
self.data = data
self.type = type
self.auth = auth
self.user = user
self.password = password
self.cookie = cookie
if 'referer' in header:
self.referer = header[referer]
else:
self.referer = None
if 'user-agent' in header:
self.user_agent = header[user-agent]
else:
self.user_agent = None
self.setup_request()
self.send_request()
def setup_request(self):
'''
setup a request
'''
if self.url == None or self.url == '':
raise 'The url should not empty!'
# set request type
#print self.url
#print self.type
#print self.data
#print self.auth
#print self.user
#print self.password
if self.type == 'POST':
self.Req = urllib2.Request(self.url, self.data)
elif self.type == 'GET':
if self.data == None:
self.Req = urllib2.Request(self.url)
else:
self.Req = urllib2.Request(self.url + '?' + self.data)
else:
print 'The http request type NOT support now!'
##set auth type
if self.auth == 'base':
if self.user == None or self.password == None:
raise 'The user or password was not given!'
else:
auth_info = base64.encodestring(self.user + ':' + self.password).replace('\n','')
auth_info = 'Basic ' + auth_info
#print auth_info
self.Req.add_header("Authorization", auth_info)
elif self.auth == 'cookie':
if self.cookie == None:
raise 'The cookie was not given!'
else:
self.Req.add_header("Cookie", self.cookie)
else:
pass ##add other auth type here
##set other header info
if self.referer:
self.Req.add_header('referer', self.referer)
if self.user_agent:
self.Req.add_header('user-agent', self.user_agent)
def send_request(self):
'''
send a request
'''
# get a response object
try:
self.Res = urllib2.urlopen(self.Req)
self.source = self.Res.read()
self.goal_url = self.Res.geturl()
self.code = self.Res.getcode()
self.head_dict = self.Res.info().dict
self.Res.close()
except urllib2.HTTPError, e:
self.code = e.code
print e
def get_code(self):
return self.code
def get_url(self):
return self.goal_url
def get_source(self):
return self.source
def get_header_info(self):
return self.head_dict
def get_cookie(self):
if 'set-cookie' in self.head_dict:
return self.head_dict['set-cookie']
else:
return None
def get_content_type(self):
if 'content-type' in self.head_dict:
return self.head_dict['content-type']
else:
return None
def get_expires_time(self):
if 'expires' in self.head_dict:
return self.head_dict['expires']
else:
return None
def get_server_name(self):
if 'server' in self.head_dict:
return self.head_dict['server']
else:
return None
def __del__(self):
pass
__all__ = [SendRequest,]
if __name__ == '__main__':
'''
The example for using the SendRequest class
'''
value = {'source':'216274069'}
data = urllib.urlencode(value)
url = 'http://10.75.0.103:8850/2/photos/square/type.json'
user = 'wz_0001'
password = '111111'
auth = 'base'
type = 'POST'
t2 = time.time()
rs = SendRequest('http://www.google.com')
#rs = SendRequest(url, data=data, type=type, auth=auth, user=user, password=password)
print 't2: ' + str(time.time() - t2)
print '---------------get_code()---------------'
print rs.get_code()
print '---------------get_url()---------------'
print rs.get_url()
print '---------------get_source()---------------'
print rs.get_source()
print '---------------get_cookie()---------------'
print rs.get_cookie()
rs = None
- 零基礎(chǔ)寫python爬蟲之使用urllib2組件抓取網(wǎng)頁內(nèi)容
- 【Python】Python的urllib模塊、urllib2模塊批量進(jìn)行網(wǎng)頁下載文件
- Python urllib、urllib2、httplib抓取網(wǎng)頁代碼實(shí)例
- 在Python中使用cookielib和urllib2配合PyQuery抓取網(wǎng)頁信息
- Python3爬蟲之urllib攜帶cookie爬取網(wǎng)頁的方法
- python 解析html之BeautifulSoup
- Python使用BeautifulSoup庫解析HTML基本使用教程
- python爬蟲入門教程--HTML文本的解析庫BeautifulSoup(四)
- python網(wǎng)絡(luò)編程學(xué)習(xí)筆記(七):HTML和XHTML解析(HTMLParser、BeautifulSoup)
- Python利用BeautifulSoup解析Html的方法示例
- Python HTML解析器BeautifulSoup用法實(shí)例詳解【爬蟲解析器】
- Python獲取基金網(wǎng)站網(wǎng)頁內(nèi)容、使用BeautifulSoup庫分析html操作示例
相關(guān)文章
Python Pygame實(shí)現(xiàn)俄羅斯方塊
這篇文章主要為大家詳細(xì)介紹了Python Pygame實(shí)現(xiàn)俄羅斯方塊,文中示例代碼介紹的非常詳細(xì),具有一定的參考價(jià)值,感興趣的小伙伴們可以參考一下2021-02-02python tkinter實(shí)現(xiàn)定時(shí)關(guān)機(jī)
這篇文章主要為大家詳細(xì)介紹了python tkinter實(shí)現(xiàn)定時(shí)關(guān)機(jī),文中示例代碼介紹的非常詳細(xì),具有一定的參考價(jià)值,感興趣的小伙伴們可以參考一下2021-04-04通過python實(shí)現(xiàn)隨機(jī)交換禮物程序詳解
這篇文章主要介紹了通過python實(shí)現(xiàn)隨機(jī)交換禮物程序詳解的,文中通過示例代碼介紹的非常詳細(xì),對大家的學(xué)習(xí)或者工作具有一定的參考學(xué)習(xí)價(jià)值,需要的朋友可以參考下2019-07-07python實(shí)現(xiàn)列表中由數(shù)值查到索引的方法
今天小編就為大家分享一篇python實(shí)現(xiàn)列表中由數(shù)值查到索引的方法,具有很好的參考價(jià)值,希望對大家有所幫助。一起跟隨小編過來看看吧2018-06-06基于Python實(shí)現(xiàn)對比Exce的工具
這篇文章主要介紹了基于Python實(shí)現(xiàn)對比Excel的小工具,通過循環(huán)對比組合列(主鍵+對比列)結(jié)合示例代碼給大家介紹的非常詳細(xì),感興趣的朋友跟隨小編一起看看吧2022-04-04Matplotlib 生成不同大小的subplots實(shí)例
今天小編就為大家分享一篇Matplotlib 生成不同大小的subplots實(shí)例,具有很好的參考價(jià)值,希望對大家有所幫助。一起跟隨小編過來看看吧2018-05-05