1
2
3 u"""
4 Модуль, который умеет получать данные по HTTP.
5
6 Функция L{loadUrlData} возвращает данные, расположенные по переданному ей URL.
7
8 Используемые модули:
9 - pycurl - easy_install pycurl
10 """
11
12 __author__ = "Zasimov Alexey"
13 __email__ = "zasimov-a@yandex-team.ru"
14
15
16 from StringIO import StringIO
17 import pycurl
18
19
20
21 -def getCurl(url, username=None, password=None):
22 curl = pycurl.Curl()
23
24 curl.setopt(pycurl.NOSIGNAL, 1)
25 curl.setopt(pycurl.HEADER, 0)
26 curl.setopt(pycurl.NOPROGRESS, 1)
27 curl.setopt(pycurl.FOLLOWLOCATION, 0)
28 curl.setopt(pycurl.DNS_CACHE_TIMEOUT, -1)
29 curl.setopt(pycurl.VERBOSE, 0)
30 curl.setopt(pycurl.CONNECTTIMEOUT, 30)
31 curl.setopt(pycurl.TIMEOUT, 300)
32 curl.setopt(pycurl.HTTP_VERSION, pycurl.CURL_HTTP_VERSION_1_1)
33
34 curl.resp_data = StringIO()
35
36 curl.setopt( pycurl.WRITEFUNCTION, curl.resp_data.write )
37 curl.setopt( pycurl.URL, url )
38
39 if username and password:
40 credentials = "%s:%s" % (username, password)
41
42 curl.setopt(pycurl.USERPWD, credentials)
43 curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_ANY)
44
45 return curl
46
48 curl = getCurl(url, username, password)
49
50 data = None
51 try:
52 curl.perform()
53
54 code = curl.getinfo( pycurl.HTTP_CODE )
55
56 if 200 <= code <= 399:
57 data = curl.resp_data.getvalue()
58 else:
59 err_msg = "incorrect http response code: url = <%s>, code = %d" % ( url, code )
60 raise RuntimeError( err_msg )
61
62 except pycurl.error, e:
63 curl.close()
64 err_msg = "exception while curl.perform: <%s>, url: <%s>" % ( str(e), url )
65 raise RuntimeError( err_msg )
66
67 curl.resp_data.close()
68 curl.close()
69
70 if not data:
71 err_msg = "no data from url(empty response): <%s>" % ( url )
72 raise RuntimeError( err_msg )
73
74 return data
75