Package stocks3 :: Package share :: Module curl
[hide private]
[frames] | no frames]

Source Code for Module stocks3.share.curl

 1  # -*- coding: utf-8 -*- 
 2   
 3  u""" 
 4  Модуль, который умеет получать данные по HTTP. 
 5   
 6  Функция L{loadUrlData} возвращает данные, расположенные по переданному ей URL. 
 7   
 8  Используемые модули: 
 9      - pycurl - easy_install pycurl 
10  """ 
11   
12  __author__ = "Zasimov Alexey" 
13  __email__ = "zasimov-a@yandex-team.ru" 
14   
15   
16  from StringIO import StringIO 
17  import pycurl 
18   
19   
20  # make curl object 
21 -def getCurl(url, username=None, password=None):
22 curl = pycurl.Curl() 23 24 curl.setopt(pycurl.NOSIGNAL, 1) # Signals are not thread safe 25 curl.setopt(pycurl.HEADER, 0) # Don't write HTTP headers to buffer 26 curl.setopt(pycurl.NOPROGRESS, 1) # No need to trace download progress 27 curl.setopt(pycurl.FOLLOWLOCATION, 0) # Do not use redirects 28 curl.setopt(pycurl.DNS_CACHE_TIMEOUT, -1) # in seconds 29 curl.setopt(pycurl.VERBOSE, 0) 30 curl.setopt(pycurl.CONNECTTIMEOUT, 30) 31 curl.setopt(pycurl.TIMEOUT, 300) 32 curl.setopt(pycurl.HTTP_VERSION, pycurl.CURL_HTTP_VERSION_1_1) 33 34 curl.resp_data = StringIO() 35 36 curl.setopt( pycurl.WRITEFUNCTION, curl.resp_data.write ) 37 curl.setopt( pycurl.URL, url ) 38 39 if username and password: 40 credentials = "%s:%s" % (username, password) 41 42 curl.setopt(pycurl.USERPWD, credentials) 43 curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_ANY) 44 45 return curl
46
47 -def loadUrlData(url, username=None, password=None):
48 curl = getCurl(url, username, password) 49 50 data = None 51 try: 52 curl.perform() 53 54 code = curl.getinfo( pycurl.HTTP_CODE ) 55 56 if 200 <= code <= 399: 57 data = curl.resp_data.getvalue() 58 else: 59 err_msg = "incorrect http response code: url = <%s>, code = %d" % ( url, code ) 60 raise RuntimeError( err_msg ) 61 62 except pycurl.error, e: 63 curl.close() 64 err_msg = "exception while curl.perform: <%s>, url: <%s>" % ( str(e), url ) 65 raise RuntimeError( err_msg ) 66 67 curl.resp_data.close() 68 curl.close() 69 70 if not data: 71 err_msg = "no data from url(empty response): <%s>" % ( url ) 72 raise RuntimeError( err_msg ) 73 74 return data
75