curlwrapper.py

   1 import os
   2 import xmlrpclib
   3 import urllib
   4 import pycurl
   5 from cStringIO import StringIO
   6
   7 import logger
   8
   9 # a pycurl-based replacement for the previous version that relied on forking curl
  10
  11 debug=False
  12
  13 def retrieve(url, cacert=None, postdata=None, timeout=90):
  14     curl= pycurl.Curl()
  15     curl.setopt(pycurl.URL,url)
  16     if debug: logger.verbose('curlwrapper: new instance %r -> %s'%(curl,url))
  17
  18     # reproduce --fail from the previous version
  19     curl.setopt(pycurl.FAILONERROR,1)
  20     # don't want curl sending any signals
  21     curl.setopt(pycurl.NOSIGNAL, 1)
  22
  23     # do not follow location when attempting to download a file
  24     # curl.setopt(pycurl.FOLLOWLOCATION, 0)
  25
  26     # store result on the fly
  27     buffer=StringIO()
  28     curl.setopt(pycurl.WRITEFUNCTION,buffer.write)
  29
  30     # set timeout
  31     if timeout:
  32         curl.setopt(pycurl.CONNECTTIMEOUT, timeout)
  33         curl.setopt(pycurl.TIMEOUT, timeout)
  34         if debug: logger.verbose('curlwrapper: timeout set to %r'%timeout)
  35
  36     # set cacert
  37     if cacert:
  38         curl.setopt(pycurl.CAINFO, cacert)
  39         curl.setopt(pycurl.SSL_VERIFYPEER, 2)
  40         if debug: logger.verbose('curlwrapper: using cacert %s'%cacert)
  41     else:
  42         curl.setopt(pycurl.SSL_VERIFYPEER, 0)
  43
  44     # set postdata
  45     if postdata:
  46         if isinstance(postdata,dict):
  47             postfields = urllib.urlencode(postdata)
  48             if debug: logger.verbose('curlwrapper: using encoded postfields %s'%postfields)
  49         else:
  50             postfields=postdata
  51             if debug: logger.verbose('curlwrapper: using raw postfields %s'%postfields)
  52         curl.setopt(pycurl.POSTFIELDS, postfields)
  53
  54     # go
  55     try:
  56         curl.perform()
  57         errcode = curl.getinfo(pycurl.HTTP_CODE)
  58
  59         if debug: logger.verbose('curlwrapper: closing pycurl object')
  60         curl.close()
  61
  62         # check the code, return 1 if successfull
  63         if errcode == 60:
  64             raise xmlrpclib.ProtocolError (url,errcode, "SSL certificate validation failed", postdata)
  65         elif errcode != 200:
  66             raise xmlrpclib.ProtocolError (url,errcode, "http error %d"%errcode, postdata)
  67
  68     except pycurl.error, err:
  69         errno, errstr = err
  70         raise xmlrpclib.ProtocolError(url, errno, "curl error %d: '%s'\n" %(errno,curl.errstr()),postdata )
  71
  72     return buffer.getvalue()