def download(self, source, dest):
        """
        Download an archive file.

        :param str source: URL pointing to an archive file.
        :param str dest: Local path location to download archive file to.
        """
        # propogate all exceptions
        # URLError, OSError, etc
        proto, netloc, path, params, query, fragment = urlparse(source)
        if proto in ('http', 'https'):
            auth, barehost = splituser(netloc)
            if auth is not None:
                source = urlunparse((proto, barehost, path, params, query, fragment))
                username, password = splitpasswd(auth)
                passman = HTTPPasswordMgrWithDefaultRealm()
                # Realm is set to None in add_password to force the username and password
                # to be used whatever the realm
                passman.add_password(None, source, username, password)
                authhandler = HTTPBasicAuthHandler(passman)
                opener = build_opener(authhandler)
                install_opener(opener)
        response = urlopen(source)
        try:
            with open(dest, 'w') as dest_file:
                dest_file.write(response.read())
        except Exception as e:
            if os.path.isfile(dest):
                os.unlink(dest)
            raise e

    def __init__(
        self,
        cachedir="/tmp",
        api_host_options={},
        urllist=[],
        http_debug=False,
        cookiejar=None,
        offline=False,
        enable_cpio=True,
    ):
        # set up progress bar callback
        if sys.stdout.isatty() and TextMeter:
            self.progress_obj = TextMeter(fo=sys.stdout)
        else:
            self.progress_obj = None

        self.cachedir = cachedir
        self.urllist = urllist
        self.http_debug = http_debug
        self.offline = offline
        self.cpio = {}
        self.enable_cpio = enable_cpio

        passmgr = HTTPPasswordMgrWithDefaultRealm()
        for host in api_host_options:
            passmgr.add_password(None, host, api_host_options[host]["user"], api_host_options[host]["pass"])
        openers = (HTTPBasicAuthHandler(passmgr),)
        if cookiejar:
            openers += (HTTPCookieProcessor(cookiejar),)
        self.gr = OscFileGrabber(progress_obj=self.progress_obj)

  def execute(self, method, *args, **kwargs):
    header = {
        'Content-Type' : 'application/json',
        'User-Agent' : 'python-xbmc'
        }
    # Params are given as a dictionnary
    if len(args) == 1:
      args=args[0]
      params = kwargs
      # Use kwargs for param=value style
    else:
      args = kwargs
    params={}
    params['jsonrpc']='2.0'
    params['id']=self.id
    self.id +=1
    params['method']=method
    params['params']=args

    values=json.dumps(params)
    # HTTP Authentication
    password_mgr = HTTPPasswordMgrWithDefaultRealm()
    password_mgr.add_password(None, self.url, self.username, self.password)
    auth_handler = HTTPBasicAuthHandler(password_mgr)
    opener = build_opener(auth_handler)
    install_opener(opener)

    data = values
    req = Request(self.url, data.encode('utf-8'), header)
    response = urlopen(req)
    the_page = response.read()
    if len(the_page) > 0 :
      return json.load(StringIO(the_page.decode('utf-8')))
    else:
      return None # for readability

def urlopen(url, headers=None, data=None, timeout=None):
    """
    An URL opener with the User-agent set to gPodder (with version)
    """
    username, password = username_password_from_url(url)
    if username is not None or password is not None:
        url = url_strip_authentication(url)
        password_mgr = HTTPPasswordMgrWithDefaultRealm()
        password_mgr.add_password(None, url, username, password)
        handler = HTTPBasicAuthHandler(password_mgr)
        opener = build_opener(handler)
    else:
        opener = build_opener()

    if headers is None:
        headers = {}
    else:
        headers = dict(headers)

    headers.update({'User-agent': USER_AGENT})
    request = Request(url, data=data, headers=headers)
    if timeout is None:
        return opener.open(request)
    else:
        return opener.open(request, timeout=timeout)

 def __init__(self, host, port, username, password):
     self.url = "http://%s:%s" % (host, port)
     
     pwdmgr = HTTPPasswordMgrWithDefaultRealm()
     pwdmgr.add_password(None, self.url, username, password)
     pwdhandler = HTTPBasicAuthHandler(pwdmgr)
     
     self.opener = build_opener(pwdhandler)

def openURL(url_base, data, method='Get', cookies=None, username=None, password=None):
    ''' function to open urls - wrapper around urllib2.urlopen but with additional checks for OGC service exceptions and url formatting, also handles cookies and simple user password authentication'''
    url_base.strip() 
    lastchar = url_base[-1]
    if lastchar not in ['?', '&']:
        if url_base.find('?') == -1:
            url_base = url_base + '?'
        else:
            url_base = url_base + '&'
            
    if username and password:
        # Provide login information in order to use the WMS server
        # Create an OpenerDirector with support for Basic HTTP 
        # Authentication...
        passman = HTTPPasswordMgrWithDefaultRealm()
        passman.add_password(None, url_base, username, password)
        auth_handler = HTTPBasicAuthHandler(passman)
        opener = urllib.request.build_opener(auth_handler)
        openit = opener.open
    else:
        # NOTE: optionally set debuglevel>0 to debug HTTP connection
        #opener = urllib2.build_opener(urllib2.HTTPHandler(debuglevel=0))
        #openit = opener.open
        openit = urlopen
   
    try:
        if method == 'Post':
            req = Request(url_base, data)
            # set appropriate header if posting XML
            try:
                xml = etree.fromstring(data)
                req.add_header('Content-Type', "text/xml")
            except:
                pass
        else:
            req=Request(url_base + data)
        if cookies is not None:
            req.add_header('Cookie', cookies)
        u = openit(req)
    except HTTPError as e: #Some servers may set the http header to 400 if returning an OGC service exception or 401 if unauthorised.
        if e.code in [400, 401]:
            raise ServiceException(e.read())
        else:
            raise e
    # check for service exceptions without the http header set
    if u.info()['Content-Type'] in ['text/xml', 'application/xml']:          
        #just in case 400 headers were not set, going to have to read the xml to see if it's an exception report.
        #wrap the url stram in a extended StringIO object so it's re-readable
        u=RereadableURL(u)      
        se_xml= u.read()
        se_tree = etree.fromstring(se_xml)
        serviceException=se_tree.find('{http://www.opengis.net/ows}Exception')
        if serviceException is None:
            serviceException=se_tree.find('ServiceException')
        if serviceException is not None:
            raise ServiceException(str(serviceException.text).strip())
        u.seek(0) #return cursor to start of u      
    return u

 def init_opener(self):
     """ chinachuのAPIを実行するためのopenerを初期化する """
     pm = HTTPPasswordMgrWithDefaultRealm()
     url = self._config["chinachu"]["apiEndpoint"]
     user = self._config["chinachu"]["username"]
     password = self._config["chinachu"]["password"]
     pm.add_password(None, url, user, password)
     handler = HTTPBasicAuthHandler(pm)
     self._opener = build_opener(handler)

def create_opener(aur_server_tag: str) -> OpenerDirector:
    server = _aur_server(aur_server_tag)
    password_manager = HTTPPasswordMgrWithDefaultRealm()
    password_manager.add_password(realm=None,
                                  uri=server.address,
                                  user=server.user,
                                  passwd=server.password)
    handler = HTTPBasicAuthHandler(password_manager)
    return build_opener(handler)

    def auth(cls, username, password, uri, realm=None, timeout=None):
        '''Create an httplib1 instance witn a basic authentication handler.
The authentication'''
        if realm is None:
            password_mgr = HTTPPasswordMgrWithDefaultRealm()
        else:
            password_mgr = HTTPPasswordMgr()
        password_mgr.add_password(realm, uri, user, passwd)
        opener = HTTPBasicAuthHandler(password_mgr)
        return cls(opener,timeout)

def authorize():
    mal_config = read_mal_config()

    pass_manager = HTTPPasswordMgrWithDefaultRealm()
    pass_manager.add_password(None, 'http://myanimelist.net/api',
                              mal_config['UserName'], mal_config['Password'])

    auth_handler = HTTPBasicAuthHandler(pass_manager)

    opener = build_opener(auth_handler)
    install_opener(opener)

class HttpAuthenticated(HttpTransport):
    """
    Provides basic http authentication that follows the RFC-2617 specification.
    As defined by specifications, credentials are provided to the server
    upon request (HTTP/1.0 401 Authorization Required) by the server only.
    @ivar pm: The password manager.
    @ivar handler: The authentication handler.
    """
    
    def __init__(self, **kwargs):
        """
        @param kwargs: Keyword arguments.
            - B{proxy} - An http proxy to be specified on requests.
                 The proxy is defined as {protocol:proxy,}
                    - type: I{dict}
                    - default: {}
            - B{timeout} - Set the url open timeout (seconds).
                    - type: I{float}
                    - default: 90
            - B{username} - The username used for http authentication.
                    - type: I{str}
                    - default: None
            - B{password} - The password used for http authentication.
                    - type: I{str}
                    - default: None
        """
        HttpTransport.__init__(self, **kwargs)
        self.pm = HTTPPasswordMgrWithDefaultRealm()
        
    def open(self, request):
        self.addcredentials(request)
        return  HttpTransport.open(self, request)
    
    def send(self, request):
        self.addcredentials(request)
        return  HttpTransport.send(self, request)
    
    def addcredentials(self, request):
        credentials = self.credentials()
        if not (None in credentials):
            u = credentials[0]
            p = credentials[1]
            self.pm.add_password(None, request.url, u, p)
    
    def credentials(self):
        return (self.options.username, self.options.password)
    
    def u2handlers(self):
            handlers = HttpTransport.u2handlers(self)
            handlers.append(HTTPBasicAuthHandler(self.pm))
            return handlers

def authenticated_urlopen(location):
    """ A wrapper around urlopen adding authentication information if provided by the user. """
    passman = HTTPPasswordMgrWithDefaultRealm()
    server_name = urlparse.urlsplit(location).netloc
    access = get_server_access(server_name)
    if access is not None:
        user = access.username
        password = access.password
        if user is not None and password is not None:
            passman.add_password(None, location, user, password)
    authhandler = HTTPBasicAuthHandler(passman)
    opener = build_opener(authhandler)
    install_opener(opener)
    return urlopen(location)

    def get_opener(self):
        # The opener is yet build ?
        if self.opener is not None:
            return self.opener

        # Build a new opener
        opener = build_opener()
        headers = [ ('User-agent', 'restedit/%s' % __version__) ]

        # Add the "includes"
        for include in self.includes:
            headers.append(include)

        # An authentication ?
        auth_header = self.metadata.get('auth')
        if auth_header is not None:
            if auth_header.lower().startswith('basic'):
                cls_handler = HTTPBasicAuthHandler
                chal = auth_header[6:].strip()
                # Automatically find the username and the password
                username, password = decode_base64(chal).split(':', 1)
            elif auth_header.lower().startswith('digest'):
                cls_handler = HTTPDigestAuthHandler
                # Automatically find the username, but we must ask the password
                # XXX undocumented functions
                chal = parse_keqv_list(parse_http_list(auth_header[7:]))
                username = chal['username']
                password = askPassword(chal['realm'], username)
            else:
                raise NotImplemented

            password_mgr = HTTPPasswordMgrWithDefaultRealm()
            password_mgr.add_password(realm=None,
                                      uri=self.url,
                                      user=username,
                                      passwd=password)

            auth_handler = cls_handler(password_mgr)
            opener.add_handler(auth_handler)

        # A cookie ?
        if self.metadata.get('cookie'):
            headers.append( ('Cookie', self.metadata['cookie']) )

        # All OK
        opener.addheaders = headers
        self.opener = opener
        return opener

def run_query(search_terms):
    root_url = "https://api.datamarket.azure.com/Bing/Search/"
    source = "Web"
    results_per_page = 10
    offset = 0

    # Wrap quotes around our query terms as required by the Bing API.
    # The query we will then use is stored within variable query.
    query = "'{0}'".format(search_terms)
    query = quote(query)

    # Construct the latter part of our request's URL.
    # Sets the format of the response to JSON and sets other properties.
    search_url = "{0}{1}?$format=json&$top={2}&$skip={3}&Query={4}".format(
        root_url, source, results_per_page, offset, query
    )

    # Setup authentication with the Bing servers.
    # The username MUST be a blank string, and put in your API key!
    username = ""

    # Create a 'password manager' which handles authentication for us.
    password_mgr = HTTPPasswordMgrWithDefaultRealm()
    password_mgr.add_password(None, search_url, username, BING_API_KEY)

    # Create our results list which we'll populate.
    results = []
    print(search_url)
    try:
        # Prepare for connecting to Bing's servers.
        handler = HTTPBasicAuthHandler(password_mgr)
        opener = build_opener(handler)
        install_opener(opener)

        # Connect to the server and read the response generated.
        response = urlopen(search_url).read().decode(encoding="utf-8").replace("<", "")
        # print(response)
        json_response = json.loads(response, parse_int=True)

        # Loop through each page returned, populating out results list.
        for result in json_response["d"]["results"]:
            results.append({"title": result["Title"], "link": result["Url"], "summary": result["Description"]})

    except URLError as e:
        print("Error when querying the Bing API: ", e)

    return results

    def push_to_web_interface(self, json_body_dict):
        destination_url = urljoin(self.notification_server_host, URL_PUSH_PATH)
        data = json.dumps(json_body_dict).encode("utf-8")

        passman = HTTPPasswordMgrWithDefaultRealm()
        un = self.notification_server_username
        pw = self.notification_server_password
        top_level_url = self.notification_server_host
        passman.add_password(None, top_level_url, un, pw)

        auth_handler = HTTPBasicAuthHandler(passman)
        opener = build_opener(auth_handler)
        opener.open(top_level_url)
        install_opener(opener)

        request = Request(destination_url, data=data, headers={'Content-Type': 'application/json',
                                                               'User-Agent': self.user_agent})
        opener.open(request)

def curl(url, params=None, auth=None, req_type="GET", data=None, headers=None):
  post_req = ["POST", "PUT"]
  get_req = ["GET", "DELETE"]

  if params is not None:
    url += "?" + urlencode(params)

  if req_type not in post_req + get_req:
    raise IOError("Wrong request type \"%s\" passed" % req_type)

  _headers = {}
  handler_chain = []

  if auth is not None:
    manager = HTTPPasswordMgrWithDefaultRealm()
    manager.add_password(None, url, auth["user"], auth["pass"])
    handler_chain.append(HTTPBasicAuthHandler(manager))

  if req_type in post_req and data is not None:
    _headers["Content-Length"] = len(data)

  if headers is not None:
    _headers.update(headers)

  director = build_opener(*handler_chain)

  if req_type in post_req:
    if sys.version_info.major == 3:
      _data = bytes(data, encoding='utf8')
    else:
      _data = bytes(data)

    req = Request(url, headers=_headers, data=_data)
  else:
    req = Request(url, headers=_headers)

  req.get_method = lambda: req_type
  result = director.open(req)

  return {
    "httpcode": result.code,
    "headers": result.info(),
    "content": result.read()
  }

def _netrc_open(uri, filename=None):
    '''
    open uri using netrc credentials.

    :param uri: uri to open
    :param filename: optional, path to non-default netrc config file
    :returns: file-like object from opening a socket to uri, or None
    :raises IOError: if opening .netrc file fails (unless file not found)
    '''
    if not uri:
        return None
    parsed_uri = urlparse(uri)
    machine = parsed_uri.netloc
    if not machine:
        return None
    opener = None
    try:
        info = netrc.netrc(filename).authenticators(machine)
        if info is not None:
            (username, _ , password) = info
            if username and password:
                pass_man = HTTPPasswordMgrWithDefaultRealm()
                pass_man.add_password(None, machine, username, password)
                authhandler = HTTPBasicAuthHandler(pass_man)
                opener = build_opener(authhandler)
                return opener.open(uri)
        else:
            # caught below, like other netrc parse errors
            raise netrc.NetrcParseError('No authenticators for "%s"' % machine)
    except IOError as ioe:
        if ioe.errno != 2:
            # if = 2, User probably has no .netrc, this is not an error
            raise
    except netrc.NetrcParseError as neterr:
        logger = logging.getLogger('vcstools')
        logger.warn('WARNING: parsing .netrc: %s' % str(neterr))
    # we could install_opener() here, but prefer to keep
    # default opening clean. Client can do that, though.
    return None

def urlopener_with_auth(url):
    """
    Given a URL, return the URL with auth stripped, and a urlopener that can
    open it.
    Uses urllib2, so SSL certs aren't verified.
    """
    opener = build_opener()
    parsed = urlparse(url)
    if parsed.username and parsed.password:
        host = parsed.hostname
        if parsed.port:
            host += ':%i' % parsed.port
        stripped_auth = (parsed[0], host) + parsed[2:]
        url = urlunparse(stripped_auth)

        passwd_mgr = HTTPPasswordMgrWithDefaultRealm()
        base_url = urlunparse((parsed[0], host, '', '', '', ''))
        passwd_mgr.add_password(realm=None, uri=base_url,
                                user=parsed.username, passwd=parsed.password)
        auth_handler = HTTPBasicAuthHandler(passwd_mgr)
        opener = build_opener(auth_handler)
    return url, opener

    def __init__(self, cachedir='/tmp', api_host_options={}, urllist=[],
            http_debug=False, cookiejar=None, offline=False, enable_cpio=True):
        # set up progress bar callback
        self.progress_obj = None
        if sys.stdout.isatty():
            self.progress_obj = create_text_meter(use_pb_fallback=False)

        self.cachedir = cachedir
        self.urllist = urllist
        self.http_debug = http_debug
        self.offline = offline
        self.cpio = {}
        self.enable_cpio = enable_cpio

        passmgr = HTTPPasswordMgrWithDefaultRealm()
        for host in api_host_options:
            passmgr.add_password(None, host, api_host_options[host]['user'],
                                 api_host_options[host]['pass'])
        openers = (HTTPBasicAuthHandler(passmgr), )
        if cookiejar:
            openers += (HTTPCookieProcessor(cookiejar), )
        self.gr = OscFileGrabber(progress_obj=self.progress_obj)

 def __init__(self, **kwargs):
     """
     @param kwargs: Keyword arguments.
         - B{proxy} - An http proxy to be specified on requests.
              The proxy is defined as {protocol:proxy,}
                 - type: I{dict}
                 - default: {}
         - B{timeout} - Set the url open timeout (seconds).
                 - type: I{float}
                 - default: 90
         - B{username} - The username used for http authentication.
                 - type: I{str}
                 - default: None
         - B{password} - The password used for http authentication.
                 - type: I{str}
                 - default: None
     """
     HttpTransport.__init__(self, **kwargs)
     self.pm = HTTPPasswordMgrWithDefaultRealm()