return
        
    #客户端方法
    def csqueryConents(self,csdompagination):
        return csdompagination 
        
    def csqueryPagination(self,csdom,pagesPath):
        pages=[]
        for index,item in enumerate(pagesPath):
            csquery=item
            cspage=csdom.Select(csquery)           
            if cspage:                
                #找到所有的a标签 
                children=cspage.Find("a")                
                if children.Length>0:
                    for i in range(0,children.Length):
                        cshyper=CsQuery.CQ.Create(children[i])              
                        href=cshyper.Attr("href")
                        text=cshyper.Text()
                        pagelamda=self.config.cfgContent.Options.PageLamda
                        if pagelamda:
                            str=pagelamda(cshyper,href,text)
                            print "this is type for url : %s" % (type(str))
                            
                            if str:
                                href=str
                                print " this is true"
                            else:
                                print "this is false"
                                continue

                        if href and href[0:1]=="/":
                            proto, rest = urllib2.splittype(self.config.cfgUrl)  
                            host, rest = urllib2.splithost(rest)
                            href=proto+"://"+host+href
                        elif href and href[0:1]=="?":                            
                            proto, rest = urllib2.splittype(self.config.cfgUrl)  
                            host, rest = urllib2.splithost(rest)
                            p=rest.split("?")
                            p[1]=href[1:]
                            href=proto+"://"+host+"?".join(p)
                        elif href.find("http")==-1:
                            proto, rest = urllib2.splittype(self.config.cfgUrl)  
                            host, rest = urllib2.splithost(rest)
                            p_rest=rest.split("/")
                            p_rest[len(p_rest)-1]=href
                            href=proto+"://"+host+"/".join(p_rest)
                            
                        scale=self.config.cfgContent.Options.PageSimilarity          
                        rate=0.0
                        simlilar=StringHelper.LevenshteinDistance(self.__url,href,rate)
                        print "this is simliar :%f " % simlilar[1]
                        if href and simlilar[1]>scale and simlilar[1]<1:
                            pages.append(href) 

def get_host_from_url(url):
	"""
	功能：把url转换为域名
	"""
	root_proto, root_rest = urllib2.splittype(url)
	root_host, root_rest = urllib2.splithost(root_rest)
	return root_host

def _expand_recipe(content, url=''):
    urls = []
    for line in content.splitlines():
        line = line.lstrip().rstrip()
        try:
            target_type, target = line.split(':', 1)
        except ValueError:
            continue # blank line in recipe
        if target_type in ACCEPTED_RECIPE_TYPES:
            if isinstance(target, unicode):
                target = target.encode('utf-8')
            target = target.lstrip().rstrip()
            # translate well-known variables
            for name in COOK_VARIABLES:
                target = target.replace("$"+name, COOK_VARIABLES[name])
            # Check to see if the target is a URL (has a scheme)
            # if not we want to join it to the current url before
            # carrying on.
            scheme, _ = urllib2.splittype(target)
            if not scheme:
                if not '%' in target:
                    target = urllib.quote(target)
                target = urlparse.urljoin(url, target)
            if target_type == 'recipe':
                urls.extend(recipe_to_urls(target))
            else:
                urls.append(target)
    return urls

    def doQuery(self,query,name):
        # urllib doesn't honor user Content-type, use urllib2
        garbage, path = urllib2.splittype(FieldVal(self.site,"url"))
        host, selector = urllib2.splithost(path)
        response=False
        try:
            errmsg= "** An ERROR occurred attempting HTTPS connection to"
            h = httplib.HTTPSConnection(host, timeout=5)

            errmsg= "** An ERROR occurred sending POST request to"
            p = h.request('POST', selector, query, 
                     {"Content-type": "application/x-ofx",
                      "Accept": "*/*, application/x-ofx"}
                     )

            errmsg= "** An ERROR occurred retrieving POST response from"
            #allow up to 30 secs for the server response (it has to assemble the statement)
            h.sock.settimeout(30)      
            response = h.getresponse().read()
            f = file(name,"w")
            f.write(response)
            f.close()
        except Exception as inst:
            self.status = False
            print errmsg, host
            print "   Exception type:", type(inst)
            print "   Exception Val :", inst
            if response:
                print "   HTTPS ResponseCode  :", response.status
                print "   HTTPS ResponseReason:", response.reason

        if h: h.close()

def _add_proxies():
    if sickrage.app.config.proxy_setting:
        sickrage.app.log.debug("Using global proxy: " + sickrage.app.config.proxy_setting)
        scheme, address = urllib2.splittype(sickrage.app.config.proxy_setting)
        address = ('http://{}'.format(sickrage.app.config.proxy_setting),
                   sickrage.app.config.proxy_setting)[scheme]
        return {"http": address, "https": address}

    def download_file(self, url):
        injectd_url = self.extract_url(urllib2.unquote(url))
        try:
            req = urllib2.Request(injectd_url)
            # Set User-Agent to look more credible
            req.add_unredirected_header('User-Agent', '-')
            # FIXME: We need a timeout on read here
            injected_file = urllib2.urlopen(req, timeout=4).read()
            #  If the file is hosted on a SSL enabled host get the certificate
            if re.match('^https', injectd_url, re.IGNORECASE):
                proto, rest = urllib2.splittype(injectd_url)
                host, rest = urllib2.splithost(rest)
                host, port = urllib2.splitport(host)
                if port is None:
                    port = 443

                cert_file = ssl.get_server_certificate((host, int(port)))
                cert_name = self.store_file(cert_file)

        except IOError as e:
            logger.exception("Failed to fetch injected file, I/O error: {0}".format(e))
            # TODO: We want to handle the case where we can't download
            # the injected file but pretend to be vulnerable.
            file_name = None
        else:
            file_name, file_sha256 = self.store_file(injected_file)
        return file_name, file_sha256

    def request(self, method, url, headers=None, params=None, proxies=None, cache=True, verify=False, *args, **kwargs):
        if headers is None: headers = {}
        if params is None: params = {}
        if proxies is None: proxies = {}

        headers['Accept-Encoding'] = 'gzip, deflate'
        headers["User-Agent"] = sickrage.app.user_agent

        # request session ssl verify
        if sickrage.app.config.ssl_verify:
            try:
                verify = certifi.where()
            except:
                pass

        # request session proxies
        if 'Referer' not in headers and sickrage.app.config.proxy_setting:
            sickrage.app.log.debug("Using global proxy: " + sickrage.app.config.proxy_setting)
            scheme, address = urllib2.splittype(sickrage.app.config.proxy_setting)
            address = ('http://{}'.format(sickrage.app.config.proxy_setting),
                       sickrage.app.config.proxy_setting)[scheme]
            proxies.update({"http": address, "https": address})
            headers.update({'Referer': address})

        # setup caching adapter
        if cache:
            adapter = CacheControlAdapter(DBCache(os.path.abspath(os.path.join(sickrage.app.data_dir, 'sessions.db'))))
            self.mount('http://', adapter)
            self.mount('https://', adapter)

        # get web response
        response = super(WebSession, self).request(
            method,
            url,
            headers=headers,
            params=params,
            verify=verify,
            proxies=proxies,
            hooks={'response': WebHooks.log_url},
            *args, **kwargs
        )

        try:
            # check web response for errors
            response.raise_for_status()
        except requests.exceptions.SSLError as e:
            if ssl.OPENSSL_VERSION_INFO < (1, 0, 1, 5):
                sickrage.app.log.info(
                    "SSL Error requesting url: '{}' You have {}, try upgrading OpenSSL to 1.0.1e+".format(
                        e.request.url, ssl.OPENSSL_VERSION))

            if sickrage.app.config.ssl_verify:
                sickrage.app.log.info(
                    "SSL Error requesting url: '{}', try disabling cert verification in advanced settings".format(
                        e.request.url))
        except Exception:
            pass

        return response

    def request(self, method, url, headers=None, params=None, proxies=None, cache=True, verify=False, *args, **kwargs):
        if headers is None: headers = {}
        if params is None: params = {}
        if proxies is None: proxies = {}

        url = self.normalize_url(url)

        headers.update({'Accept-Encoding': 'gzip, deflate'})
        headers.update(random.choice(USER_AGENTS))

        # request session ssl verify
        if sickrage.srCore.srConfig.SSL_VERIFY:
            try:
                verify = certifi.where()
            except:
                pass

        # request session proxies
        if 'Referer' not in headers and sickrage.srCore.srConfig.PROXY_SETTING:
            sickrage.srCore.srLogger.debug("Using global proxy: " + sickrage.srCore.srConfig.PROXY_SETTING)
            scheme, address = urllib2.splittype(sickrage.srCore.srConfig.PROXY_SETTING)
            address = ('http://{}'.format(sickrage.srCore.srConfig.PROXY_SETTING),
                       sickrage.srCore.srConfig.PROXY_SETTING)[scheme]
            proxies.update({"http": address, "https": address})
            headers.update({'Referer': address})

        # setup session caching
        if cache:
            cache_file = os.path.abspath(os.path.join(sickrage.DATA_DIR, 'sessions.db'))
            self.__class__ = cachecontrol.CacheControl(self,
                                                       cache=DBCache(cache_file),
                                                       heuristic=ExpiresAfter(days=7)).__class__

        # get web response
        response = super(srSession, self).request(method,
                                                  url,
                                                  headers=headers,
                                                  params=params,
                                                  verify=verify,
                                                  proxies=proxies,
                                                  *args, **kwargs)

        try:
            # check web response for errors
            response.raise_for_status()
        except requests.exceptions.SSLError as e:
            if ssl.OPENSSL_VERSION_INFO < (1, 0, 1, 5):
                sickrage.srCore.srLogger.info(
                    "SSL Error requesting url: '{}' You have {}, try upgrading OpenSSL to 1.0.1e+".format(
                        e.request.url, ssl.OPENSSL_VERSION))

            if sickrage.srCore.srConfig.SSL_VERIFY:
                sickrage.srCore.srLogger.info(
                    "SSL Error requesting url: '{}', try disabling cert verification in advanced settings".format(
                        e.request.url))
        except Exception:
            pass

        return response

    def download(self, url, insecure):
        """ Tries to download a file from url.

        Returns the path to the local file.
        """
        scheme = urllib2.splittype(url)[0]
        DL = downloaders.get(scheme, Downloader)
        return DL(url, self, insecure).execute()

 def decorator(*args, **kwargs):
     request = args[0]
     enabled_https = getattr(settings, 'SESSION_COOKIE_SECURE', False)
     if enabled_https and not request.is_secure():
         http_url = request.build_absolute_uri(request.get_full_path())
         https_url = 'https:' + urllib2.splittype(http_url)[1]
         return HttpResponseRedirect(https_url)
     return func(*args, **kwargs)

def get_local_name(url):
    url = url.strip()
    url = re.sub('[\/]+$', '', url)
    rest = urllib2.splittype(url)[1]
    host, rest = urllib2.splithost(rest)
    if rest is None or  rest == '':
        return host
    return os.path.basename(rest)

def _setUpSession(session=None, headers=None, params=None):
    """
    Returns a session initialized with default cache and parameter settings

    :param session: session object to (re)use
    :param headers: Headers to pass to session
    :return: session object
    """

    # request session
    if headers is None:
        headers = {}

    sessionCache = None
    FileCacheDir = sickrage.srConfig.CACHE_DIR or get_temp_dir()
    if FileCacheDir:
        sessionCache = FileCache(os.path.join(FileCacheDir, 'sessions'), use_dir_lock=True)
    session = cachecontrol.CacheControl(sess=session or requests.Session(), cache=sessionCache, cache_etags=False)

    # request session headers
    session.headers.update(headers)
    session.headers.update({'Accept-Encoding': 'gzip,deflate'})
    session.headers.update(random.choice(USER_AGENTS))

    # request session clear residual referer
    if 'Referer' in session.headers and 'Referer' not in headers:
        session.headers.pop('Referer')

    try:
        # request session ssl verify
        session.verify = False
        if sickrage.srConfig.SSL_VERIFY:
            session.verify = certifi.where()
    except:pass

    # request session proxies
    if 'Referer' not in session.headers and sickrage.srConfig.PROXY_SETTING:
        sickrage.srLogger.debug("Using global proxy: " + sickrage.srConfig.PROXY_SETTING)
        scheme, address = urllib2.splittype(sickrage.srConfig.PROXY_SETTING)
        address = ('http://{}'.format(sickrage.srConfig.PROXY_SETTING), sickrage.srConfig.PROXY_SETTING)[scheme]
        session.proxies = {
            "http": address,
            "https": address,
        }
        session.headers.update({'Referer': address})

    if 'Content-Type' in session.headers:
        session.headers.pop('Content-Type')

    if params and isinstance(params, (list, dict)):
        for param in params:
            if isinstance(params[param], unicode):
                params[param] = params[param].encode('utf-8')
        session.params = params

    return session

    def getRSSFeed(self, url, params=None):
        handlers = []

        if sickrage.app.config.proxy_setting:
            sickrage.app.log.debug("Using global proxy for url: " + url)
            scheme, address = urllib2.splittype(sickrage.app.config.proxy_setting)
            address = sickrage.app.config.proxy_setting if scheme else 'http://' + sickrage.app.config.proxy_setting
            handlers = [urllib2.ProxyHandler({'http': address, 'https': address})]

        return getFeed(url, params=params, handlers=handlers)

    def getRSSFeed(self, url, params=None):
        handlers = []

        if sickrage.srCore.srConfig.PROXY_SETTING:
            sickrage.srCore.srLogger.debug("Using global proxy for url: " + url)
            scheme, address = urllib2.splittype(sickrage.srCore.srConfig.PROXY_SETTING)
            address = sickrage.srCore.srConfig.PROXY_SETTING if scheme else 'http://' + sickrage.srCore.srConfig.PROXY_SETTING
            handlers = [urllib2.ProxyHandler({'http': address, 'https': address})]

        return getFeed(url, params=params, handlers=handlers)

 def _setup_server(self, server=None):
     if server:
         host, path = urllib2.splithost(urllib2.splittype(server)[-1])
         if not path:
             path = '/'
         self.client_con = python_webdav.client.Client(host,
                                                       webdav_path=path)
         self.client_con.set_connection('wibble', 'fish')
     else:
         print "I need a server!"
         self.client_con = None

def url_size(url):
	import httplib, urllib2
	proto, url = urllib2.splittype(url)
	assert(proto.lower() == 'http')
	host, path = urllib2.splithost(url)
	# http://stackoverflow.com/questions/107405/how-do-you-send-a-head-http-request-in-python
	conn = httplib.HTTPConnection(host)
	conn.request('HEAD', path)
	res = conn.getresponse()
	# FIXME: Follow any redirects
	return int(res.getheader('content-length'))

 def lamda(self,csblock):
     href=csblock[0].Attr("href") 
     if not href:
         href=csblock[1]
     if href:
         href=href.replace("\\","").replace("\"","")   
     if href and href[0:1]=="/":
         proto, rest = urllib2.splittype(self.config.cfgUrl)  
         host, rest = urllib2.splithost(rest)
         href=proto+"://"+host+href        
     return href

            echo(content)
        except Exception as err:
            pass
        finally:
            return content

    def pageCsContentImage(self,cspage):
        """本地img替换为完全img路径"""
        proto, rest = urllib2.splittype(self.config.cfgUrl)  
        host, rest = urllib2.splithost(rest)
        csimgs=cspage.Find("img")        

def parse_protocols(ctx, base_uri=None):
    """ Parse ``protocols`` from a root context.

    If protocols are not provided in root, use baseUri protocol.
    """
    protocols = ctx.get_property_with_schema(
        'protocols', RamlRoot.protocols)
    if protocols is None and base_uri is not None:
        protocols = [urllib2.splittype(base_uri)[0]]
    if protocols:
        protocols = [p.upper() for p in protocols]
    return protocols

def RemoteAccess(url, *args, **kwargs):
    """Connect to a remote Subversion server

    :param url: URL to connect to
    :return: RemoteAccess object
    """
    if isinstance(url, bytes):
        url = url.decode("utf-8")
    (type, opaque) = splittype(url)
    if type not in url_handlers:
        raise SubversionException("Unknown URL type '%s'" % type, ERR_BAD_URL)
    return url_handlers[type](url, *args, **kwargs)