def retrieveDNAPDBonServer(self,path,name=None,pathTo=None):
        done = False
        cut= 0
        dnafile = None
        print ("http://w3dna.rutgers.edu/"+path[1:-1]+"/s0.pdb")
        if name is None :
            name = "s0.pdb"
        if pathTo is None :
            pathTo = self.vf.rcFolder+os.sep+"pdbcache"+os.sep 
        tmpFileName =  pathTo+name   
        while not done :
            if cut > 100 :        
                break
            try :
#                dnafile = urllib2.urlopen("http://w3dna.rutgers.edu/data/usr/"+path+"/rebuild/s0.pdb")
#                dnafile = urllib2.urlopen("http://w3dna.rutgers.edu/"+path[1:-1]+"/s0.pdb")
                urllib.urlretrieve("http://w3dna.rutgers.edu/"+path[1:-1]+"/s0.pdb", tmpFileName)
                done = True
            except :
                cut+=1        
                continue
        if done :
            #should download in the  rcFolder
#
#            output = open(pathTo+name,'w')
#            output.write(dnafile.read())
#            output.close()
            return name,pathTo
        return None,None

    def getArtwork(self, mp4Path, filename='cover', thumbnail=False):
        # Check for local cover.jpg or cover.png artwork in the same directory as the mp4
        extensions = valid_poster_extensions
        poster = None
        for e in extensions:
            head, tail = os.path.split(os.path.abspath(mp4Path))
            path = os.path.join(head, filename + os.extsep + e)
            if (os.path.exists(path)):
                poster = path
                self.log.info("Local artwork detected, using %s." % path)
                break
        # Pulls down all the poster metadata for the correct season and sorts them into the Poster object
        if poster is None:
            if thumbnail:
                try:
                    poster = urlretrieve(self.episodedata['filename'], os.path.join(tempfile.gettempdir(), "poster-%s.jpg" % self.title))[0]
                except Exception as e:
                    self.log.error("Exception while retrieving poster %s.", str(e))
                    poster = None
            else:
                posters = posterCollection()
                try:
                    for bannerid in self.showdata['_banners']['season']['season'].keys():
                        if str(self.showdata['_banners']['season']['season'][bannerid]['season']) == str(self.season):
                            poster = Poster()
                            poster.ratingcount = int(self.showdata['_banners']['season']['season'][bannerid]['ratingcount'])
                            if poster.ratingcount > 0:
                                poster.rating = float(self.showdata['_banners']['season']['season'][bannerid]['rating'])
                            poster.bannerpath = self.showdata['_banners']['season']['season'][bannerid]['_bannerpath']
                            posters.addPoster(poster)

                    poster = urlretrieve(posters.topPoster().bannerpath, os.path.join(tempfile.gettempdir(), "poster-%s.jpg" % self.title))[0]
                except:
                    poster = None
        return poster

def _load_mnist():
    data_dir = os.path.dirname(os.path.abspath(__file__))
    data_file = os.path.join(data_dir, "mnist.pkl.gz")

    print("Looking for data file: ", data_file)

    if not os.path.isfile(data_file):
        import urllib.request as url

        origin = "http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz"
        print("Downloading data from: ", origin)
        url.urlretrieve(origin, data_file)

    print("Loading MNIST data")
    f = gzip.open(data_file, "rb")
    u = pickle._Unpickler(f)
    u.encoding = "latin1"
    train_set, valid_set, test_set = u.load()
    f.close()

    train_x, train_y = train_set
    valid_x, valid_y = valid_set
    testing_x, testing_y = test_set

    training_x = np.vstack((train_x, valid_x))
    training_y = np.concatenate((train_y, valid_y))

    training_x = training_x.reshape((training_x.shape[0], 1, 28, 28))
    testing_x = testing_x.reshape((testing_x.shape[0], 1, 28, 28))

    return training_x, training_y, testing_x, testing_y

    def download_results(self, savedir=None, only_raw=True, only_calib=False,
                         index=None):
        """Download the previously found and stored Opus obsids.

        Parameters
        ==========
        savedir: str or pathlib.Path, optional
            If the database root folder as defined by the config.ini should not be used,
            provide a different savedir here. It will be handed to PathManager.
        """
        obsids = self.obsids if index is None else [self.obsids[index]]
        for obsid in obsids:
            pm = io.PathManager(obsid.img_id, savedir=savedir)
            pm.basepath.mkdir(exist_ok=True)
            if only_raw is True:
                to_download = obsid.raw_urls
            elif only_calib is True:
                to_download = obsid.calib_urls
            else:
                to_download = obsid.all_urls
            for url in to_download:
                basename = Path(url).name
                print("Downloading", basename)
                store_path = str(pm.basepath / basename)
                try:
                    urlretrieve(url, store_path)
                except Exception as e:
                    urlretrieve(url.replace('https', 'http'), store_path)
            return str(pm.basepath)

def install_from_source(setuptools_source, pip_source):
    setuptools_temp_dir = tempfile.mkdtemp('-setuptools', 'ptvs-')
    pip_temp_dir = tempfile.mkdtemp('-pip', 'ptvs-')
    cwd = os.getcwd()

    try:
        os.chdir(setuptools_temp_dir)
        print('Downloading setuptools from ' + setuptools_source)
        sys.stdout.flush()
        setuptools_package, _ = urlretrieve(setuptools_source, 'setuptools.tar.gz')

        package = tarfile.open(setuptools_package)
        try:
            safe_members = [m for m in package.getmembers() if not m.name.startswith(('..', '\\'))]
            package.extractall(setuptools_temp_dir, members=safe_members)
        finally:
            package.close()

        extracted_dirs = [d for d in os.listdir(setuptools_temp_dir) if os.path.exists(os.path.join(d, 'setup.py'))]
        if not extracted_dirs:
            raise OSError("Failed to find setuptools's setup.py")
        extracted_dir = extracted_dirs[0]

        print('\nInstalling from ' + extracted_dir)
        sys.stdout.flush()
        os.chdir(extracted_dir)
        subprocess.check_call(
            EXECUTABLE + ['setup.py', 'install', '--single-version-externally-managed', '--record', 'setuptools.txt']
        )

        os.chdir(pip_temp_dir)
        print('Downloading pip from ' + pip_source)
        sys.stdout.flush()
        pip_package, _ = urlretrieve(pip_source, 'pip.tar.gz')

        package = tarfile.open(pip_package)
        try:
            safe_members = [m for m in package.getmembers() if not m.name.startswith(('..', '\\'))]
            package.extractall(pip_temp_dir, members=safe_members)
        finally:
            package.close()

        extracted_dirs = [d for d in os.listdir(pip_temp_dir) if os.path.exists(os.path.join(d, 'setup.py'))]
        if not extracted_dirs:
            raise OSError("Failed to find pip's setup.py")
        extracted_dir = extracted_dirs[0]

        print('\nInstalling from ' + extracted_dir)
        sys.stdout.flush()
        os.chdir(extracted_dir)
        subprocess.check_call(
            EXECUTABLE + ['setup.py', 'install', '--single-version-externally-managed', '--record', 'pip.txt']
        )

        print('\nInstallation Complete')
        sys.stdout.flush()
    finally:
        os.chdir(cwd)
        shutil.rmtree(setuptools_temp_dir, ignore_errors=True)
        shutil.rmtree(pip_temp_dir, ignore_errors=True)

def download(url):
    """ saves url to current folder. returns filename """
    #TODO: check that download was successful
    filename = os.path.basename(url)
    print('downloading', filename)
    urlretrieve(url, filename)
    return filename

def get_tool(tool):
    sys_name = platform.system()
    archive_name = tool['archiveFileName']
    local_path = dist_dir + archive_name
    url = tool['url']
    #real_hash = tool['checksum'].split(':')[1]
    if not os.path.isfile(local_path):
        print('Downloading ' + archive_name);
        sys.stdout.flush()
        if 'CYGWIN_NT' in sys_name:
            ctx = ssl.create_default_context()
            ctx.check_hostname = False
            ctx.verify_mode = ssl.CERT_NONE
            urlretrieve(url, local_path, report_progress, context=ctx)
        elif 'Windows' in sys_name:
            r = requests.get(url)
            f = open(local_path, 'wb')
            f.write(r.content)
            f.close()
        else:
            urlretrieve(url, local_path, report_progress)
        sys.stdout.write("\rDone\n")
        sys.stdout.flush()
    else:
        print('Tool {0} already downloaded'.format(archive_name))
        sys.stdout.flush()
    #local_hash = sha256sum(local_path)
    #if local_hash != real_hash:
    #    print('Hash mismatch for {0}, delete the file and try again'.format(local_path))
    #    raise RuntimeError()
    unpack(local_path, '.')

def navigate_dl():
    browser.get('https://www.urltodlownloadfrom.com/specificaddress')

    while True:
        wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "body > div.course-mainbar.lecture-content > "
                                                                      "div:nth-child(2) > div.video-options > a")))
        dl_url = browser.find_element_by_css_selector("body > div.course-mainbar.lecture-content > "
                                                      "div:nth-child(2) > div.video-options > a").get_attribute("href")
        next_btn = browser.find_element_by_css_selector("#lecture_complete_button > span")

        title = get_title()

        try:
            dl_extras = browser.find_element_by_css_selector("body > div.course-mainbar.lecture-content > "
                                                             "div:nth-child(4) > div:nth-child(3) > a").get_attribute("href")
            print(dl_extras)
            urlretrieve(dl_extras, save_path + title + '.pptx', reporthook)
        except NoSuchElementException:
            pass

        try:
            print(dl_url)
            urlretrieve(dl_url, save_path+title+'.mp4', reporthook)
            next_btn.click()
        except NoSuchElementException:
            break

 def do_local(self):
     attrs_dict = [tag[1] for tag in self.list_tags]
     for attrs in attrs_dict:
         if 'src' in attrs and 'http://' in attrs['src']:
             filename = attrs['src'].split('/')[-1]
             urlretrieve(attrs['src'], filename)
             attrs['src'] = filename

 def retrieve_nxml_abstract(pmid, outfile = None):
     """
     Retrieves nxml file of the abstract associated with the provided pmid
     """
     query = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id={}&rettype=abstract".format(pmid)
     nxml_file = outfile or "{}.nxml".format(pmid)
     urlretrieve(query, nxml_file)

 def retrieve_nxml_paper(pmcid, outfile = None):
     """
     Retrieves nxml file for the provided pmcid
     """
     query = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pmc&id={}".format(pmcid)
     nxml_file = outfile or "{}.nxml".format(pmcid)
     urlretrieve(query, nxml_file)

    def fetch(self, directory):
        if not os.path.exists(directory):
            os.makedirs(directory)

        self.path = os.path.join(directory, self.filename)

        url = self.artifact
        if self.check_sum():
            logger.info("Using cached artifact for %s" % self.filename)
            return

        logger.info("Fetching %s from  %s." % (self.filename, url))

        try:
            if os.path.basename(url) == url:
                raise CCTError("Artifact is referenced by filename - can't download it.")
            urlrequest.urlretrieve(url, self.path)
        except Exception as ex:
            if self.hint:
                raise CCTError('artifact: "%s" was not found. %s' % (self.path, self.hint))
            else:
                raise CCTError("cannot download artifact from url %s, error: %s" % (url, ex))

        if not self.check_sum():
            if self.hint:
                raise CCTError('hash is not correct for artifact: "%s". %s' % (self.path, self.hint))
            else:
                raise CCTError("artifact from %s doesn't match required chksum" % url)

 def _download_episode(self, url, title):
     """Save the video stream to the disk.
     The filename will be sanitized(title).mp4."""
     filename = self._sanitize_filename(title) + '.mp4'
     print('Downloading {}...'.format(title))
     filename = '{}/{}'.format(self.folder, filename)
     urlretrieve(url, filename=filename)

def _fetch_remote(remote, dirname=None):
    """Helper function to download a remote dataset into path

    Fetch a dataset pointed by remote's url, save into path using remote's
    filename and ensure its integrity based on the SHA256 Checksum of the
    downloaded file.

    Parameters
    ----------
    remote : RemoteFileMetadata
        Named tuple containing remote dataset meta information: url, filename
        and checksum

    dirname : string
        Directory to save the file to.

    Returns
    -------
    file_path: string
        Full path of the created file.
    """

    file_path = (remote.filename if dirname is None
                 else join(dirname, remote.filename))
    urlretrieve(remote.url, file_path)
    checksum = _sha256(file_path)
    if remote.checksum != checksum:
        raise IOError("{} has an SHA256 checksum ({}) "
                      "differing from expected ({}), "
                      "file may be corrupted.".format(file_path, checksum,
                                                      remote.checksum))
    return file_path

def GetOsmTileData(z,x,y):
  """Download OSM data for the region covering a slippy-map tile"""
  if(x < 0 or y < 0 or z < 0 or z > 25):
    print("Disallowed (%d,%d) at zoom level %d" % (x, y, z))
    return
  
  directory = 'cache/%d/%d/%d' % (z,x,y)
  filename = '%s/data.osm.pkl' % (directory)
  if(not os.path.exists(directory)):
    os.makedirs(directory)

  if(z == DownloadLevel()):
    # Download the data
    s,w,n,e = tileEdges(x,y,z)
    # /api/0.6/map?bbox=left,bottom,right,top
    URL = 'http://api.openstreetmap.org/api/0.6/map?bbox={},{},{},{}'.format(w,s,e,n)

     
    if(not os.path.exists(filename)): # TODO: allow expiry of old data
      urlretrieve(URL, filename)
    return(filename)
    
  elif(z > DownloadLevel()):
    # use larger tile
    while(z > DownloadLevel()):
      z = z - 1
      x = int(x / 2)
      y = int(y / 2)
    return(GetOsmTileData(z,x,y))
  return(None)

def main():
    scripts_path = os.path.dirname(os.path.abspath(__file__))
    repo = os.path.dirname(scripts_path)
    cldr_dl_path = os.path.join(repo, 'cldr')
    cldr_path = os.path.join(repo, 'cldr', os.path.splitext(FILENAME)[0])
    zip_path = os.path.join(cldr_dl_path, FILENAME)
    changed = False

    while not is_good_file(zip_path):
        log('Downloading \'%s\'', FILENAME)
        if os.path.isfile(zip_path):
            os.remove(zip_path)
        urlretrieve(URL, zip_path, reporthook)
        changed = True
        print()
    common_path = os.path.join(cldr_path, 'common')

    if changed or not os.path.isdir(common_path):
        if os.path.isdir(common_path):
            log('Deleting old CLDR checkout in \'%s\'', cldr_path)
            shutil.rmtree(common_path)

        log('Extracting CLDR to \'%s\'', cldr_path)
        with contextlib.closing(zipfile.ZipFile(zip_path)) as z:
            z.extractall(cldr_path)

    subprocess.check_call([
        sys.executable,
        os.path.join(scripts_path, 'import_cldr.py'),
        common_path])

def run(params):
    bucket_name, prefix_name, key_name = params
    s3_key_name = '{}/{}'.format(prefix_name, key_name)
    git_key_url = 'http://data.githubarchive.org/{}'.format(key_name)
    print('Processing {} to s3...'.format(s3_key_name))

    s3_conn = S3Connection()
    bucket = s3_conn.get_bucket(bucket_name)
    key = bucket.get_key(s3_key_name)

    if key:
        print('{} is already in the bucket'.format(key))
    elif exists_url(git_key_url) is False:
        print('{} does not exist'.format(git_key_url))
    else:
        urlretrieve(git_key_url, key_name)

        # pre-process data
        preprocess.process_file(key_name)

        retry_count = 0
        while not upload_to_s3(key_name, bucket, s3_key_name) and retry_count <= MAX_RETRIES:
            retry_count += 1
            print('Failed to upload {} !'.format(s3_key_name))
        else:
            print('File {} is uploaded to {}/{}!'.format(key_name, bucket_name, prefix_name))
        os.remove(key_name)

    def download(self):
        if 'rc' in self.version:
            base_url = 'https://git.kernel.org/torvalds/t'
            url = '{0}/linux-{1}.tar.gz'.format(base_url, self.version)
        else:
            base_url = 'https://cdn.kernel.org/pub/linux/kernel'
            major = 'v' + self.version[0] + '.x'
            url = '{0}/{1}/linux-{2}.tar.gz'.format(
                base_url,
                major,
                self.version
            )

        destination = '{0}/archives/linux-{1}.tar.gz'.format(
            self.build_dir,
            self.version
        )

        if os.path.isfile(destination):
            self.log('Kernel already downloaded: {0}'.format(self.version))
            return
        self.log('Downloading kernel: {0}'.format(self.version))
        if self.verbose:
            hook = download_progress
        else:
            hook = None
        try:
            urlretrieve(
                url,
                filename=destination,
                reporthook=hook
            )
        except Exception:
            os.remove(destination)
            raise

def download_grocery_data():
    base_folder = os.path.dirname(os.path.abspath(__file__))
    dataset_folder = os.path.join(base_folder, "..")
    if not os.path.exists(os.path.join(dataset_folder, "Grocery", "testImages")):
        filename = os.path.join(dataset_folder, "Grocery.zip")
        if not os.path.exists(filename):
            url = "https://www.cntk.ai/DataSets/Grocery/Grocery.zip"
            print('Downloading data from ' + url + '...')
            urlretrieve(url, filename)
            
        try:
            print('Extracting ' + filename + '...')
            with zipfile.ZipFile(filename) as myzip:
                myzip.extractall(dataset_folder)
            if platform != "win32":
                testfile  = os.path.join(dataset_folder, "Grocery", "test.txt")
                unixfile = os.path.join(dataset_folder, "Grocery", "test_unix.txt")
                out = open(unixfile, 'w')
                with open(testfile) as f:
                    for line in f:
                        out.write(line.replace('\\', '/'))
                out.close()
                shutil.move(unixfile, testfile)
        finally:
            os.remove(filename)
        print('Done.')
    else:
        print('Data already available at ' + dataset_folder + '/Grocery')

def get_remote(url = 'http://www.trainingimages.org/uploads/3/4/7/0/34703305/ti_strebelle.sgems',local_file = 'ti.dat', is_zip=0, filename_in_zip=''):
    #import os    
    
    if (is_zip==1):
        local_file_zip = local_file + '.zip'
    
    if not (os.path.exists(local_file)):
        if (is_zip==1):
            import zipfile
            # download zip file
            print('Beginning download of ' + url + ' to ' + local_file_zip)
            urlretrieve(url, local_file_zip)
            # unzip file
            print('Unziping %s to %s' % (local_file_zip,local_file))
            zip_ref = zipfile.ZipFile(local_file_zip, 'r')
            zip_ref.extractall('.')
            zip_ref.close()
            # rename unzipped file            
            if len(filename_in_zip)>0:
                os.rename(filename_in_zip,local_file)
            
            
        else:
            print('Beginning download of ' + url + ' to ' + local_file)
            urlretrieve(url, local_file)
        
        
    return local_file