Python mxnet.gluon.utils.download() Examples

def download_img_labels():
    """ Download an image and imagenet1k class labels for test"""
    from mxnet.gluon.utils import download

    img_name = 'cat.png'
    synset_url = ''.join(['https://gist.githubusercontent.com/zhreshold/',
                      '4d0b62f3d01426887599d4f7ede23ee5/raw/',
                      '596b27d23537e5a1b5751d2b0481ef172f58b539/',
                      'imagenet1000_clsid_to_human.txt'])
    synset_name = 'synset.txt'
    download('https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true', img_name)
    download(synset_url, synset_name)

    with open(synset_name) as fin:
        synset = eval(fin.read())

    with open("synset.csv", "w") as fout:
        w = csv.writer(fout)
        w.writerows(synset.items()) 

def _get_model():
    if not os.path.exists('model/Inception-7-symbol.json'):
        download('http://data.mxnet.io/models/imagenet/inception-v3.tar.gz')
        with tarfile.open(name="inception-v3.tar.gz", mode="r:gz") as tf:
            tf.extractall() 

def _get_data(shape):
    hash_test_img = "355e15800642286e7fe607d87c38aeeab085b0cc"
    hash_inception_v3 = "91807dfdbd336eb3b265dd62c2408882462752b9"
    utils.download("http://data.mxnet.io/data/test_images_%d_%d.npy" % (shape),
                   path="data/test_images_%d_%d.npy" % (shape),
                   sha1_hash=hash_test_img)
    utils.download("http://data.mxnet.io/data/inception-v3-dump.npz",
                   path='data/inception-v3-dump.npz',
                   sha1_hash=hash_inception_v3) 

def get_workload(model_path):
    """ Import workload from frozen protobuf

    Parameters
    ----------
    model_path: str
        model_path on remote repository to download from.

    Returns
    -------
    graph_def: graphdef
        graph_def is the tensorflow workload for mobilenet.

    """

    repo_base = 'https://github.com/dmlc/web-data/raw/master/tensorflow/models/'
    model_name = os.path.basename(model_path)
    model_url = os.path.join(repo_base, model_path)

    from mxnet.gluon.utils import download

    temp = util.tempdir()
    path_model = temp.relpath(model_name)

    download(model_url, path_model)

    # Creates graph from saved graph_def.pb.
    with tf.gfile.FastGFile(path_model, 'rb') as f:
        graph_def = tf.GraphDef()
        graph_def.ParseFromString(f.read())
        graph = tf.import_graph_def(graph_def, name='')
        temp.remove()
        return graph_def

#######################################################################
# PTB LSTMBlockCell Model
# ----------------------- 

def get_workload_ptb():
    """ Import ptb workload from frozen protobuf

    Parameters
    ----------
        Nothing.

    Returns
    -------
    graph_def: graphdef
        graph_def is the tensorflow workload for ptb.

    word_to_id : dict
        English word to integer id mapping

    id_to_word : dict
        Integer id to English word mapping
    """
    sample_repo = 'http://www.fit.vutbr.cz/~imikolov/rnnlm/'
    sample_data_file = 'simple-examples.tgz'
    sample_url = sample_repo+sample_data_file
    ptb_model_file = 'RNN/ptb/ptb_model_with_lstmblockcell.pb'

    import tarfile
    from tvm.contrib.download import download
    DATA_DIR = './ptb_data/'
    if not os.path.exists(DATA_DIR):
        os.mkdir(DATA_DIR)
    download(sample_url, DATA_DIR+sample_data_file)
    t = tarfile.open(DATA_DIR+sample_data_file, 'r')
    t.extractall(DATA_DIR)

    word_to_id, id_to_word = _create_ptb_vocabulary(DATA_DIR)
    return word_to_id, id_to_word, get_workload(ptb_model_file) 

def download_imdb(data_dir='../data'):
    """Download the IMDB data set for sentiment analysis."""
    url = ('http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz')
    sha1 = '01ada507287d82875905620988597833ad4e0903'
    fname = gutils.download(url, data_dir, sha1_hash=sha1)
    with tarfile.open(fname, 'r') as f:
        f.extractall(data_dir) 

def _download_pikachu(data_dir):
    root_url = ('https://apache-mxnet.s3-accelerate.amazonaws.com/'
                'gluon/dataset/pikachu/')
    dataset = {'train.rec': 'e6bcb6ffba1ac04ff8a9b1115e650af56ee969c8',
               'train.idx': 'dcf7318b2602c06428b9988470c731621716c393',
               'val.rec': 'd6c33f799b4d058e82f2cb5bd9a976f69d72d520'}
    for k, v in dataset.items():
        gutils.download(root_url + k, os.path.join(data_dir, k), sha1_hash=v) 

def download_voc_pascal(data_dir='../data'):
    """Download the Pascal VOC2012 Dataset."""
    voc_dir = os.path.join(data_dir, 'VOCdevkit/VOC2012')
    url = ('http://host.robots.ox.ac.uk/pascal/VOC/voc2012'
           '/VOCtrainval_11-May-2012.tar')
    sha1 = '4e443f8a2eca6b1dac8a6c57641b67dd40621a49'
    fname = gutils.download(url, data_dir, sha1_hash=sha1)
    with tarfile.open(fname, 'r') as f:
        f.extractall(data_dir)
    return voc_dir 

def download_imdb(data_dir='../data'):
    """Download the IMDB data set for sentiment analysis."""
    url = ('http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz')
    sha1 = '01ada507287d82875905620988597833ad4e0903'
    fname = gutils.download(url, data_dir, sha1_hash=sha1)
    with tarfile.open(fname, 'r') as f:
        f.extractall(data_dir) 

def _download_pikachu(data_dir):
    root_url = ('https://apache-mxnet.s3-accelerate.amazonaws.com/'
                'gluon/dataset/pikachu/')
    dataset = {'train.rec': 'e6bcb6ffba1ac04ff8a9b1115e650af56ee969c8',
               'train.idx': 'dcf7318b2602c06428b9988470c731621716c393',
               'val.rec': 'd6c33f799b4d058e82f2cb5bd9a976f69d72d520'}
    for k, v in dataset.items():
        gutils.download(root_url + k, os.path.join(data_dir, k), sha1_hash=v) 

def _get_model():
    if not os.path.exists('model/Inception-7-symbol.json'):
        download('http://data.mxnet.io/models/imagenet/inception-v3.tar.gz', dirname='model')
        os.system("cd model; tar -xf inception-v3.tar.gz --strip-components 1") 

def _get_data(shape):
    hash_test_img = "355e15800642286e7fe607d87c38aeeab085b0cc"
    hash_inception_v3 = "91807dfdbd336eb3b265dd62c2408882462752b9"
    utils.download("http://data.mxnet.io/data/test_images_%d_%d.npy" % (shape),
                   path="data/test_images_%d_%d.npy" % (shape),
                   sha1_hash=hash_test_img)
    utils.download("http://data.mxnet.io/data/inception-v3-dump.npz",
                   path='data/inception-v3-dump.npz',
                   sha1_hash=hash_inception_v3) 

def get_model_file(model_name,
                   local_model_store_dir_path=os.path.join("~", ".mxnet", "models")):
    """
    Return location for the pretrained on local file system. This function will download from online model zoo when
    model cannot be found or has mismatch. The root directory will be created if it doesn't exist.

    Parameters
    ----------
    model_name : str
        Name of the model.
    local_model_store_dir_path : str, default $MXNET_HOME/models
        Location for keeping the model parameters.

    Returns
    -------
    file_path
        Path to the requested pretrained model file.
    """
    error, sha1_hash, repo_release_tag = get_model_name_suffix_data(model_name)
    short_sha1 = sha1_hash[:8]
    file_name = "{name}-{error}-{short_sha1}.params".format(
        name=model_name,
        error=error,
        short_sha1=short_sha1)
    local_model_store_dir_path = os.path.expanduser(local_model_store_dir_path)
    file_path = os.path.join(local_model_store_dir_path, file_name)
    if os.path.exists(file_path):
        if check_sha1(file_path, sha1_hash):
            return file_path
        else:
            logging.warning("Mismatch in the content of model file detected. Downloading again.")
    else:
        logging.info("Model file not found. Downloading to {}.".format(file_path))

    if not os.path.exists(local_model_store_dir_path):
        os.makedirs(local_model_store_dir_path)

    zip_file_path = file_path + ".zip"
    download(
        url="{repo_url}/releases/download/{repo_release_tag}/{file_name}.zip".format(
            repo_url=imgclsmob_repo_url,
            repo_release_tag=repo_release_tag,
            file_name=file_name),
        path=zip_file_path,
        overwrite=True)
    with zipfile.ZipFile(zip_file_path) as zf:
        zf.extractall(local_model_store_dir_path)
    os.remove(zip_file_path)

    if check_sha1(file_path, sha1_hash):
        return file_path
    else:
        raise ValueError("Downloaded file has different hash. Please try again.")