Python fasttext.load_model() Examples

def __init__(self):
        resource_package = __name__

        yelp_acc_path = 'acc_yelp.bin'
        yelp_ppl_path = 'ppl_yelp.binary'
        yelp_ref0_path = 'yelp.refs.0'
        yelp_ref1_path = 'yelp.refs.1'

        
        yelp_acc_file = pkg_resources.resource_stream(resource_package, yelp_acc_path)
        yelp_ppl_file = pkg_resources.resource_stream(resource_package, yelp_ppl_path)
        yelp_ref0_file = pkg_resources.resource_stream(resource_package, yelp_ref0_path)
        yelp_ref1_file = pkg_resources.resource_stream(resource_package, yelp_ref1_path)

        
        self.yelp_ref = []
        with open(yelp_ref0_file.name, 'r') as fin:
            self.yelp_ref.append(fin.readlines())
        with open(yelp_ref1_file.name, 'r') as fin:
            self.yelp_ref.append(fin.readlines())
        self.classifier_yelp = fasttext.load_model(yelp_acc_file.name)
        self.yelp_ppl_model = kenlm.Model(yelp_ppl_file.name) 

def cache(self, name, cache, url=None):
        path = os.path.join(cache, name)
        if not os.path.isfile(path) and url:
            logger.info('Downloading vectors from {}'.format(url))
            if not os.path.exists(cache):
                os.makedirs(cache)
            if not os.path.isfile(self.destination):
                if 'drive.google.com' in url:
                    download_from_url(url, self.destination)
                else:
                    urlretrieve(url, self.destination)
            logger.info('Extracting vectors into {}'.format(cache))
            ext = os.path.splitext(self.destination)[1][1:]
            if ext == 'zip':
                with zipfile.ZipFile(self.destination, "r") as zf:
                    zf.extractall(cache)
            elif ext == 'gz':
                with tarfile.open(self.destination, 'r:gz') as tar:
                    tar.extractall(path=cache)
        if not os.path.isfile(path):
            raise RuntimeError('no vectors found at {}'.format(path))

        self.model = fasttext.load_model(path)
        self.dim = len(self['a']) 

def fasttext(self):
    if self._fasttext is None and self.fasttext_model_file:
      LOG.info("Loading fasttext embeddings from %s", self.fasttext_model_file)
      self._fasttext = fasttext.load_model(self.fasttext_model_file)
    return self._fasttext 

def loadpretrained(self,X):
		'returns the model with pretrained weights'
		self.classifier=ft.load_model(X,label_prefix=self.lpr) 

def load(self, embedding_fname, embedding_url=None, *args, **kwargs):
        """
        Method initializes dict of embeddings from file
        Args:
            fname: file name

        Returns:
            Nothing
        """

        if not embedding_fname:
            raise RuntimeError('Please, provide path to model')
        fasttext_model_file = embedding_fname

        if not Path(fasttext_model_file).is_file():
            emb_path = embedding_url
            if not emb_path:
                raise RuntimeError('Fasttext model file does not exist locally. URL does not contain  fasttext model file')
            embedding_fname = Path(fasttext_model_file).name
            try:
                download(dest_file_path=fasttext_model_file, source_url=embedding_url)
            except Exception as e:
                raise RuntimeError('Looks like the `EMBEDDINGS_URL` variable is set incorrectly', e)

        if self.module == "fastText":
            import fastText
            self.fasttext_model = fastText.load_model(fasttext_model_file)
        if self.module == "fasttext":
            import fasttext
            self.fasttext_model = fasttext.load_model(fasttext_model_file)
        return 

def read_embedding_df_fasttext_format(filepath):
    """Read embedding from fasttext format."""
    model = load_fasttext(filepath)
    return pd.DataFrame({
        word: model.get_word_vector(word)
        for word in model.get_words()
    }).T 

def __init__(self, path_to_model: str) -> None:
        "Input fastText trained sentiment model"
        import fasttext
        self.classifier = fasttext.load_model(path_to_model) 

def load(self) -> None:
        """
        Load fastText binary model from self.load_path
        """
        log.info(f"[loading fastText embeddings from `{self.load_path}`]")
        self.model = fasttext.load_model(str(self.load_path))
        self.dim = self.model.get_dimension() 

def load_model(self, filename=''):

        # Define path to the feature extractor model filename
        if (len(filename) > 0) and os.path.exists(filename):
            self.filename = filename
        if not os.path.exists(self.filename):
            print('Feature file %s does not exist' % self.filename)
            return -1

        print('Loading model %s' % self.filename)
        if self.method == 'fasttext':
            self.model = fasttext.load_model(self.filename)
        elif self.method == 'bow':
            self.model = joblib.load(self.filename) 

def __init__(self, method='fasttext', filename=''):

        self.method = method
        self.filename = filename
        self.model = None

        # If a filename is given, try to load the model
        if os.path.exists(self.filename):
            self.load_model() 

def load(cls, load_dir, batch_size=4, gpu=False):
        import fasttext

        if os.path.isfile(load_dir):
            return cls(model=fasttext.load_model(load_dir))
        else:
            logger.error(f"Fasttext model file does not exist at: {load_dir}") 

def __init__(self, vecs_txt_fname, vecs_bin_fname=None, method="word2vec", dim=100, tokenizer_name="mecab"):
        self.tokenizer = get_tokenizer(tokenizer_name)
        self.tokenizer_name = tokenizer_name
        self.dim = dim
        self.method = method
        self.dictionary, self.words, self.vecs = self.load_vectors(vecs_txt_fname, method)
        if "fasttext" in method:
            self.model = load_ft_model(vecs_bin_fname) 

def load(self, model_path):
        """
        加载训练好的模型
        :param model_path: 训练好的模型路径
        :return:
        """
        if os.path.exists(self.model_path + '.bin'):
            return fasttext.load_model(model_path + '.bin')
        else:
            return None 

def load(self):
        if os.path.exists(self.model_path):
            return fasttext.load_model(self.model_path)
        else:
            return None 

def get_compiled_model(self):
        return load_fasttext_model(self.MODEL_PATH) 

def __init__(self, model_file: str=None) -> None:
        super().__init__()
        # pip install fasttext
        import fasttext
        try:
            self.model = fasttext.load_model(model_file)
        except ValueError:
            raise Exception("Please specify a valid trained FastText model file (.bin or .ftz extension)'{}'."
                            .format(model_file)) 

def __init__(self, path):
        ft = self.ft = fasttext.load_model(path)
        vectors = (ft.get_output_matrix() @ ft.get_input_matrix().T).T
        rows, _cols = vectors.shape
        # Add counts and evals
        vectors = np.hstack([
            np.ones(rows).reshape(rows, 1),
            vectors])
        # maybe its an occ model?
        self.occ = False

        # Start with bias. No bias for eval.
        bias = np.hstack([[0], vectors[0]])

        # Parse remaining words
        piece_to_vec = defaultdict(lambda: 0)
        castling = {}
        for w, v in zip(ft.words[1:], vectors[1:]):
            sq = getattr(chess, w[:2].upper())
            if w.endswith('-Occ'):
                self.occ = True
                for color in chess.COLORS:
                    for piece_type in chess.PIECE_TYPES:
                        piece_to_vec[piece_type, color, sq] += np.hstack([[0], v])
            elif w.endswith('-C'):
                e = pst.castling[sq]
                castling[sq] = np.hstack([[e], v])
            else:
                p = chess.Piece.from_symbol(w[2])
                e = pst.piece[p.piece_type-1] * (1 if p.color else -1)
                e += pst.pst[0 if p.color else 1][p.piece_type-1][sq]
                #print(w[2], p, e)
                piece_to_vec[p.piece_type, p.color, sq] += np.hstack([[e], v])

        # Convert to two-colours
        # We keep a record of the board from both perspectives
        piece_to_vec2 = {}
        for (piece_type, color, sq), v in piece_to_vec.items():
            inv = piece_to_vec[piece_type, not color, chess.square_mirror(sq)]
            piece_to_vec2[piece_type, color, sq] = np.vstack([v, inv])

        self.bias = np.vstack([bias, bias])
        self.piece_to_vec = piece_to_vec2
        self.castling = {sq: np.vstack([v, castling[chess.square_mirror(sq)]])
                         for sq, v in castling.items()}

        # Parse labels
        self.moves = [chess.Move.from_uci(label_uci[len('__label__'):])
                      for label_uci in ft.labels]

        # Adding 2 to the move ids, since the first entry will be the count,
        # and the second entry will be the evaluation
        self.move_to_id = {move: i + 2 for i, move in enumerate(self.moves)}