grape.embiggen

"""Module with models for graph and text embedding and their Keras Sequences."""
from .embedders import (
    CBOW, GloVe, GraphCBOW, GraphGloVe, GraphSkipGram,
    SkipGram, TransE, TransH, TransR, SimplE, Siamese
)
from .node_prediction import GraphConvolutionalNeuralNetwork
from .sequences import Word2VecSequence
from .transformers import (CorpusTransformer, EdgeTransformer,
                           GraphTransformer, LinkPredictionTransformer,
                           NodeTransformer)
from .visualizations import GraphVisualization

__all__ = [
    "CBOW",
    "SkipGram",
    "GloVe",
    "GraphCBOW",
    "GraphSkipGram",
    "GraphGloVe",
    "Word2VecSequence",
    "NodeTransformer",
    "EdgeTransformer",
    "GraphTransformer",
    "CorpusTransformer",
    "LinkPredictionTransformer",
    "GraphVisualization",
    "TransE",
    "TransH",
    "TransR",
    "SimplE",
    "Siamese",
    "GraphConvolutionalNeuralNetwork"
]

class CBOW(Embedder):
    """CBOW model for sequence embedding.

    The CBOW model for graoh embedding receives a list of contexts and tries
    to predict the central word. The model makes use of an NCE loss layer
    during the training process to generate the negatives.
    """

    def __init__(
        self,
        window_size: int = 4,
        negative_samples: int = 10,
        use_gradient_centralization: bool = True,
        **kwargs: Dict
    ):
        """Create new sequence Embedder model.

        Parameters
        -------------------------------------------
        window_size: int = 4,
            Window size for the local context.
            On the borders the window size is trimmed.
        negative_samples: int = 10,
            The number of negative classes to randomly sample per batch.
            This single sample of negative classes is evaluated for each element in the batch.
        use_gradient_centralization: bool = True,
            Whether to wrap the provided optimizer into a normalized
            one that centralizes the gradient.
            It is automatically enabled if the current version of
            TensorFlow supports gradient transformers.
            More detail here: https://arxiv.org/pdf/2004.01461.pdf
        **kwargs: Dict,
            Additional kwargs to pass to parent constructor.
        """
        # TODO! Figure out a way to test for Zifian distribution in the
        # data used for the word2vec sampling! The values in the vocabulary
        # should have a decreasing node degree order!
        self._window_size = validate_window_size(window_size)
        self._negative_samples = negative_samples
        super().__init__(
            use_gradient_centralization=use_gradient_centralization,
            **kwargs
        )

    def _build_model(self) -> Model:
        """Return CBOW model."""
        # Creating the inputs layers

        # Create first the input with the central terms
        central_terms_input = Input(
            (1, ),
            dtype=tf.int32,
            name="CentralTermsInput"
        )

        # Then we create the input of the contextual terms
        contextual_terms_input = Input(
            (self._window_size*2, ),
            dtype=tf.int32,
            name="ContextualTermsInput"
        )

        # Creating the embedding layer for the contexts
        contextual_terms_embedding = Embedding(
            input_dim=self._vocabulary_size,
            output_dim=self._embedding_size,
            input_length=self._window_size*2,
            name=Embedder.TERMS_EMBEDDING_LAYER_NAME,
        )(contextual_terms_input)

        contextual_embedding = GlobalAveragePooling1D()(
            contextual_terms_embedding
        )

        # Adding layer that also executes the loss function
        sampled_softmax = SampledSoftmax(
            vocabulary_size=self._vocabulary_size,
            embedding_size=self._embedding_size,
            negative_samples=self._negative_samples,
        )((contextual_embedding, central_terms_input))

        # Creating the actual model
        model = Model(
            inputs=[contextual_terms_input, central_terms_input],
            outputs=sampled_softmax,
            name="CBOW"
        )
        return model

    def _compile_model(self) -> Model:
        """Compile model."""
        # No loss function is needed because it is already executed in
        # the Sampled Softmax loss layer.
        self._model.compile(
            optimizer=self._optimizer
        )

class SkipGram(Embedder):
    """SkipGram model for sequence embedding.

    The SkipGram model for graoh embedding receives a central word and tries
    to predict its contexts. The model makes use of an NCE loss layer
    during the training process to generate the negatives.
    """

    def __init__(
        self,
        window_size: int = 4,
        negative_samples: int = 10,
        use_gradient_centralization: bool = True,
        **kwargs: Dict
    ):
        """Create new sequence Embedder model.

        Parameters
        -------------------------------------------
        window_size: int = 4,
            Window size for the local context.
            On the borders the window size is trimmed.
        negative_samples: int = 10,
            The number of negative classes to randomly sample per batch.
            This single sample of negative classes is evaluated for each element in the batch.
        use_gradient_centralization: bool = True,
            Whether to wrap the provided optimizer into a normalized
            one that centralizes the gradient.
            It is automatically enabled if the current version of
            TensorFlow supports gradient transformers.
            More detail here: https://arxiv.org/pdf/2004.01461.pdf
        **kwargs: Dict,
            Additional kwargs to pass to parent constructor.
        """
        # TODO! Figure out a way to test for Zifian distribution in the
        # data used for the word2vec sampling! The values in the vocabulary
        # should have a decreasing node degree order!
        self._window_size = validate_window_size(window_size)
        self._negative_samples = negative_samples
        super().__init__(**kwargs)

    def _build_model(self) -> Model:
        """Return SkipGram model."""
        # Creating the inputs layers

        # Create first the input with the central terms
        central_terms_input = Input(
            (1, ),
            dtype=tf.int32,
            name="CentralTermsInput"
        )

        # Then we create the input of the contextual terms
        contextual_terms_input = Input(
            (self._window_size*2, ),
            dtype=tf.int32,
            name="ContextualTermsInput"
        )

        # Creating the embedding layer for the contexts
        central_terms_embedding = Embedding(
            input_dim=self._vocabulary_size,
            output_dim=self._embedding_size,
            input_length=1,
            name=Embedder.TERMS_EMBEDDING_LAYER_NAME,
        )(central_terms_input)

        central_embedding = Flatten()(
            central_terms_embedding
        )

        # Adding layer that also executes the loss function
        output = NoiseContrastiveEstimation(
            vocabulary_size=self._vocabulary_size,
            embedding_size=self._embedding_size,
            negative_samples=self._negative_samples,
            positive_samples=self._window_size*2
        )((central_embedding, contextual_terms_input))

        # Creating the actual model
        model = Model(
            inputs=[contextual_terms_input, central_terms_input],
            outputs=output,
            name="SkipGram"
        )
        return model

    def _compile_model(self) -> Model:
        """Compile model."""
        # No loss function is needed because it is already executed in
        # the Noise Contrastive Estimation loss layer.
        self._model.compile(
            optimizer=self._optimizer
        )

class GloVe(Embedder):
    """GloVe model for graph and words embedding.

    The GloVe model for graph embedding receives two words and is asked to
    predict its cooccurrence probability.
    """

    def __init__(
        self,
        vocabulary_size: int,
        embedding_size: int,
        embedding: Union[np.ndarray, pd.DataFrame] = None,
        extra_features: Union[np.ndarray, pd.DataFrame] = None,
        optimizer: Union[str, Optimizer] = None,
        alpha: float = 0.75,
        random_state: int = 42,
        directed: bool = False,
        use_gradient_centralization: bool = True,
    ):
        """Create new GloVe-based Embedder object.

        Parameters
        -------------------------------
        vocabulary_size: int,
            Number of terms to embed.
            In a graph this is the number of nodes, while in a text is the
            number of the unique words.
        embedding_size: int,
            Dimension of the embedding.
        embedding: Union[np.ndarray, pd.DataFrame] = None,
            The seed embedding to be used.
            Note that it is not possible to provide at once both
            the embedding and either the vocabulary size or the embedding size.
        extra_features: Union[np.ndarray, pd.DataFrame] = None,
            Optional extra features to be used during the computation
            of the embedding. The features must be available for all the
            elements considered for the embedding.
        optimizer: Union[str, Optimizer] = "nadam",
            The optimizer to be used during the training of the model.
            By default, if None is provided, Nadam with learning rate
            set at 0.01 is used.
        alpha: float = 0.75,
            Alpha to use for the function.
        random_state: int = 42,
            The random state to reproduce the training sequence.
        directed: bool = False,
            Whether to treat the data as directed or not.
        use_gradient_centralization: bool = True,
            Whether to wrap the provided optimizer into a normalized
            one that centralizes the gradient.
            It is automatically enabled if the current version of
            TensorFlow supports gradient transformers.
            More detail here: https://arxiv.org/pdf/2004.01461.pdf
        """
        self._alpha = alpha
        self._random_state = random_state
        self._directed = directed
        super().__init__(
            vocabulary_size=vocabulary_size,
            embedding_size=embedding_size,
            embedding=embedding,
            extra_features=extra_features,
            optimizer=optimizer,
            use_gradient_centralization=use_gradient_centralization
        )

    def _glove_loss(self, y_true: tf.Tensor, y_pred: tf.Tensor) -> float:
        """Compute the glove loss function.

        Parameters
        ---------------------------
        y_true: tf.Tensor,
            The true values Tensor for this batch.
        y_pred: tf.Tensor,
            The predicted values Tensor for this batch.

        Returns
        ---------------------------
        Loss function score related to this batch.
        """
        return K.sum(
            K.pow(K.clip(y_true, 0.0, 1.0), self._alpha) *
            K.square(y_pred - K.log(y_true)),
            axis=-1
        )

    def _build_model(self):
        """Create new Glove model."""
        # Creating the input layers
        left_input_layer = Input((1,), name="left_input_layer")
        right_input_layer = Input((1,), name="right_input_layer")

        trainable_left_embedding = Embedding(
            self._vocabulary_size,
            self._embedding_size,
            input_length=1,
            weights=None if self._embedding is None else [
                self._embedding
            ],
            name=Embedder.TERMS_EMBEDDING_LAYER_NAME
        )(left_input_layer)

        trainable_right_embedding = Embedding(
            self._vocabulary_size,
            self._embedding_size,
            input_length=1,
        )(right_input_layer)

        if self._extra_features is not None:
            extra_features_matrix = Embedding(
                *self._extra_features,
                input_length=1,
                weights=self._extra_features,
                trainable=False,
                name="extra_features_matrix"
            )
            trainable_left_embedding = Concatenate()([
                extra_features_matrix(left_input_layer),
                trainable_left_embedding
            ])
            trainable_right_embedding = Concatenate()([
                extra_features_matrix(right_input_layer),
                trainable_right_embedding
            ])

        # Creating the dot product of the embedding layers
        dot_product_layer = Dot(axes=2)([
            trainable_left_embedding,
            trainable_right_embedding
        ])

        # Creating the biases layer
        biases = [
            Embedding(self._vocabulary_size, 1, input_length=1)(input_layer)
            for input_layer in (left_input_layer, right_input_layer)
        ]

        # Concatenating with an add the three layers
        prediction = Flatten()(Add()([dot_product_layer, *biases]))

        # Creating the model
        glove = Model(
            inputs=[
                left_input_layer,
                right_input_layer
            ],
            outputs=prediction,
            name="GloVe"
        )

        return glove

    def _compile_model(self) -> Model:
        """Compile model."""
        self._model.compile(
            loss=self._glove_loss,
            optimizer=self._optimizer
        )

    def fit(
        self,
        X: Tuple[np.ndarray, np.ndarray],
        frequencies: np.ndarray,
        *args: List,
        epochs: int = 1000,
        batch_size: int = 2**20,
        early_stopping_monitor: str = "loss",
        early_stopping_min_delta: float = 0.001,
        early_stopping_patience: int = 10,
        early_stopping_mode: str = "min",
        reduce_lr_monitor: str = "loss",
        reduce_lr_min_delta: float = 0.01,
        reduce_lr_patience: int = 10,
        reduce_lr_mode: str = "min",
        reduce_lr_factor: float = 0.9,
        verbose: int = 1,
        **kwargs: Dict
    ) -> pd.DataFrame:
        """Return pandas dataframe with training history.

        Parameters
        -----------------------
        X: Tuple[np.ndarray, np.ndarray],
            Tuple with source and destinations.
        frequencies: np.ndarray,
            The frequencies to predict.
        *args: List,
            Other arguments to provide to the model.
        epochs: int = 1000,
            Epochs to train the model for.
        batch_size: int = 2**20,
            The batch size.
            Tipically batch sizes for the GloVe model can be immense.
        early_stopping_monitor: str = "loss",
            Metric to monitor for early stopping.
        early_stopping_min_delta: float = 0.001,
            Minimum delta of metric to stop the training.
        early_stopping_patience: int = 10,
            Number of epochs to wait for when the given minimum delta is not
            achieved after which trigger early stopping.
        early_stopping_mode: str = "min",
            Direction of the variation of the monitored metric for early stopping.
        reduce_lr_monitor: str = "loss",
            Metric to monitor for reducing learning rate.
        reduce_lr_min_delta: float = 0.01,
            Minimum delta of metric to reduce learning rate.
        reduce_lr_patience: int = 10,
            Number of epochs to wait for when the given minimum delta is not
            achieved after which reducing learning rate.
        reduce_lr_mode: str = "min",
            Direction of the variation of the monitored metric for learning rate.
        reduce_lr_factor: float = 0.9,
            Factor for reduction of learning rate.
        verbose: int = 1,
            Wethever to show the loading bar.
            Specifically, the options are:
            * 0 or False: No loading bar.
            * 1 or True: Showing only the loading bar for the epochs.
            * 2: Showing loading bar for both epochs and batches.
        **kwargs: Dict,
            Additional kwargs to pass to the Keras fit call.

        Raises
        -----------------------
        ValueError,
            If given verbose value is not within the available set (-1, 0, 1).

        Returns
        -----------------------
        Dataframe with training history.
        """
        return super().fit(
            GloveSequence(
                *X, frequencies,
                batch_size=batch_size,
                directed=self._directed,
                random_state=self._random_state
            ),
            *args,
            epochs=epochs,
            early_stopping_monitor=early_stopping_monitor,
            early_stopping_min_delta=early_stopping_min_delta,
            early_stopping_patience=early_stopping_patience,
            early_stopping_mode=early_stopping_mode,
            reduce_lr_monitor=reduce_lr_monitor,
            reduce_lr_min_delta=reduce_lr_min_delta,
            reduce_lr_patience=reduce_lr_patience,
            reduce_lr_mode=reduce_lr_mode,
            reduce_lr_factor=reduce_lr_factor,
            verbose=verbose,
            **kwargs
        )

class GraphCBOW(Node2Vec):
    """GraphCBOW model for graph embedding.

    The GraphCBOW model for graoh embedding receives a list of contexts and tries
    to predict the central word. The model makes use of an NCE loss layer
    during the training process to generate the negatives.
    """

    def __init__(
        self,
        graph: Graph,
        embedding_size: int = 100,
        embedding: Union[np.ndarray, pd.DataFrame] = None,
        extra_features: Union[np.ndarray, pd.DataFrame] = None,
        optimizer: Union[str, Optimizer] = None,
        negative_samples: int = 10,
        walk_length: int = 128,
        batch_size: int = 256,
        iterations: int = 16,
        window_size: int = 4,
        return_weight: float = 1.0,
        explore_weight: float = 1.0,
        change_node_type_weight: float = 1.0,
        change_edge_type_weight: float = 1.0,
        max_neighbours: int = None,
        elapsed_epochs: int = 0,
        support_mirrored_strategy: bool = False,
        random_state: int = 42,
        dense_node_mapping: Dict[int, int] = None,
        use_gradient_centralization: bool = True,
    ):
        """Create new sequence Embedder model.

        Parameters
        -------------------------------------------
        graph: Graph,
            Graph to be embedded.
        word2vec_model: Word2Vec,
            Word2Vec model to use.
        embedding_size: int = 100,
            Dimension of the embedding.
        embedding: Union[np.ndarray, pd.DataFrame] = None,
            The seed embedding to be used.
            Note that it is not possible to provide at once both
            the embedding and either the vocabulary size or the embedding size.
        extra_features: Union[np.ndarray, pd.DataFrame] = None,
            Optional extra features to be used during the computation
            of the embedding. The features must be available for all the
            elements considered for the embedding.
        optimizer: Union[str, Optimizer] = None,
            The optimizer to be used during the training of the model.
            By default, if None is provided, Nadam with learning rate
            set at 0.01 is used.
        window_size: int = 4,
            Window size for the local context.
            On the borders the window size is trimmed.
        negative_samples: int = 10,
            The number of negative classes to randomly sample per batch.
            This single sample of negative classes is evaluated for each element in the batch.
        walk_length: int = 128,
            Maximal length of the walks.
        batch_size: int = 256,
            Number of nodes to include in a single batch.
        iterations: int = 16,
            Number of iterations of the single walks.
        window_size: int = 4,
            Window size for the local context.
            On the borders the window size is trimmed.
        return_weight: float = 1.0,
            Weight on the probability of returning to the same node the walk just came from
            Having this higher tends the walks to be
            more like a Breadth-First Search.
            Having this very high  (> 2) makes search very local.
            Equal to the inverse of p in the Node2Vec paper.
        explore_weight: float = 1.0,
            Weight on the probability of visiting a neighbor node
            to the one we're coming from in the random walk
            Having this higher tends the walks to be
            more like a Depth-First Search.
            Having this very high makes search more outward.
            Having this very low makes search very local.
            Equal to the inverse of q in the Node2Vec paper.
        change_node_type_weight: float = 1.0,
            Weight on the probability of visiting a neighbor node of a
            different type than the previous node. This only applies to
            colored graphs, otherwise it has no impact.
        change_edge_type_weight: float = 1.0,
            Weight on the probability of visiting a neighbor edge of a
            different type than the previous edge. This only applies to
            multigraphs, otherwise it has no impact.
        max_neighbours: int = None,
            Number of maximum neighbours to consider when using approximated walks.
            By default, None, we execute exact random walks.
            This is mainly useful for graphs containing nodes with extremely high degrees.
        elapsed_epochs: int = 0,
            Number of elapsed epochs to init state of generator.
        support_mirrored_strategy: bool = False,
            Wethever to patch support for mirror strategy.
            At the time of writing, TensorFlow's MirrorStrategy does not support
            input values different from floats, therefore to support it we need
            to convert the unsigned int 32 values that represent the indices of
            the embedding layers we receive from Ensmallen to floats.
            This will generally slow down performance, but in the context of
            exploiting multiple GPUs it may be unnoticeable.
        random_state: int = 42,
            The random state to reproduce the training sequence.
        dense_node_mapping: Dict[int, int] = None,
            Mapping to use for converting sparse walk space into a dense space.
            This object can be created using the method (available from the
            graph object created using Graph)
            called `get_dense_node_mapping` that returns a mapping from
            the non trap nodes (those from where a walk could start) and
            maps these nodes into a dense range of values.
        use_gradient_centralization: bool = True,
            Whether to wrap the provided optimizer into a normalized
            one that centralizes the gradient.
            It is automatically enabled if the current version of
            TensorFlow supports gradient transformers.
            More detail here: https://arxiv.org/pdf/2004.01461.pdf
        """
        if not graph.has_nodes_sorted_by_decreasing_outbound_node_degree():
            raise ValueError(
                "The given graph does not have the nodes sorted by decreasing "
                "order, therefore the NCE loss sampling (which follows a zipfian "
                "distribution) would not approximate well the Softmax.\n"
                "In order to sort the given graph in such a way that the node IDs "
                "are sorted by decreasing outbound node degrees, you can use "
                "the Graph method "
                "`graph.sort_by_decreasing_outbound_node_degree()`"
            )
        super().__init__(
            graph=graph,
            word2vec_model=CBOW,
            embedding_size=embedding_size,
            embedding=embedding,
            extra_features=extra_features,
            optimizer=optimizer,
            negative_samples=negative_samples,
            walk_length=walk_length,
            batch_size=batch_size,
            iterations=iterations,
            window_size=window_size,
            return_weight=return_weight,
            explore_weight=explore_weight,
            change_node_type_weight=change_node_type_weight,
            change_edge_type_weight=change_edge_type_weight,
            max_neighbours=max_neighbours,
            elapsed_epochs=elapsed_epochs,
            support_mirrored_strategy=support_mirrored_strategy,
            random_state=random_state,
            dense_node_mapping=dense_node_mapping,
            use_gradient_centralization=use_gradient_centralization
        )

class GraphSkipGram(Node2Vec):
    """GraphSkipGram model for graph embedding.

    The SkipGram model for graoh embedding receives a central word and tries
    to predict its contexts. The model makes use of an NCE loss layer
    during the training process to generate the negatives.
    """

    def __init__(
        self,
        graph: Graph,
        embedding_size: int = 100,
        embedding: Union[np.ndarray, pd.DataFrame] = None,
        extra_features: Union[np.ndarray, pd.DataFrame] = None,
        optimizer: Union[str, Optimizer] = None,
        negative_samples: int = 10,
        walk_length: int = 128,
        batch_size: int = 256,
        iterations: int = 16,
        window_size: int = 3,
        return_weight: float = 1.0,
        explore_weight: float = 1.0,
        change_node_type_weight: float = 1.0,
        change_edge_type_weight: float = 1.0,
        max_neighbours: int = None,
        elapsed_epochs: int = 0,
        support_mirrored_strategy: bool = False,
        random_state: int = 42,
        dense_node_mapping: Dict[int, int] = None,
        use_gradient_centralization: bool = True,
    ):
        """Create new sequence Embedder model.

        Parameters
        -------------------------------------------
        graph: Graph,
            Graph to be embedded.
        word2vec_model: Word2Vec,
            Word2Vec model to use.
        embedding_size: int = 100,
            Dimension of the embedding.
        embedding: Union[np.ndarray, pd.DataFrame] = None,
            The seed embedding to be used.
            Note that it is not possible to provide at once both
            the embedding and either the vocabulary size or the embedding size.
        extra_features: Union[np.ndarray, pd.DataFrame] = None,
            Optional extra features to be used during the computation
            of the embedding. The features must be available for all the
            elements considered for the embedding.
        optimizer: Union[str, Optimizer] = None,
            The optimizer to be used during the training of the model.
            By default, if None is provided, Nadam with learning rate
            set at 0.01 is used.
        window_size: int = 4,
            Window size for the local context.
            On the borders the window size is trimmed.
        negative_samples: int = 10,
            The number of negative classes to randomly sample per batch.
            This single sample of negative classes is evaluated for each element in the batch.
        walk_length: int = 128,
            Maximal length of the walks.
        batch_size: int = 256,
            Number of nodes to include in a single batch.
        iterations: int = 16,
            Number of iterations of the single walks.
        window_size: int = 4,
            Window size for the local context.
            On the borders the window size is trimmed.
        return_weight: float = 1.0,
            Weight on the probability of returning to the same node the walk just came from
            Having this higher tends the walks to be
            more like a Breadth-First Search.
            Having this very high  (> 2) makes search very local.
            Equal to the inverse of p in the Node2Vec paper.
        explore_weight: float = 1.0,
            Weight on the probability of visiting a neighbor node
            to the one we're coming from in the random walk
            Having this higher tends the walks to be
            more like a Depth-First Search.
            Having this very high makes search more outward.
            Having this very low makes search very local.
            Equal to the inverse of q in the Node2Vec paper.
        change_node_type_weight: float = 1.0,
            Weight on the probability of visiting a neighbor node of a
            different type than the previous node. This only applies to
            colored graphs, otherwise it has no impact.
        change_edge_type_weight: float = 1.0,
            Weight on the probability of visiting a neighbor edge of a
            different type than the previous edge. This only applies to
            multigraphs, otherwise it has no impact.
        max_neighbours: int = None,
            Number of maximum neighbours to consider when using approximated walks.
            By default, None, we execute exact random walks.
            This is mainly useful for graphs containing nodes with extremely high degrees.
        elapsed_epochs: int = 0,
            Number of elapsed epochs to init state of generator.
        support_mirrored_strategy: bool = False,
            Wethever to patch support for mirror strategy.
            At the time of writing, TensorFlow's MirrorStrategy does not support
            input values different from floats, therefore to support it we need
            to convert the unsigned int 32 values that represent the indices of
            the embedding layers we receive from Ensmallen to floats.
            This will generally slow down performance, but in the context of
            exploiting multiple GPUs it may be unnoticeable.
        random_state: int = 42,
            The random state to reproduce the training sequence.
        dense_node_mapping: Dict[int, int] = None,
            Mapping to use for converting sparse walk space into a dense space.
            This object can be created using the method (available from the
            graph object created using Graph)
            called `get_dense_node_mapping` that returns a mapping from
            the non trap nodes (those from where a walk could start) and
            maps these nodes into a dense range of values.
        use_gradient_centralization: bool = True,
            Whether to wrap the provided optimizer into a normalized
            one that centralizes the gradient.
            It is automatically enabled if the current version of
            TensorFlow supports gradient transformers.
            More detail here: https://arxiv.org/pdf/2004.01461.pdf
        """
        if not graph.has_nodes_sorted_by_decreasing_outbound_node_degree():
            raise ValueError(
                "The given graph does not have the nodes sorted by decreasing "
                "order, therefore the NCE loss sampling (which follows a zipfian "
                "distribution) would not approximate well the Softmax.\n"
                "In order to sort the given graph in such a way that the node IDs "
                "are sorted by decreasing outbound node degrees, you can use "
                "the Graph method "
                "`graph.sort_by_decreasing_outbound_node_degree()`"
            )

        super().__init__(
            graph=graph,
            word2vec_model=SkipGram,
            embedding_size=embedding_size,
            embedding=embedding,
            extra_features=extra_features,
            optimizer=optimizer,
            negative_samples=negative_samples,
            walk_length=walk_length,
            batch_size=batch_size,
            iterations=iterations,
            window_size=window_size,
            return_weight=return_weight,
            explore_weight=explore_weight,
            change_node_type_weight=change_node_type_weight,
            change_edge_type_weight=change_edge_type_weight,
            max_neighbours=max_neighbours,
            elapsed_epochs=elapsed_epochs,
            support_mirrored_strategy=support_mirrored_strategy,
            random_state=random_state,
            dense_node_mapping=dense_node_mapping,
            use_gradient_centralization=use_gradient_centralization
        )

class GraphGloVe(GloVe):
    """GloVe model for graph and words embedding.

    The GloVe model for graoh embedding receives two words and is asked to
    predict its cooccurrence probability.
    """

    def __init__(
        self,
        graph: Graph,
        embedding_size: int = 100,
        embedding: Union[np.ndarray, pd.DataFrame] = None,
        extra_features: Union[np.ndarray, pd.DataFrame] = None,
        optimizer: Union[str, Optimizer] = None,
        alpha: float = 0.75,
        directed: bool = False,
        walk_length: int = 128,
        iterations: int = 16,
        window_size: int = 4,
        return_weight: float = 1.0,
        explore_weight: float = 1.0,
        change_node_type_weight: float = 1.0,
        change_edge_type_weight: float = 1.0,
        max_neighbours: int = None,
        support_mirrored_strategy: bool = False,
        random_state: int = 42,
        dense_node_mapping: Dict[int, int] = None,
        use_gradient_centralization: bool = True,
    ):
        """Create new GloVe-based Embedder object.

        Parameters
        ----------------------------
        vocabulary_size: int,
            Number of terms to embed.
            In a graph this is the number of nodes, while in a text is the
            number of the unique words.
        embedding_size: int,
            Dimension of the embedding.
        embedding: Union[np.ndarray, pd.DataFrame] = None,
            The seed embedding to be used.
            Note that it is not possible to provide at once both
            the embedding and either the vocabulary size or the embedding size.
        extra_features: Union[np.ndarray, pd.DataFrame] = None,
            Optional extra features to be used during the computation
            of the embedding. The features must be available for all the
            elements considered for the embedding.
        optimizer: Union[str, Optimizer] = "nadam",
            The optimizer to be used during the training of the model.
            By default, if None is provided, Nadam with learning rate
            set at 0.01 is used.
        alpha: float = 0.75,
            Alpha to use for the function.
        directed: bool = False,
            Whether to treat the data as directed or not.
        walk_length: int = 128,
            Maximal length of the walks.
        iterations: int = 16,
            Number of iterations of the single walks.
        window_size: int = 4,
            Window size for the local context.
            On the borders the window size is trimmed.
        return_weight: float = 1.0,
            Weight on the probability of returning to the same node the walk just came from
            Having this higher tends the walks to be
            more like a Breadth-First Search.
            Having this very high  (> 2) makes search very local.
            Equal to the inverse of p in the Node2Vec paper.
        explore_weight: float = 1.0,
            Weight on the probability of visiting a neighbor node
            to the one we're coming from in the random walk
            Having this higher tends the walks to be
            more like a Depth-First Search.
            Having this very high makes search more outward.
            Having this very low makes search very local.
            Equal to the inverse of q in the Node2Vec paper.
        change_node_type_weight: float = 1.0,
            Weight on the probability of visiting a neighbor node of a
            different type than the previous node. This only applies to
            colored graphs, otherwise it has no impact.
        change_edge_type_weight: float = 1.0,
            Weight on the probability of visiting a neighbor edge of a
            different type than the previous edge. This only applies to
            multigraphs, otherwise it has no impact.
        max_neighbours: int = None,
            Number of maximum neighbours to consider when using approximated walks.
            By default, None, we execute exact random walks.
            This is mainly useful for graphs containing nodes with extremely high degrees.
        elapsed_epochs: int = 0,
            Number of elapsed epochs to init state of generator.
        support_mirrored_strategy: bool = False,
            Wethever to patch support for mirror strategy.
            At the time of writing, TensorFlow's MirrorStrategy does not support
            input values different from floats, therefore to support it we need
            to convert the unsigned int 32 values that represent the indices of
            the embedding layers we receive from Ensmallen to floats.
            This will generally slow down performance, but in the context of
            exploiting multiple GPUs it may be unnoticeable.
        random_state: int = 42,
            The random state to reproduce the training sequence.
        dense_node_mapping: Dict[int, int] = None,
            Mapping to use for converting sparse walk space into a dense space.
            This object can be created using the method (available from the
            graph object created using Graph)
            called `get_dense_node_mapping` that returns a mapping from
            the non trap nodes (those from where a walk could start) and
            maps these nodes into a dense range of values.
        use_gradient_centralization: bool = True,
            Whether to wrap the provided optimizer into a normalized
            one that centralizes the gradient.
            It is automatically enabled if the current version of
            TensorFlow supports gradient transformers.
            More detail here: https://arxiv.org/pdf/2004.01461.pdf
        """
        self._graph = graph
        self._walk_length = walk_length
        self._iterations = iterations
        self._window_size = validate_window_size(window_size)
        self._return_weight = return_weight
        self._explore_weight = explore_weight
        self._change_node_type_weight = change_node_type_weight
        self._change_edge_type_weight = change_edge_type_weight
        self._max_neighbours = max_neighbours
        self._support_mirrored_strategy = support_mirrored_strategy
        self._random_state = random_state
        self._dense_node_mapping = dense_node_mapping
        super().__init__(
            alpha=alpha,
            random_state=random_state,
            vocabulary_size=self._graph.get_number_of_nodes(),
            embedding_size=embedding_size,
            embedding=embedding,
            extra_features=extra_features,
            optimizer=optimizer,
            use_gradient_centralization=use_gradient_centralization
        )

    def get_embedding_dataframe(self) -> pd.DataFrame:
        """Return terms embedding using given index names."""
        return super().get_embedding_dataframe(self._graph.get_node_names())

    def fit(
        self,
        epochs: int = 1000,
        batch_size: int = 2**20,
        early_stopping_monitor: str = "loss",
        early_stopping_min_delta: float = 0.0001,
        early_stopping_patience: int = 10,
        early_stopping_mode: str = "min",
        reduce_lr_monitor: str = "loss",
        reduce_lr_min_delta: float = 0.0001,
        reduce_lr_patience: int = 5,
        reduce_lr_mode: str = "min",
        reduce_lr_factor: float = 0.9,
        verbose: int = 2,
        **kwargs: Dict
    ) -> pd.DataFrame:
        """Return pandas dataframe with training history.

        Parameters
        -----------------------
        epochs: int = 1000,
            Epochs to train the model for.
        batch_size: int = 2**20,
            The batch size.
            Tipically batch sizes for the GloVe model can be immense.
        early_stopping_monitor: str = "loss",
            Metric to monitor for early stopping.
        early_stopping_min_delta: float = 0.001,
            Minimum delta of metric to stop the training.
        early_stopping_patience: int = 10,
            Number of epochs to wait for when the given minimum delta is not
            achieved after which trigger early stopping.
        early_stopping_mode: str = "min",
            Direction of the variation of the monitored metric for early stopping.
        reduce_lr_monitor: str = "loss",
            Metric to monitor for reducing learning rate.
        reduce_lr_min_delta: float = 0.01,
            Minimum delta of metric to reduce learning rate.
        reduce_lr_patience: int = 10,
            Number of epochs to wait for when the given minimum delta is not
            achieved after which reducing learning rate.
        reduce_lr_mode: str = "min",
            Direction of the variation of the monitored metric for learning rate.
        reduce_lr_factor: float = 0.9,
            Factor for reduction of learning rate.
        verbose: int = 2,
            Wethever to show the loading bar.
            Specifically, the options are:
            * 0 or False: No loading bar.
            * 1 or True: Showing only the loading bar for the epochs.
            * 2: Showing loading bar for both epochs and batches.
        **kwargs: Dict,
            Additional kwargs to pass to the Keras fit call.

        Raises
        -----------------------
        ValueError,
            If given verbose value is not within the available set (-1, 0, 1).

        Returns
        -----------------------
        Dataframe with training history.
        """
        sources, destinations, frequencies = self._graph.cooccurence_matrix(
            walk_length=self._walk_length,
            window_size=self._window_size,
            iterations=self._iterations,
            return_weight=self._return_weight,
            explore_weight=self._explore_weight,
            change_edge_type_weight=self._change_edge_type_weight,
            change_node_type_weight=self._change_node_type_weight,
            dense_node_mapping=self._dense_node_mapping,
            max_neighbours=self._max_neighbours,
            random_state=self._random_state,
            verbose=verbose > 0
        )
        if self._support_mirrored_strategy:
            sources = sources.astype(float)
            destinations = destinations.astype(float)
        return super().fit(
            (sources, destinations), frequencies,
            epochs=epochs,
            batch_size=batch_size,
            early_stopping_monitor=early_stopping_monitor,
            early_stopping_min_delta=early_stopping_min_delta,
            early_stopping_patience=early_stopping_patience,
            early_stopping_mode=early_stopping_mode,
            reduce_lr_monitor=reduce_lr_monitor,
            reduce_lr_min_delta=reduce_lr_min_delta,
            reduce_lr_patience=reduce_lr_patience,
            reduce_lr_mode=reduce_lr_mode,
            reduce_lr_factor=reduce_lr_factor,
            verbose=verbose,
            **kwargs
        )

class Word2VecSequence(AbstractWord2VecSequence):
    """Keras Sequence object for running CBOW and SkipGram on texts."""

    def __getitem__(self, idx: int) -> Tuple[Tuple[np.ndarray, np.ndarray], None]:
        """Return batch corresponding to given index.

        The return tuple of tuples is composed of an inner tuple, containing
        the words vector and the vector of vectors of the contexts.
        Depending on the order of the input_layers of the models that can
        accept these data format, one of the vectors is used as training
        input and the other one is used as the output for the NCE loss layer.

        The words vectors and contexts vectors contain numeric IDs, that
        represent the index of the words' embedding column.

        The true output value is None, since no loss function is used after
        the NCE loss, that is implemented as a layer, and this vastly improves
        the speed of the training process since it does not require to allocate
        empty vectors of considerable size for the one-hot encoding process.

        Parameters
        ---------------
        idx: int,
            Index corresponding to batch to be returned.

        Returns
        ---------------
        Tuple of tuples with input data.
        """
        contexts, words = preprocessing.word2vec(
            self._sequences[idx],
            window_size=self._window_size,
        )

        if self._support_mirrored_strategy:
            return (contexts.astype(float), words.astype(float)), None
        return (contexts, words), None

class NodeTransformer:
    """NodeTransformer class to convert nodes to edge embeddings."""

    def __init__(
        self,
        numeric_node_ids: bool = False,
        aligned_node_mapping: bool = False,
        support_mirrored_strategy: bool = False,
    ):
        """Create new NodeTransformer object.

        Parameters
        -------------------
        numeric_node_ids: bool = False,
            Wether to return the numeric node IDs instead of the node embedding.
        aligned_node_mapping: bool = False,
            This parameter specifies whether the mapping of the embeddings nodes
            matches the internal node mapping of the given graph.
            If these two mappings do not match, the generated edge embedding
            will be meaningless.
        support_mirrored_strategy: bool = False,
            Wethever to patch support for mirror strategy.
            At the time of writing, TensorFlow's MirrorStrategy does not support
            input values different from floats, therefore to support it we need
            to convert the unsigned int 32 values that represent the indices of
            the embedding layers we receive from Ensmallen to floats.
            This will generally slow down performance, but in the context of
            exploiting multiple GPUs it may be unnoticeable.
        """
        self._numeric_node_ids = numeric_node_ids
        self._support_mirrored_strategy = support_mirrored_strategy
        self._embedding = None
        self._embedding_numpy = None
        self._aligned_node_mapping = aligned_node_mapping

    @property
    def numeric_node_ids(self) -> bool:
        """Return whether the transformer returns numeric node IDs."""
        return self._numeric_node_ids

    def fit(self, embedding: pd.DataFrame):
        """Fit the model.

        Parameters
        -------------------------
        embedding: pd.DataFrame,
            Embedding to use to fit the transformer.
            This is a pandas DataFrame and NOT a numpy array because we need
            to be able to remap correctly the vector embeddings in case of
            graphs that do not respect the same internal node mapping but have
            the same node set. It is possible to remap such graphs using
            Ensmallen's remap method but it may be less intuitive to users.
        """
        if not isinstance(embedding, pd.DataFrame):
            raise ValueError("Given embedding is not a pandas DataFrame.")
        self._embedding = embedding
        self._embedding_numpy = embedding.to_numpy()

    def transform(self, nodes: Union[List[str], List[int]]) -> np.ndarray:
        """Return embeddings from given node.

        Parameters
        --------------------------
        nodes: Union[List[str], List[int]],
            List of nodes whose embedding is to be returned.
            By default this should be a list of strings, if the
            aligned_node_mapping is setted, then this methods also accepts
            a list of ints.

        Raises
        --------------------------
        ValueError,
            If embedding is not fitted.

        Returns
        --------------------------
        Numpy array of embeddings.
        """
        if self._embedding is None and not self.numeric_node_ids:
            raise ValueError(
                "Transformer was not fitted yet."
            )

        if self._aligned_node_mapping:
            if self.numeric_node_ids:
                if self._support_mirrored_strategy:
                    return nodes.astype(float)
                return nodes
            return self._embedding_numpy[nodes]

        if self.numeric_node_ids:
            ids = np.where(self._embedding.index.isin(nodes))
            if self._support_mirrored_strategy:
                return ids.astype(float)
            return ids

        return self._embedding.loc[nodes].to_numpy()

class EdgeTransformer:
    """EdgeTransformer class to convert edges to edge embeddings."""

    methods = {
        "Hadamard": lambda x1, x2: np.multiply(x1, x2, out=x1),
        "Sum": lambda x1, x2: np.add(x1, x2, out=x1),
        "Average": lambda x1, x2: np.divide(np.add(x1, x2, out=x1), 2, out=x1),
        "L1": lambda x1, x2: np.subtract(x1, x2, out=x1),
        "AbsoluteL1": lambda x1, x2: np.abs(np.subtract(x1, x2, out=x1), out=x1),
        "L2": lambda x1, x2: np.power(np.subtract(x1, x2, out=x1), 2, out=x1),
        "Concatenate": lambda x1, x2: np.hstack((x1, x2)),
        None: lambda x1, x2: np.vstack((x1, x2)).T,
    }

    def __init__(
        self,
        method: str = "Hadamard",
        aligned_node_mapping: bool = False,
        support_mirrored_strategy: bool = False,
    ):
        """Create new EdgeTransformer object.

        Parameters
        ------------------------
        method: str = "Hadamard",
            Method to use for the embedding.
            If None is used, we return instead the numeric tuples.
            Can either be 'Hadamard', 'Sum', 'Average', 'L1', 'AbsoluteL1', 'L2' or 'Concatenate'.
        aligned_node_mapping: bool = False,
            This parameter specifies whether the mapping of the embeddings nodes
            matches the internal node mapping of the given graph.
            If these two mappings do not match, the generated edge embedding
            will be meaningless.
        support_mirrored_strategy: bool = False,
            Wethever to patch support for mirror strategy.
            At the time of writing, TensorFlow's MirrorStrategy does not support
            input values different from floats, therefore to support it we need
            to convert the unsigned int 32 values that represent the indices of
            the embedding layers we receive from Ensmallen to floats.
            This will generally slow down performance, but in the context of
            exploiting multiple GPUs it may be unnoticeable.
        """
        if isinstance(method, str) and method not in EdgeTransformer.methods:
            raise ValueError((
                "Given method '{}' is not supported. "
                "Supported methods are {}, or alternatively a lambda."
            ).format(
                method, ", ".join(list(EdgeTransformer.methods.keys()))
            ))
        self._transformer = NodeTransformer(
            numeric_node_ids=method is None,
            aligned_node_mapping=aligned_node_mapping,
            support_mirrored_strategy=support_mirrored_strategy
        )
        self._method_name = method
        self._method = EdgeTransformer.methods[self._method_name]

    @property
    def numeric_node_ids(self) -> bool:
        """Return whether the transformer returns numeric node IDs."""
        return self._transformer.numeric_node_ids

    @property
    def method(self) -> str:
        """Return the used edge embedding method."""
        return self._method_name

    def fit(self, embedding: pd.DataFrame):
        """Fit the model.

        Parameters
        -------------------------
        embedding: pd.DataFrame,
            Embedding to use to fit the transformer.
            This is a pandas DataFrame and NOT a numpy array because we need
            to be able to remap correctly the vector embeddings in case of
            graphs that do not respect the same internal node mapping but have
            the same node set. It is possible to remap such graphs using
            Ensmallen's remap method but it may be less intuitive to users.

        Raises
        -------------------------
        ValueError,
            If the given method is None there is no need to call the fit method.
        """
        if self._method is None:
            raise ValueError(
                "There is no need to call the fit when edge method is None."
            )
        self._transformer.fit(embedding)

    def transform(self, sources: List[str], destinations: List[str]) -> np.ndarray:
        """Return embedding for given edges using provided method.

        Parameters
        --------------------------
        sources: List[str],
            List of source nodes whose embedding is to be returned.
        destinations: List[str],
            List of destination nodes whose embedding is to be returned.

        Raises
        --------------------------
        ValueError,
            If embedding is not fitted.

        Returns
        --------------------------
        Numpy array of embeddings.
        """
        return self._method(
            self._transformer.transform(sources),
            self._transformer.transform(destinations)
        )

class GraphTransformer:
    """GraphTransformer class to convert graphs to edge embeddings."""

    def __init__(
        self,
        method: str = "Hadamard",
        aligned_node_mapping: bool = False,
        support_mirrored_strategy: bool = False,
    ):
        """Create new GraphTransformer object.

        Parameters
        ------------------------
        method: str = "hadamard",
            Method to use for the embedding.
            Can either be 'Hadamard', 'Sum', 'Average', 'L1', 'AbsoluteL1', 'L2' or 'Concatenate'.
        aligned_node_mapping: bool = False,
            This parameter specifies whether the mapping of the embeddings nodes
            matches the internal node mapping of the given graph.
            If these two mappings do not match, the generated edge embedding
            will be meaningless.
        support_mirrored_strategy: bool = False,
            Wethever to patch support for mirror strategy.
            At the time of writing, TensorFlow's MirrorStrategy does not support
            input values different from floats, therefore to support it we need
            to convert the unsigned int 32 values that represent the indices of
            the embedding layers we receive from Ensmallen to floats.
            This will generally slow down performance, but in the context of
            exploiting multiple GPUs it may be unnoticeable.
        """
        self._transformer = EdgeTransformer(
            method=method,
            aligned_node_mapping=aligned_node_mapping,
            support_mirrored_strategy=support_mirrored_strategy
        )
        self._aligned_node_mapping = aligned_node_mapping

    @property
    def numeric_node_ids(self) -> bool:
        """Return whether the transformer returns numeric node IDs."""
        return self._transformer.numeric_node_ids

    @property
    def method(self) -> str:
        """Return the used edge embedding method."""
        return self._transformer.method

    def fit(self, embedding: pd.DataFrame):
        """Fit the model.

        Parameters
        -------------------------
        embedding: pd.DataFrame,
            Embedding to use to fit the transformer.
            This is a pandas DataFrame and NOT a numpy array because we need
            to be able to remap correctly the vector embeddings in case of
            graphs that do not respect the same internal node mapping but have
            the same node set. It is possible to remap such graphs using
            Ensmallen's remap method but it may be less intuitive to users.
        """
        self._transformer.fit(embedding)

    def transform(
        self,
        graph: Union[Graph, np.ndarray, List[List[str]], List[List[int]]],
    ) -> np.ndarray:
        """Return edge embedding for given graph using provided method.

        Parameters
        --------------------------
        graph: Union[Graph, np.ndarray, List[List[str]], List[List[int]]],
            The graph whose edges are to embed.
            It can either be an Graph or a list of lists of edges.

        Raises
        --------------------------
        ValueError,
            If embedding is not fitted.

        Returns
        --------------------------
        Numpy array of embeddings.
        """
        if isinstance(graph, Graph):
            if self._aligned_node_mapping:
                graph = graph.get_edge__node_ids(directed=False)
            else:
                graph = graph.get_edge_node_names(directed=False)
        if isinstance(graph, List):
            graph = np.array(graph)
        if isinstance(graph, np.ndarray):
            sources = graph[:, 0]
            destinations = graph[:, 1]
        return self._transformer.transform(sources, destinations)

class CorpusTransformer:
    """Simple class to tekenize textual corpuses."""

    def __init__(
        self,
        synonyms: Dict = None,
        language: str = "english",
        tokenizer_method: str = "nltk",
        apply_stemming: bool = True,
        remove_stop_words: bool = True,
        remove_punctuation: bool = True,
        remove_digits: bool = False,
        extra_stop_words: Set[str] = None,
        min_word_length: int = 2,
        min_sequence_length: int = 0,
        min_count: int = 0,
        max_count: int = math.inf,
        to_lower_case: bool = True,
        verbose: bool = True,
        processes: int = None,
        use_multiprocessing: bool = True
    ):
        """Create new CorpusTransformer object.

        This is a GENERIC text tokenizer and is only useful for basic examples
        as in any advanced settings there will be need for a custom tokenizer.

        Parameters
        ----------------------------
        synonyms: Dict = None,
            The synonyms to use.
        language: str = "english",
            The language for the stopwords.
        tokenizer_method: str = "nltk",
            The tokenizer method to be used.
            Can either be `nltk`, that is, using the nltk default method,
            or alternatively can be `space`, that is splitting only on spaces.
        apply_stemming: bool = True,
            Wethever to apply or not a stemming procedure, which
            by default is enabled.
            The algorithm used is a Porter Stemmer.
        remove_stop_words: bool = True,
            Whether to remove stopwords,
            as defined from NLTK for the given language.
        remove_punctuation: bool = True,
            Whether to remove punctuation, as defined from the string package.
        remove_digits: bool = False,
            Whether to remove words composed of only digits.
        extra_stop_words: Set[str] = None,
            The additional stop words to be removed.
        min_word_length: int = 2,
            Minimum length of the corpus words.
        min_sequence_length: int = 0,
            Minimum length of the tokenized sequences.
            If you are using word2vec, the sequences MUST be longer than
            two times the window size plus one.
        min_count: int = 0,
            Whether to drop terms that appear less than the given amount.
        max_count: int = math.inf,
            Whether to drop terms that appear more than the given amount.
        to_lower_case: bool = True,
            Whether to convert terms to lowercase.
        processes: int = None,
            Number of parallel processes to use.
            If given processes is None, all the available processes is used.
        verbose: bool = True,
            Whether to show loading bars and log process.
        use_multiprocessing: bool = True,
            Whether to use or not multiprocessing.

        Raises
        --------------------------
        ValueError,
            If the given tokenizer method is not supported.
        """
        try:
            import nltk
        except ImportError:
            raise ImportError(
                "The package nltk is not installed!\n"
                "If you need to use the CorpusTransformer object, "
                "please do install the nltk package.\n"
                "This package has to be installed separetely because it "
                "comes with added complexity that we prefer to spare the user "
                "when their main interest lies within graph embedding."
            )
        self._synonyms = {} if synonyms is None else synonyms
        self._stopwords = set() if extra_stop_words is None else extra_stop_words
        if remove_stop_words:
            self._stopwords |= set(stopwords.words(language))
        if remove_punctuation:
            self._stopwords |= set(string.punctuation)
        self._remove_digits = remove_digits
        self._min_word_length = min_word_length
        self._min_count = min_count
        self._max_count = max_count
        self._min_sequence_length = min_sequence_length
        self._to_lower_case = to_lower_case
        self._use_multiprocessing = use_multiprocessing
        self._processes = cpu_count() if processes is None else processes
        self._verbose = verbose
        self._stemmer = PorterStemmer() if apply_stemming else None
        if tokenizer_method not in ("nltk", "space"):
            raise ValueError(
                (
                    "Given tokenizer method `{}` is not supported. "
                    "The supported methods are `nltk` and `space`."
                ).format(tokenizer_method)
            )
        self._tokenizer_method = tokenizer_method
        self._tokenizer = None

    def get_synonym(self, word: str) -> str:
        """Return the synonym of the given word, if available.

        Parameters
        ----------------------------
        word: str,
            The word whose synonym is to be found.

        Returns
        ----------------------------
        The given word synonym.
        """
        return self._synonyms.get(word, word)

    def split_line(self, line: str) -> List[str]:
        """Return preliminary tokenization of the line.

        Parameters
        ---------------------
        line: str,
            The line to be tokenized.

        Returns
        ---------------------
        The list of string tokens.
        """
        if self._to_lower_case:
            line = line.lower()

        if self._tokenizer_method == "nltk":
            return word_tokenize(line)

        return line.split(" ")

    def tokenize_line(self, line: str) -> List[str]:
        """Return tokenized line.

        Parameters
        ---------------------
        line: str,
            The line to be tokenized.

        Returns
        ---------------------
        The list of string tokens.
        """
        return [
            self._stemmer.stem(self.get_synonym(word))
            if self._stemmer is not None
            else self.get_synonym(word)
            for word in word_tokenize(line.lower() if self._to_lower_case else line)
            if word not in self._stopwords and
            len(word) > self._min_word_length and
            (not self._remove_digits or not word.isnumeric())
        ]

    def tokenize_lines(self, lines: List[str]) -> List[List[str]]:
        """Return tokenized lines.

        Parameters
        ---------------------
        lines: List[str],
            List of lines to be tokenized.

        Returns
        ---------------------
        The list of string tokens.
        """
        return [
            self.tokenize_line(line)
            for line in lines
        ]

    def tokenize(self, texts: List[str], return_counts: bool = False):
        """Fit model using stemming from given text.

        Parameters
        ----------------------------
        texts: List[str],
            The text to use to fit the transformer.
        return_counts: bool = False,
            Wethever to return the counts of the terms or not.

        Return
        -----------------------------
        Either the tokens or tuple containing the tokens and the counts.
        """
        processes = min(cpu_count(), len(texts))
        chunks_number = processes*2
        chunk_size = max(len(texts) // chunks_number, 1)
        tasks = (
            texts[i:i + chunk_size]
            for i in range(0, len(texts), chunk_size)
        )
        if self._use_multiprocessing:
            with Pool(processes) as p:
                all_tokens = [
                    line
                    for chunk in tqdm(
                        p.imap(
                            self.tokenize_lines,
                            tasks
                        ),
                        desc="Tokenizing",
                        total=chunks_number,
                        disable=not self._verbose
                    )
                    for line in chunk
                ]
                p.close()
                p.join()
        else:
            all_tokens = [
                line
                for chunk in tqdm(
                    tasks,
                    desc="Tokenizing",
                    total=chunks_number,
                    disable=not self._verbose
                )
                for line in self.tokenize_lines(chunk)
            ]

        if return_counts:
            counter = Counter((
                term
                for terms in tqdm(
                    all_tokens,
                    desc="Computing counts of terms",
                    disable=not self._verbose
                )
                for term in terms
            ))
            return all_tokens, counter
        return all_tokens

    def parse_tokens_for_low_frequency(self, tokens_list: List[List[str]]) -> Generator:
        """Yields tokens parsed according to updated stopwords.

        Parameters
        --------------------
        tokens_list: List[List[str]],
            List of the string tokens.

        Yields
        --------------------
        The filtered out tokens.
        """
        for tokens in tqdm(
            tokens_list,
            total=len(tokens_list),
            desc="Filtering low frequency terms",
            disable=not self._verbose
        ):
            new_tokens = [
                token
                for token in tokens
                if token not in self._stopwords
            ]
            if len(new_tokens) > 0:
                yield new_tokens

    def fit(self, texts: List[str]):
        """Fit the transformer.

        Parameters
        ----------------------------
        texts: List[str],
            The texts to use for the fitting.

        Raises
        ----------------------------
        ValueError,
            If there are NaN values within given texts.
        ValueError,
            If there are non string values within given texts.
        """
        if pd.isna(texts).any():
            raise ValueError(
                "There are NaN values within the given texts."
            )
        if any(not isinstance(text, str) for text in texts):
            raise ValueError(
                "There are not string values within the given texts."
            )
        tokens_list, counts = self.tokenize(texts, True)

        if self._min_count > 0 or math.isfinite(self._max_count):
            self._stopwords |= {
                word
                for word, count in counts.items()
                if count <= self._min_count or count >= self._max_count
            }
            tokens_list = self.parse_tokens_for_low_frequency(tokens_list)

        self._tokenizer = Tokenizer(
            lower=self._to_lower_case
        )
        self._tokenizer.fit_on_texts((
            " ".join(tokens)
            for tokens in tqdm(
                tokens_list,
                desc="Fitting tokenizer",
                total=len(texts),
                disable=not self._verbose
            )
        ))

    @property
    def vocabulary_size(self) -> int:
        """Return number of different terms."""
        return len(self._tokenizer.word_counts)

    def reverse_transform(self, sequences: np.ndarray) -> List[str]:
        """Reverse the sequence to texts.

        Parameters
        ------------------------
        sequences: np.ndarray,
            The sequences to counter transform.

        Returns
        ------------------------
        The texts created from the given sequences.
        """
        if isinstance(sequences, (list, tuple)):
            sequences = np.array(sequences)
        return self._tokenizer.sequences_to_texts(sequences)

    def get_word_id(self, word: str) -> int:
        """Get the given words IDs.

        Parameters
        ------------------------
        word: int
            The word whose ID is to be retrieved.

        Returns
        ------------------------
        The word numeric ID.
        """
        return self._tokenizer.word_index[word]

    def transform(self, texts: List[str]) -> np.ndarray:
        """Transform given text.

        Parameters
        --------------------------
        texts: List[str],
            The texts to encode as digits.

        Raises
        ----------------------------
        ValueError,
            If there are NaN values within given texts.
        ValueError,
            If there are non string values within given texts.

        Returns
        --------------------------
        Numpy array with numpy arrays of tokens.
        """
        if pd.isna(texts).any():
            raise ValueError(
                "There are NaN values within the given texts."
            )
        if any(not isinstance(text, str) for text in texts):
            raise ValueError(
                "There are not string values within the given texts."
            )
        return np.array([
            np.array(tokens, dtype=np.uint64)
            for tokens in self._tokenizer.texts_to_sequences((
                " ".join(tokens)
                for tokens in tqdm(
                    self.tokenize(texts),
                    desc="Transform texts",
                    total=len(texts),
                    disable=not self._verbose
                )
                if len(tokens) >= self._min_sequence_length
            ))
        ], dtype=object)