Source code for ynlu.sdk.evaluation.intent_recall_score_with_threshold

from typing import List, Dict
from sklearn.metrics import recall_score

from .utils import preprocess_intent_prediction_by_threshold
from ynlu.sdk.evaluation import UNKNOWN


def intent_recall_score_with_threshold(
        intent_predictions: List[List[Dict[str, str]]],
        y_trues: List[str],
        threshold: float = 0.5,
        average: str = "weighted",
        sample_weight: List[float] = None,
    ) -> float:
    """Top1 recall classification score subject to score > threshold

    Only the top1 predicted intent by score will be considered
    when computing recall. Moreover, the score of the predicted
    label should be more than threshold. Otherwise, it would be
    replaced with a ``UNK`` token before computing recall score.
    That is to say, the situation ``correctly classified`` occurs
    only when two requirements are satisfied:
        1. the top1 predicted label is the same as true label and
        2. score of predicted label is larger than threshold.


    Args:
        intent_predictions (list of list of dicts):
            A list of intent_prediction which can contains all possible
            intent sorted by score.
        y_trues (list of strings):
            A list of ground truth (correct) intents.
        threshold (float):
            A threshold which limits the efficacy of top1
            predicted intent if its score is less than threshold.
        average (string):
            Options are as follows:
                [‘None’, ‘binary’, ‘micro’, ‘macro’, ‘samples’, ‘weighted’(default)].
            Please look at
             ``http://scikit-learn.org/stable/modules/generated/sklearn.metrics.recall_score.html``
            for more details.
        sample_weight (list of float):
            Sample weights.

    Returns:
        float (if average is not None):
            recall of the positive class in binary classification
            or weighted average of the recall of each class for
            the multiclass task.
        or
        array of float (shape = [n_unique_labels]):
            A list of recall of each class.

    Examples:
        >>> from ynlu.sdk.evaluation import intent_recall_score_with_threshold
        >>> intent_recall_score_with_threshold(
                intent_predictions=[
                    [{"intent": "a", "score": 0.7}],
                    [{"intent": "b", "score": 0.3}],
                    [{"intent": "b", "score": 0.8}],
                ],
                y_trues=["a", "b", "a"],
            )
        >>> 0.3333

    """

    if len(intent_predictions) != len(y_trues):
        raise ValueError(
            "Intent predictions and labels must have same amount!!!",
        )
    y_preds = preprocess_intent_prediction_by_threshold(
        intent_predictions=intent_predictions,
        threshold=threshold,
    )
    labels = sorted(set(y_trues) | set(y_preds) - set([UNKNOWN]))
    return recall_score(
        y_true=y_trues,
        y_pred=y_preds,
        average=average,
        labels=labels,
        sample_weight=sample_weight,
    )