Auto-CS 2.0

https://sourceforge.net/projects/autocs/

Module CuedSpeech.wherecue

Class TargetProbabilitiesEstimator

Description

Generate the probabilities of the targets (pos&shape).

For each image of the video (each interval in vowels_coords tier), an interval with the probability of the shape, and another one with the probability of the position are estimated.

Constructor

Create a new instance.

Parameters
  • cue_rules: (CuedSpeechKeys) Rules and codes for vowel positions and hand shapes
View Source
def __init__(self, cue_rules: CuedSpeechCueingRules=CuedSpeechCueingRules()):
    """Create a new instance.

    :param cue_rules: (CuedSpeechKeys) Rules and codes for vowel positions and hand shapes

    """
    self.__cued = None
    self._vrank = ()
    self.set_cue_rules(cue_rules)

Public functions

get_vowel_rank

Return an index from the code of a vowel or -1.

Parameters
  • vowel_code: (char) One of n, b, c, s, m, t for French.
View Source
def get_vowel_rank(self, vowel_code):
    """Return an index from the code of a vowel or -1.

        :param vowel_code: (char) One of n, b, c, s, m, t for French.

        """
    if vowel_code in self._vrank:
        return self._vrank.index(vowel_code)
    return -1

set_cue_rules

Set new rules.

Parameters
  • cue_rules: (CuedSpeechKeys) Rules and codes for vowel positions and hand shapes
Raises
  • sppasTypeError: given parameter is not CuedSpeechCueingRules
View Source
def set_cue_rules(self, cue_rules: CuedSpeechCueingRules) -> None:
    """Set new rules.

        :param cue_rules: (CuedSpeechKeys) Rules and codes for vowel positions and hand shapes
        :raises: sppasTypeError: given parameter is not CuedSpeechCueingRules

        """
    if isinstance(cue_rules, CuedSpeechCueingRules) is False:
        raise sppasTypeError('cue_rules', 'CuedSpeechCueingRules')
    self.__cued = cue_rules
    self._vrank = tuple(self.__cued.get_vowels_codes())

positions_discretization

Generate the probabilities of the positions for each annotation in tier_pos_coords.

Discretizing the transitions between a position to the next one implies to assign a probability to the source and to the destination during the whole transition time.

Parameters
  • tierposcoords: (sppasTier) Tier with coordinates of the vowel positions and neutral
  • tierpostransitions: (sppasTier) Tier with transition intervals between vowels
Returns
  • (sppasTier) Tier "CS-PosProbas" with probabilities for each position
View Source
def positions_discretization(self, tier_pos_coords, tier_pos_transitions):
    """Generate the probabilities of the positions for each annotation in tier_pos_coords.

        Discretizing the transitions between a position to the next 
        one implies to assign a probability to the source and to the 
        destination during the whole transition time.

        :param tier_pos_coords: (sppasTier) Tier with coordinates of the vowel positions and neutral
        :param tier_pos_transitions: (sppasTier) Tier with transition intervals between vowels
        :return: (sppasTier) Tier "CS-PosProbas" with probabilities for each position 

        """
    pos_tier = sppasTier('CS-PosProbas')
    pos_probas = self.__discretize_positions(tier_pos_transitions, tier_pos_coords)
    if len(pos_probas) != len(tier_pos_coords):
        raise Exception('Target vowels probas estimation: {:d} != {:d}'.format(len(pos_probas), len(tier_pos_coords)))
    for (i, ann) in enumerate(tier_pos_coords):
        loc = ann.get_location()
        (tags, scores) = self.__probas_to_lists(pos_probas[i])
        label = sppasLabel(None)
        for t in range(len(tags)):
            label.append(tags[t], score=scores[t], add=False)
        pos_tier.create_annotation(loc.copy(), [label])
    return pos_tier

shapes_discretization

Generate the probabilities of the shapes for each annotation in tier_pos_coords.

Discretizing the transitions between a shape to the next one implies to assign a probability to the source and to the destination during the whole transition time.

Parameters
  • tierposcoords: (sppasTier) Tier with coordinates of the vowel positions and neutral
  • tiershapestransitions: (sppasTier) Tier with transition intervals between shapes
Returns
  • (sppasTier) Tier "CS-ShapeProbas" with probabilities for each shape
View Source
def shapes_discretization(self, tier_pos_coords, tier_shapes_transitions):
    """Generate the probabilities of the shapes for each annotation in tier_pos_coords.

        Discretizing the transitions between a shape to the next 
        one implies to assign a probability to the source and to the 
        destination during the whole transition time.

        :param tier_pos_coords: (sppasTier) Tier with coordinates of the vowel positions and neutral
        :param tier_shapes_transitions: (sppasTier) Tier with transition intervals between shapes
        :return: (sppasTier) Tier "CS-ShapeProbas" with probabilities for each shape 

        """
    shape_tier = sppasTier('CS-ShapeProbas')
    shape_probas = self.__discretize_shapes(tier_shapes_transitions, tier_pos_coords)
    if len(shape_probas) != len(tier_pos_coords):
        raise Exception('Target consonants probas estimation: {:d} != {:d}'.format(len(shape_probas), len(tier_pos_coords)))
    for (i, ann) in enumerate(tier_pos_coords):
        loc = ann.get_location()
        (tags, scores) = self.__probas_to_lists(shape_probas[i])
        label = sppasLabel(None)
        for t in range(len(tags)):
            label.append(tags[t], score=scores[t], add=False)
        shape_tier.create_annotation(loc.copy(), [label])
    return shape_tier

hands_to_target_coords

Generate the coordinates of the target finger.

Parameters
  • handposprobas: (sppasTier) Tier with hand position probabilities
  • vowels_coords: (sppasTier) Tier with coordinates of the vowels and neutral
Returns
  • (sppasTier) CS-TargetCoords
View Source
def hands_to_target_coords(self, hand_pos_probas, vowels_coords):
    """Generate the coordinates of the target finger.

        :param hand_pos_probas: (sppasTier) Tier with hand position probabilities
        :param vowels_coords: (sppasTier) Tier with coordinates of the vowels and neutral
        :return: (sppasTier) CS-TargetCoords

        """
    position_coords = self.__eval_hand_target_coords_straight(hand_pos_probas, vowels_coords)
    if len(position_coords) != len(vowels_coords):
        raise Exception('Target vowels coords estimation: {:d} != {:d}'.format(len(position_coords), len(vowels_coords)))
    pos_tier = sppasTier('CS-TargetCoords')
    for (i, ann) in enumerate(vowels_coords):
        loc = ann.get_location()
        if len(position_coords[i]) == 1:
            (x, y, r) = position_coords[i][0][0]
            label_pos = sppasLabel(sppasTag((x, y, r), tag_type='point'))
        elif len(position_coords[i]) == 2:
            pos1 = position_coords[i][0]
            (x1, y1, r1) = pos1[0]
            tag1 = sppasTag((x1, y1, r1), tag_type='point')
            pos2 = position_coords[i][1]
            (x2, y2, r2) = pos2[0]
            tag2 = sppasTag((x2, y2, r2), tag_type='point')
            label_pos = sppasLabel([tag1, tag2], [pos1[1], pos2[1]])
        else:
            label_pos = sppasLabel(None)
        pos_tier.create_annotation(loc.copy(), [label_pos])
    return pos_tier

Protected functions

__discretize_positions

Return proba of the hand position for each ann in given tier.

Returns
  • (sppasTier)
Parameters
  • hand_pos
  • vowelscoordstier
View Source
def __discretize_positions(self, hand_pos, vowels_coords_tier):
    """Return proba of the hand position for each ann in given tier.

        :return: (sppasTier)

        """
    positions = list()
    cur_pos = [self.__cued.get_neutral_vowel()]
    i = 0
    while i < len(vowels_coords_tier):
        loc = vowels_coords_tier[i].get_location().get_best()
        new_content = self.__get_label_contents_at(hand_pos, loc)
        if new_content is None:
            if len(cur_pos) == 2:
                cur_pos.pop(0)
        else:
            cur_pos = new_content
        c1 = cur_pos[0]
        if len(cur_pos) == 1:
            positions.append(((c1, 1.0),))
            i += 1
        else:
            c2 = cur_pos[1]
            new_pos = cur_pos
            nb_img = 0
            while new_pos == cur_pos:
                if i + nb_img == len(vowels_coords_tier):
                    break
                nb_img += 1
                if i + nb_img == len(vowels_coords_tier):
                    break
                loc = vowels_coords_tier[i + nb_img].get_location().get_best()
                new_pos = self.__get_label_contents_at(hand_pos, loc)
            if nb_img == 1:
                p2 = 0.65
            elif nb_img == 2:
                p2 = 0.4
            else:
                p_step_straight = 1.0 / float(nb_img)
                p2 = max(0.1, p_step_straight)
            positions.append(((c1, 1.0 - p2), (c2, p2)))
            i += 1
            proba_step = (1.0 - p2) / float(nb_img + 1)
            for j in range(1, nb_img):
                proba_pos2 = p2 + round((j + 1) * proba_step, 2)
                positions.append(((c1, 1 - proba_pos2), (c2, proba_pos2)))
                i += 1
    return positions

__discretize_shapes

Return proba of the hand shape for each ann in given tier.

Returns
  • (sppasTier)
Parameters
  • hand_shapes
  • vowelscoordstier
View Source
def __discretize_shapes(self, hand_shapes, vowels_coords_tier):
    """Return proba of the hand shape for each ann in given tier.

        :return: (sppasTier)

        """
    shapes = list()
    cur_shape = [self.__cued.get_neutral_consonant()]
    i = 0
    while i < len(vowels_coords_tier):
        loc = vowels_coords_tier[i].get_location().get_best()
        new_content = self.__get_label_contents_at(hand_shapes, loc)
        if new_content is None:
            if len(cur_shape) == 2:
                cur_shape.pop(0)
        else:
            cur_shape = new_content
        c1 = cur_shape[0]
        if len(cur_shape) == 1:
            shapes.append(((c1, 1.0),))
            i += 1
        else:
            c2 = cur_shape[1]
            new_shape = cur_shape
            nb_img = 0
            while new_shape == cur_shape:
                if i + nb_img == len(vowels_coords_tier):
                    break
                nb_img += 1
                if i + nb_img == len(vowels_coords_tier):
                    break
                loc = vowels_coords_tier[i + nb_img].get_location().get_best()
                new_shape = self.__get_label_contents_at(hand_shapes, loc)
            if nb_img == 1:
                shapes.append(((c1, 0.25), (c2, 0.75)))
                i += 1
            elif nb_img == 2:
                shapes.append(((c1, 0.45), (c2, 0.55)))
                shapes.append(((c1, 0.15), (c2, 0.85)))
                i += 2
            elif nb_img == 3:
                shapes.append(((c1, 0.6), (c2, 0.4)))
                shapes.append(((c1, 0.25), (c2, 0.75)))
                shapes.append(((c1, 0.1), (c2, 0.9)))
                i += 3
            else:
                p = 0.35
                shapes.append(((c1, 1.0 - p), (c2, p)))
                i += 1
                proba_step = (1.0 - p) / float(nb_img)
                for j in range(1, nb_img - 2):
                    proba_shape1 = p + round((j + 1) * proba_step, 2)
                    proba_shape0 = 1.0 - proba_shape1
                    shapes.append(((c1, proba_shape0), (c2, proba_shape1)))
                    i += 1
                proba_shape1 = 1.0 - proba_step / 2.0
                proba_shape0 = 1.0 - proba_shape1
                shapes.append(((c1, proba_shape0), (c2, proba_shape1)))
                i += 1
                proba_shape1 = 1.0 - proba_step / 4.0
                proba_shape0 = 1.0 - proba_shape1
                shapes.append(((c1, proba_shape0), (c2, proba_shape1)))
                i += 1
    return shapes

__eval_hand_target_coords_straight

Return coords of the hand position for each ann in given tier.

Coordinates are following a straight line to go from a position to the next one. It is ignoring the fact that keys have a different target finger: it is a straight line from a target position to the next one.

Returns
  • (list)
Parameters
  • handposprobas
  • vowelscoordstier
View Source
def __eval_hand_target_coords_straight(self, hand_pos_probas, vowels_coords_tier):
    """Return coords of the hand position for each ann in given tier.

        Coordinates are following a straight line to go from a position to
        the next one. It is ignoring the fact that keys have a different
        target finger: it is a straight line from a target position to the
        next one.

        :return: (list)

        """
    pos_coords = list()
    for i in range(len(hand_pos_probas)):
        labels = vowels_coords_tier[i].get_labels()
        vowels_coords = [label.get_best().get_typed_content() for label in labels]
        cur_vowels = list()
        cur_probas = list()
        for label in hand_pos_probas[i].get_labels():
            for (tag, score) in label:
                cur_vowels.append(tag.get_content())
                cur_probas.append(score)
        from_vowel_idx = self._vrank.index(cur_vowels[0])
        coord1 = vowels_coords[from_vowel_idx]
        (x1, y1) = coord1.get_midpoint()
        r1 = coord1.get_radius()
        if len(cur_vowels) == 1:
            pos_coords.append([((x1, y1, r1), 1.0)])
        elif len(cur_vowels) == 2:
            try:
                to_vowel_idx = self._vrank.index(cur_vowels[1])
            except ValueError:
                logging.error('Unknown vowel: {}'.format(cur_vowels[1]))
                continue
            coord2 = vowels_coords[to_vowel_idx]
            (x2, y2) = coord2.get_midpoint()
            r2 = coord2.get_radius()
            if from_vowel_idx == to_vowel_idx and r2 is not None:
                if cur_probas[0] > cur_probas[1]:
                    x2 = x2 - r2
                    y2 = y2 + r2
                else:
                    x1 = x1 - r2
                    y1 = y1 + r2
            dx = x2 - x1
            dy = y2 - y1
            x = x1 + int(float(dx) * cur_probas[1])
            y = y1 + int(float(dy) * cur_probas[1])
            r = 1
            if r1 is not None and r2 is not None:
                dr = r2 - r1
                r = r1 + int(float(dr) * cur_probas[1])
            pos_coords.append([((x, y, r), 1.0)])
        else:
            pos_coords.append([])
            logging.error('No vowel at index {:d}'.format(i))
    return pos_coords

__eval_hand_target_coords_fixed

Return coords of the hand position for each ann in given tier.

The hand does not move from a position to the next one. Its position is the one of the vowel.

Returns
  • (list)
Parameters
  • handposprobas
  • vowelscoordstier
View Source
def __eval_hand_target_coords_fixed(self, hand_pos_probas, vowels_coords_tier):
    """Return coords of the hand position for each ann in given tier.

        The hand does not move from a position to the next one.
        Its position is the one of the vowel.

        :return: (list)

        """
    pos_coords = list()
    for i in range(len(hand_pos_probas)):
        labels = vowels_coords_tier[i].get_labels()
        vowels_coords = [label.get_best().get_typed_content() for label in labels]
        cur_vowels = list()
        cur_probas = list()
        for label in hand_pos_probas[i].get_labels():
            for (tag, score) in label:
                cur_vowels.append(tag.get_content())
                cur_probas.append(score)
        from_vowel_idx = self._vrank.index(cur_vowels[0])
        coord1 = vowels_coords[from_vowel_idx]
        (x1, y1) = coord1.get_midpoint()
        r1 = coord1.get_radius()
        if len(cur_vowels) == 1:
            pos_coords.append([((x1, y1, r1), 1.0)])
        elif len(cur_vowels) == 2:
            to_vowel_idx = self._vrank.index(cur_vowels[1])
            coord2 = vowels_coords[to_vowel_idx]
            (x2, y2) = coord2.get_midpoint()
            r2 = coord2.get_radius()
            if from_vowel_idx == to_vowel_idx:
                xm = x2 - r2
                ym = y2 + r2
                pos_coords.append([((xm, ym, r2), cur_probas[1])])
            else:
                pos_coords.append([((x1, y1, r1), cur_probas[0]), ((x2, y2, r2), cur_probas[1])])
        else:
            pos_coords.append([])
            logging.error('No vowel at index {:d}'.format(i))
    return pos_coords

__create_tier

Create a tier with a content for each given vowels coords.

Parameters
  • transition_tier
  • vowels_coords
  • consonant
View Source
def __create_tier(self, transition_tier, vowels_coords, consonant=True):
    """Create a tier with a content for each given vowels coords."""
    if consonant is True:
        tier = sppasTier('CS-Shapes')
        cur_content = [self.__cued.get_neutral_consonant()]
    else:
        tier = sppasTier('CS-Positions')
        cur_content = [self.__cued.get_neutral_vowel()]
    for ann in vowels_coords:
        loc = ann.get_location().get_best()
        new_content = self.__get_label_contents_at(transition_tier, loc)
        if new_content is None:
            if len(cur_content) == 2:
                cur_content.pop(0)
        labels = [sppasLabel(sppasTag(c)) for c in cur_content]
        tier.create_annotation(ann.get_location().copy(), labels)
    return tier

__get_label_contents_at

Return the list of label contents of the annotation at the given moment.

Parameters
  • tier
  • point
View Source
@staticmethod
def __get_label_contents_at(tier, point):
    """Return the list of label contents of the annotation at the given moment."""
    content = None
    ann_idx = tier.mindex(point, bound=2)
    if ann_idx != -1:
        labels = tier[ann_idx].get_labels()
        if len(labels) == 1:
            content = [labels[0].get_best().get_content()]
        elif len(labels) == 2:
            c1 = labels[0].get_best().get_content()
            c2 = labels[1].get_best().get_content()
            content = [c1, c2]
        else:
            raise ValueError('One or two labels were expected. Got {:d} instead.'.format(len(labels)))
    return content

__probas_to_lists

Return the tags and scores of the given list of probabilities.

Parameters
  • probas
View Source
@staticmethod
def __probas_to_lists(probas):
    """Return the tags and scores of the given list of probabilities."""
    tags = list()
    scores = list()
    for s in probas:
        tags.append(sppasTag(s[0]))
        scores.append(s[1])
    return (tags, scores)