Auto-CS 2.2

https://sourceforge.net/projects/autocs/

Module sppas.src.annotations.CuedSpeech.whowtag

Class CuedSpeechVideoTagger

Description

Create a video with hands tagged on the face of a video.

Constructor

__init__

Create a new instance.

Parameters
  • cue_rules: (CuedSpeechKeys) Rules and codes for vowel positions and hand shapes
View Source
def __init__(self, cue_rules: CuedSpeechKeys=CuedSpeechKeys()):
    """Create a new instance.

    :param cue_rules: (CuedSpeechKeys) Rules and codes for vowel positions and hand shapes

    """
    self.__gencoords = sppasHandCoordsGenerator()
    self.__video_buffer = sppasVideoReaderBuffer()
    self.__video_writer = sppasBufferVideoWriter()
    self.__video_path = None
    if isinstance(cue_rules, CuedSpeechKeys) is False:
        raise sppasTypeError(type(cue_rules), 'CuedSpeechKeys')
    self.__options = dict()
    for key in CuedSpeechVideoTagger.OPTIONS:
        self.__options[key] = CuedSpeechVideoTagger.OPTIONS[key]
    self._img_vowel_tagger = sppasImageVowelPosTagger(cue_rules)
    self._img_hand_tagger = sppasImageHandTagger(cue_rules)
    self.set_cue_rules(cue_rules)

Public functions

set_cue_rules

Set the CuedSpeechKeys used to tag the video.

Parameters
  • cue_rules: (CuedSpeechKeys) The instance of the cuedSpeechKeys to set
Raises
  • **: sppasTypeError: If the parameter is not an instance of the CuedSpeechKeys class
View Source
def set_cue_rules(self, cue_rules: CuedSpeechKeys) -> None:
    """Set the CuedSpeechKeys used to tag the video.

        :param cue_rules: (CuedSpeechKeys) The instance of the cuedSpeechKeys to set
        :raises: : sppasTypeError: If the parameter is not an instance of the CuedSpeechKeys class

        """
    if isinstance(cue_rules, CuedSpeechKeys) is False:
        raise sppasTypeError(cue_rules, 'CuedSpeechKeys')
    self._img_hand_tagger.set_cue_rules(cue_rules)
    self._img_vowel_tagger.set_cue_rules(cue_rules)
    self.__gencoords.set_cue_rules(cue_rules)

load_hands

Load the hand pictures with given name.

Parameters
  • prefix: (str) A hand set name (handcue, ...)
Raises
  • sppasIOError: If no hands are loaded (not found the hands files)
View Source
def load_hands(self, prefix: str) -> None:
    """Load the hand pictures with given name.

        :param prefix: (str) A hand set name (handcue, ...)
        :raises: sppasIOError: If no hands are loaded (not found the hands files)

        """
    success = self._img_hand_tagger.load_hands(prefix)
    if success is False:
        raise sppasIOError(f'Hand pictures with name {prefix} not loaded.')

load

Open the video.

Closes automatically the video if already was an open video.

Parameters
  • video_path: (str) File path of the input video
Raises
  • VideoBrowseError: Can't open/read the video
View Source
def load(self, video_path: str) -> None:
    """Open the video.

        Closes automatically the video if already was an open video.

        :param video_path: (str) File path of the input video
        :raises: VideoBrowseError: Can't open/read the video

        """
    self.close()
    self.__video_buffer.open(video_path)
    self.__video_path = video_path
    self.__video_writer.set_fps(self.__video_buffer.get_framerate())

is_loaded

Return the information of if a video is loaded or not.

Returns
  • (bool) True if the buffer is currently opened or False else
View Source
def is_loaded(self) -> bool:
    """Return the information of if a video is loaded or not.

        :return: (bool) True if the buffer is currently opened or False else

        """
    if self.__video_buffer is None:
        return False
    return self.__video_buffer.is_opened()

close

Release video streams.

View Source
def close(self) -> None:
    """Release video streams."""
    self.__video_buffer.close()
    self.__video_writer.close()

is_opened

Return the information of if the writer is currently opened.

Returns
  • (bool) True if the video writer is open or False else.
View Source
def is_opened(self) -> bool:
    """Return the information of if the writer is currently opened.

        :return: (bool) True if the video writer is open or False else.

        """
    return self.__video_writer.is_opened()

set_option

Set an option of the video tagger.

Available options are : - handsset - handsfilter - infotext - vowelspos

Parameters
  • option_key: (str) Name of the option
  • option_value: (any) Value of the option
Raises
  • AnnotationOptionError: If the given key is unknown
View Source
def set_option(self, option_key: str, option_value: object) -> None:
    """Set an option of the video tagger.

        Available options are :
            - handsset
            - handsfilter
            - infotext
            - vowelspos

        :param option_key: (str) Name of the option
        :param option_value: (any) Value of the option
        :raises: AnnotationOptionError: If the given key is unknown

        """
    if option_key in self.__options.keys():
        self.__options[option_key] = option_value
    else:
        raise AnnotationOptionError(option_key)

tag_with_keys

Tag the video with the given keys.

The possible tiers of the given transcription are: - CS-ShapeProbas: probabilities of the shapes of the hand - CS-PosProbas: probabilities of the positions of the hand - CS-VowelsCoords: Coordinates of the position of the vowels - CS-TargetCoords: Coordinates of the target sight - CS-HandAngle: Coordinates of the target sight - PhonAlign: Time-aligned phonemes

The video streams (buffer/writer) are not released... Use close().

Parameters
  • transcription: (sppasTranscription) The transcription to use to tag the video.
  • output: (str) Output video filename
Raises
  • sppasError: If no video is load.
Returns
  • (list) created files -- expected 1
View Source
def tag_with_keys(self, transcription: sppasTranscription, output: str) -> list:
    """Tag the video with the given keys.

        The possible tiers of the given transcription are:
            - CS-ShapeProbas: probabilities of the shapes of the hand
            - CS-PosProbas: probabilities of the positions of the hand
            - CS-VowelsCoords: Coordinates of the position of the vowels
            - CS-TargetCoords: Coordinates of the target sight
            - CS-HandAngle: Coordinates of the target sight
            - PhonAlign: Time-aligned phonemes

        The video streams (buffer/writer) are not released... Use close().

        :param transcription: (sppasTranscription) The transcription to use to tag the video.
        :param output: (str) Output video filename
        :raises: sppasError: If no video is load.
        :return: (list) created files -- expected 1

        """
    if self.is_loaded() is False:
        raise sppasError('No video in the buffer. Nothing can be tagged.')
    self.__load_options()
    self.__check_tiers(transcription)
    if self._img_hand_tagger.hand_mode() is True:
        _tagger = self._img_hand_tagger
    else:
        _tagger = None
    hand_coords_tier = self.__gencoords.hands_to_handcoords(transcription.find('CS-ShapeProbas'), transcription.find('CS-TargetCoords'), transcription.find('CS-HandAngle'), transcription.find('CS-FaceHeight'), _tagger)
    result = list()
    index = 0
    nb_buffer = 0
    read_next = True
    self.__video_buffer.seek_buffer(0)
    image_duration = 1.0 / self.__video_buffer.get_framerate()
    while read_next:
        logging.info(f' ... buffer number {nb_buffer + 1}')
        read_next = self.__video_buffer.next()
        start_time = index * image_duration
        self.__tag_buffer(start_time, hand_coords_tier, transcription.find('CS-ShapeProbas'), transcription.find('CS-PosProbas'), transcription.find('CS-VowelsCoords'), transcription.find('PhonAlign'))
        if output is not None:
            new_files = self.__video_writer.write_video(self.__video_buffer, output, '')
            result.extend(new_files)
        nb_buffer += 1
        index += len(self.__video_buffer)
    self.load(self.__video_path)
    return result

get_hands_filters

List of all available hands filters.

Returns
  • (list[str]) A list of all hands filters.
View Source
@staticmethod
def get_hands_filters() -> list:
    """List of all available hands filters.

        :return: (list[str]) A list of all hands filters.

        """
    return ['cartoon', 'sights', 'skeleton', 'sticks']

get_annotation_index

Return the index corresponding to the given time point.

Parameters
  • tier: (sppasTier) The tier that contains the annotation who searched
  • timepoint: (sppasPoint or float) The time to search for the index
Returns
  • (int) The index or -1 if no index associated with the given time point
View Source
@staticmethod
def get_annotation_index(tier: sppasTier, timepoint) -> int:
    """Return the index corresponding to the given time point.

        :param tier: (sppasTier) The tier that contains the annotation who searched
        :param timepoint: (sppasPoint or float) The time to search for the index

        :return: (int) The index or -1 if no index associated with the given time point

        """
    if tier is None:
        annotation_index = -1
    elif tier.is_point():
        annotation_index = tier.index(timepoint)
    else:
        annotation_index = tier.mindex(timepoint, bound=2)
    return annotation_index

get_annotation_index_starting_to

Return the next index corresponding to the given time point starting to the start_index parameter.

Parameters
  • tier: (sppasTier) The tier that contains the annotation who searched
  • timepoint: (sppasPoint or float) The time to search for the index
  • start_index: (int) The index to start to search for the index
Returns
  • (int) The next annotation index that corresponding with the given timepoint Or -1 if no annotation found
View Source
@staticmethod
def get_annotation_index_starting_to(tier: sppasTier, timepoint, start_index: int) -> int:
    """Return the next index corresponding to the given time point starting to the start_index parameter.

        :param tier: (sppasTier) The tier that  contains the annotation who searched
        :param timepoint: (sppasPoint or float) The time to search for the index
        :param start_index: (int) The index to start to search for the index

        :return: (int) The next annotation index that corresponding with the given timepoint
                 Or -1 if no annotation found

        """
    if tier is None:
        return -1
    if start_index >= len(tier):
        return -1
    elif start_index <= 0:
        return CuedSpeechVideoTagger.get_annotation_index(tier, timepoint)
    new_index = start_index
    while new_index != -1:
        if tier[new_index].get_lowest_localization() <= timepoint <= tier[new_index].get_highest_localization():
            break
        new_index += 1
        if new_index == len(tier):
            new_index = -1
        if timepoint > tier[new_index].get_highest_localization():
            new_index = -1
    return new_index

Protected functions

__load_options

Check the activated options and load the corresponding resources.

View Source
def __load_options(self) -> None:
    """Check the activated options and load the corresponding resources."""
    hand_set = self.__options.get('handsset')
    if len(hand_set) > 0:
        self.load_hands(hand_set)
        logging.info(f"Successfully loaded the hands set '{hand_set}'.")
        hand_filter = self.__options.get('handsfilter')
        if len(hand_filter) > 0:
            self._img_hand_tagger.apply_hands_filter(hand_filter)
            logging.info(f"Successfully applied the hands filter '{hand_filter}'.")
    else:
        logging.info('No hands set was defined. The video is tagged with badges.')

__check_tiers

Find the tiers used to tag the video.

Searched tiers (on the order of the return list): - CS-ShapeProbas: probabilities of the shapes of the hand - CS-PosProbas: probabilities of the positions of the hand - CS-VowelsCoords: Coordinates of the position of the vowels - PhonAlign: Time-aligned phonemes

Parameters
  • transcription
View Source
def __check_tiers(self, transcription: sppasTranscription) -> None:
    """Find the tiers used to tag the video.

        Searched tiers (on the order of the return list):
            - CS-ShapeProbas: probabilities of the shapes of the hand
            - CS-PosProbas: probabilities of the positions of the hand
            - CS-VowelsCoords: Coordinates of the position of the vowels
            - PhonAlign: Time-aligned phonemes

        """
    tiers = list()
    tiers.append(transcription.find('CS-ShapeProbas'))
    if tiers[0] is None:
        logging.error('Missing tier with shape probabilities.')
    tiers.append(transcription.find('CS-PosProbas'))
    if tiers[1] is None:
        logging.error('Missing tier with position probabilities.')
    tiers.append(transcription.find('CS-VowelsCoords'))
    if tiers[2] is None:
        logging.error('Missing tier with vowels coords.')
    tiers.append(transcription.find('PhonAlign'))
    if tiers[3] is None:
        logging.error('Missing tier with time-aligned phonemes.')
    tiers.append(transcription.find('CS-TargetCoords'))
    if tiers[4] is None:
        logging.error('Missing tier with target coordinates.')
    tiers.append(transcription.find('CS-HandAngle'))
    if tiers[5] is None:
        logging.error('Missing tier with angle values.')
    tiers.append(transcription.find('CS-FaceHeight'))
    if tiers[6] is None:
        logging.error('Missing tier with face heights.')
    if None in tiers:
        raise sppasError('At least one of the expected tiers was not found in the transcription. Video tagging is aborted. See logging for details.')

__tag_buffer

Browse the buffer and tag the images.

Parameters
  • start_time: (float) The time when the tag begin
  • coords_tier: (sppasTier or None) The sights of the hand for the target, S0 and S9
  • pos_tier: (sppasTier or None) Positions and probabilities (CS-PosProbas)
  • shape_tier: (sppasTier or None) Shape codes and probabilities (CS-ShapeProbas)
  • vowels_tier: (sppasTier or None) The vowels positions (CS-VowelsCoords)
  • info_tier: (sppasTier or None) The phonemes, used to print on the image in debug mode (PhonAlign)
View Source
def __tag_buffer(self, start_time: float, coords_tier, shape_tier, pos_tier, vowels_tier, info_tier) -> None:
    """Browse the buffer and tag the images.

        :param start_time: (float) The time when the tag begin
        :param coords_tier: (sppasTier or None) The sights of the hand for the target, S0 and S9
        :param pos_tier: (sppasTier or None) Positions and probabilities (CS-PosProbas)
        :param shape_tier: (sppasTier or None) Shape codes and probabilities (CS-ShapeProbas)
        :param vowels_tier: (sppasTier or None) The vowels positions (CS-VowelsCoords)
        :param info_tier: (sppasTier or None) The phonemes, used to print on the image in debug mode (PhonAlign)

        """
    image_duration = 1.0 / self.__video_buffer.get_framerate()
    radius = 0.0005
    pos_index = shape_index = vowels_index = info_index = coords_index = -1
    (start_buffer_index, _) = self.__video_buffer.get_buffer_range()
    iter_images = self.__video_buffer.__iter__()
    for i in range(len(self.__video_buffer)):
        video_image_number = start_buffer_index + i
        img = next(iter_images)
        if img is None:
            logging.warning('No frame at index {:d}.'.format(video_image_number))
            continue
        img = img.ialpha(254)
        s = start_time + i * image_duration
        middle_time = s + image_duration / 2.0
        point = sppasPoint(middle_time, radius)
        coords_index = self.get_annotation_index_starting_to(coords_tier, point, coords_index)
        pos_index = self.get_annotation_index_starting_to(pos_tier, point, pos_index)
        shape_index = self.get_annotation_index_starting_to(shape_tier, point, shape_index)
        vowels_index = self.get_annotation_index_starting_to(vowels_tier, point, vowels_index)
        info_index = self.get_annotation_index_starting_to(info_tier, point, info_index)
        sights = self.__extract_coords_data(coords_tier, coords_index)
        (shape_text, shapes) = self.__extract_shape_data(shape_tier, shape_index)
        (pos_text, score) = self.__extract_position_data(pos_tier, pos_index)
        vowels_positions = self.__extract_vowels_data(vowels_tier, vowels_index)
        if shape_index != -1 and pos_index != -1 and (len(sights) > 0) and (sights[0] is not None):
            img = self._img_hand_tagger.slap_on(img, shapes, sights)
        if self.__options['vowelspos'] is True and len(vowels_positions) > 0:
            img = self._img_vowel_tagger.slap_on(img, vowels_positions)
        if self.__options['infotext'] is True:
            info_text = 'Info: '
            if info_index != -1:
                info_text += str(round(info_tier[info_index].get_lowest_localization().get_midpoint(), 3))
                info_text += ' '
                info_text += serialize_labels(info_tier[info_index].get_labels())
            frame_text = f'Frame: {video_image_number} ({round(middle_time, 3)},{round(radius, 3)})'
            self.__put_debug_texts(img, [frame_text, shape_text, pos_text, info_text])
        self.__video_buffer.set_at(img.ibgra_to_bgr(), i)

__put_debug_texts

Put debug information on the given image.

Parameters
  • img: (sppasImage) The image to put the text on it.
  • texts: (list) The list that contains all texts to put in the image.
View Source
def __put_debug_texts(self, img: sppasImage, texts: list) -> None:
    """Put debug information on the given image.

        :param img: (sppasImage) The image to put the text on it.
        :param texts: (list) The list that contains all texts to put in the image.

        """
    step = 30
    for (i, text) in enumerate(texts):
        img.put_text((100, (i + 1) * step), (10, 10, 10), 1, text)

__extract_coords_data

Extract the sights data to compute the hand scale factor.

If the index equal to -1, this function returns a list of None.

Parameters
  • coords_tier: (sppasTier or None) The tier that contains the sights
  • index: (int) The index of the annotation
Returns
  • (list) A list of the coords for s0, s9, target

Example of returned list: [sppasFuzzyPoint: (368,780), sppasFuzzyPoint: (432,684), sppasFuzzyPoint: (540,573)]

View Source
@staticmethod
def __extract_coords_data(coords_tier, index) -> list:
    """Extract the sights data to compute the hand scale factor.

        If the index equal to -1, this function returns a list of None.

        :param coords_tier: (sppasTier or None) The tier that contains the sights
        :param index: (int) The index of the annotation
        :return: (list) A list of the coords for s0, s9, target

        Example of returned list:
        [sppasFuzzyPoint: (368,780), sppasFuzzyPoint: (432,684), sppasFuzzyPoint: (540,573)]

        """
    sights = [None, None, None]
    if coords_tier is not None and index != -1:
        return coords_tier[index].get_labels()
    return [None, None, None]

__extract_shape_data

Extract the shape data (shapes codes and scores) from a shape tier.

Parameters
  • shape_tier: (sppasTier) The tier that contains our shape data
  • index: (int) The index of the annotation
Returns
  • (str, list) The shape text information and the list of shapes codes and scores
View Source
@staticmethod
def __extract_shape_data(shape_tier: sppasTier, index: int) -> tuple:
    """Extract the shape data (shapes codes and scores) from a shape tier.

        :param shape_tier: (sppasTier) The tier that contains our shape data
        :param index: (int) The index of the annotation
        :return: (str, list) The shape text information and the list of shapes codes and scores

        """
    shape_text = 'Shape: '
    shapes = list()
    for shape_label in shape_tier[index].get_labels():
        for (tag, score) in shape_label:
            shape_code = tag.get_typed_content()
            if score is None:
                score = 1.0
            shape_text += shape_code
            shapes.append((shape_code, score))
    return (shape_text, shapes)

__extract_position_data

Extract the position data from the position tiers.

Parameters
  • pos_tier: (sppasTier or None) The tier that contains the positions text information
  • index: (int) The index of the annotation
Returns
  • (str) The position text information and the position probability
View Source
@staticmethod
def __extract_position_data(pos_tier, index: int) -> tuple:
    """Extract the position data from the position tiers.

        :param pos_tier: (sppasTier or None) The tier that contains the positions text information
        :param index: (int) The index of the annotation
        :return: (str) The position text information and the position probability

        """
    pos_text = 'Pos: '
    score = 1
    if pos_tier is not None:
        for pos_label in pos_tier[index].get_labels():
            score = pos_label.get_score(pos_label.get_best())
            for (tag, score) in pos_label:
                pos_text += tag.get_typed_content()
    return (pos_text, score)

__extract_vowels_data

Extract the vowels data from the vowels tier.

If the vowels tier is None or the index equal to -1 return an empty list by default.

Parameters
  • vowels_tier: (sppasTier or None) The tier that contains our vowels data
  • index: (int) The index of the annotation
Returns
  • (list) A list of all vowels positions
View Source
@staticmethod
def __extract_vowels_data(vowels_tier, index: int) -> list:
    """Extract the vowels data from the vowels tier.

        If the vowels tier is None or the index equal to -1 return an empty list by default.

        :param vowels_tier: (sppasTier or None) The tier that contains our vowels data
        :param index: (int) The index of the annotation
        :return: (list) A list of all vowels positions

        """
    vowels_positions = list()
    if vowels_tier is not None and index != -1:
        for vowel_label in vowels_tier[index].get_labels():
            pos = vowel_label.get_best().get_typed_content()
            vowels_positions.append(pos)
    return vowels_positions

Overloads

__del__

View Source
def __del__(self):
    self.close()