Create a video with hands tagged on the face of a video.
Module sppas.src.annotations.CuedSpeech.whowtag
Class CuedSpeechVideoTagger
Description
Constructor
__init__
Create a new instance.
Parameters
- cue_rules: (CuedSpeechKeys) Rules and codes for vowel positions and hand shapes
View Source
def __init__(self, cue_rules: CuedSpeechKeys=CuedSpeechKeys()):
"""Create a new instance.
:param cue_rules: (CuedSpeechKeys) Rules and codes for vowel positions and hand shapes
"""
self.__gencoords = sppasHandCoordsGenerator()
self.__video_buffer = sppasVideoReaderBuffer()
self.__video_writer = sppasBufferVideoWriter()
self.__video_path = None
if isinstance(cue_rules, CuedSpeechKeys) is False:
raise sppasTypeError(type(cue_rules), 'CuedSpeechKeys')
self.__options = dict()
for key in CuedSpeechVideoTagger.OPTIONS:
self.__options[key] = CuedSpeechVideoTagger.OPTIONS[key]
self._img_vowel_tagger = sppasImageVowelPosTagger(cue_rules)
self._img_hand_tagger = sppasImageHandTagger(cue_rules)
self.set_cue_rules(cue_rules)
Public functions
set_cue_rules
Set the CuedSpeechKeys used to tag the video.
Parameters
- cue_rules: (CuedSpeechKeys) The instance of the cuedSpeechKeys to set
Raises
- **: sppasTypeError: If the parameter is not an instance of the CuedSpeechKeys class
View Source
def set_cue_rules(self, cue_rules: CuedSpeechKeys) -> None:
"""Set the CuedSpeechKeys used to tag the video.
:param cue_rules: (CuedSpeechKeys) The instance of the cuedSpeechKeys to set
:raises: : sppasTypeError: If the parameter is not an instance of the CuedSpeechKeys class
"""
if isinstance(cue_rules, CuedSpeechKeys) is False:
raise sppasTypeError(cue_rules, 'CuedSpeechKeys')
self._img_hand_tagger.set_cue_rules(cue_rules)
self._img_vowel_tagger.set_cue_rules(cue_rules)
self.__gencoords.set_cue_rules(cue_rules)
load_hands
Load the hand pictures with given name.
Parameters
- prefix: (str) A hand set name (handcue, ...)
Raises
- sppasIOError: If no hands are loaded (not found the hands files)
View Source
def load_hands(self, prefix: str) -> None:
"""Load the hand pictures with given name.
:param prefix: (str) A hand set name (handcue, ...)
:raises: sppasIOError: If no hands are loaded (not found the hands files)
"""
success = self._img_hand_tagger.load_hands(prefix)
if success is False:
raise sppasIOError(f'Hand pictures with name {prefix} not loaded.')
load
Open the video.
Closes automatically the video if already was an open video.
Parameters
- video_path: (str) File path of the input video
Raises
- VideoBrowseError: Can't open/read the video
View Source
def load(self, video_path: str) -> None:
"""Open the video.
Closes automatically the video if already was an open video.
:param video_path: (str) File path of the input video
:raises: VideoBrowseError: Can't open/read the video
"""
self.close()
self.__video_buffer.open(video_path)
self.__video_path = video_path
self.__video_writer.set_fps(self.__video_buffer.get_framerate())
is_loaded
Return the information of if a video is loaded or not.
Returns
- (bool) True if the buffer is currently opened or False else
View Source
def is_loaded(self) -> bool:
"""Return the information of if a video is loaded or not.
:return: (bool) True if the buffer is currently opened or False else
"""
if self.__video_buffer is None:
return False
return self.__video_buffer.is_opened()
close
Release video streams.
View Source
def close(self) -> None:
"""Release video streams."""
self.__video_buffer.close()
self.__video_writer.close()
is_opened
Return the information of if the writer is currently opened.
Returns
- (bool) True if the video writer is open or False else.
View Source
def is_opened(self) -> bool:
"""Return the information of if the writer is currently opened.
:return: (bool) True if the video writer is open or False else.
"""
return self.__video_writer.is_opened()
set_option
Set an option of the video tagger.
Available options are : - handsset - handsfilter - infotext - vowelspos
Parameters
- option_key: (str) Name of the option
- option_value: (any) Value of the option
Raises
- AnnotationOptionError: If the given key is unknown
View Source
def set_option(self, option_key: str, option_value: object) -> None:
"""Set an option of the video tagger.
Available options are :
- handsset
- handsfilter
- infotext
- vowelspos
:param option_key: (str) Name of the option
:param option_value: (any) Value of the option
:raises: AnnotationOptionError: If the given key is unknown
"""
if option_key in self.__options.keys():
self.__options[option_key] = option_value
else:
raise AnnotationOptionError(option_key)
tag_with_keys
Tag the video with the given keys.
The possible tiers of the given transcription are: - CS-ShapeProbas: probabilities of the shapes of the hand - CS-PosProbas: probabilities of the positions of the hand - CS-VowelsCoords: Coordinates of the position of the vowels - CS-TargetCoords: Coordinates of the target sight - CS-HandAngle: Coordinates of the target sight - PhonAlign: Time-aligned phonemes
The video streams (buffer/writer) are not released... Use close().
Parameters
- transcription: (sppasTranscription) The transcription to use to tag the video.
- output: (str) Output video filename
Raises
- sppasError: If no video is load.
Returns
- (list) created files -- expected 1
View Source
def tag_with_keys(self, transcription: sppasTranscription, output: str) -> list:
"""Tag the video with the given keys.
The possible tiers of the given transcription are:
- CS-ShapeProbas: probabilities of the shapes of the hand
- CS-PosProbas: probabilities of the positions of the hand
- CS-VowelsCoords: Coordinates of the position of the vowels
- CS-TargetCoords: Coordinates of the target sight
- CS-HandAngle: Coordinates of the target sight
- PhonAlign: Time-aligned phonemes
The video streams (buffer/writer) are not released... Use close().
:param transcription: (sppasTranscription) The transcription to use to tag the video.
:param output: (str) Output video filename
:raises: sppasError: If no video is load.
:return: (list) created files -- expected 1
"""
if self.is_loaded() is False:
raise sppasError('No video in the buffer. Nothing can be tagged.')
self.__load_options()
self.__check_tiers(transcription)
if self._img_hand_tagger.hand_mode() is True:
_tagger = self._img_hand_tagger
else:
_tagger = None
hand_coords_tier = self.__gencoords.hands_to_handcoords(transcription.find('CS-ShapeProbas'), transcription.find('CS-TargetCoords'), transcription.find('CS-HandAngle'), transcription.find('CS-FaceHeight'), _tagger)
result = list()
index = 0
nb_buffer = 0
read_next = True
self.__video_buffer.seek_buffer(0)
image_duration = 1.0 / self.__video_buffer.get_framerate()
while read_next:
logging.info(f' ... buffer number {nb_buffer + 1}')
read_next = self.__video_buffer.next()
start_time = index * image_duration
self.__tag_buffer(start_time, hand_coords_tier, transcription.find('CS-ShapeProbas'), transcription.find('CS-PosProbas'), transcription.find('CS-VowelsCoords'), transcription.find('PhonAlign'))
if output is not None:
new_files = self.__video_writer.write_video(self.__video_buffer, output, '')
result.extend(new_files)
nb_buffer += 1
index += len(self.__video_buffer)
self.load(self.__video_path)
return result
get_hands_filters
List of all available hands filters.
Returns
- (list[str]) A list of all hands filters.
View Source
@staticmethod
def get_hands_filters() -> list:
"""List of all available hands filters.
:return: (list[str]) A list of all hands filters.
"""
return ['cartoon', 'sights', 'skeleton', 'sticks']
get_annotation_index
Return the index corresponding to the given time point.
Parameters
- tier: (sppasTier) The tier that contains the annotation who searched
- timepoint: (sppasPoint or float) The time to search for the index
Returns
- (int) The index or -1 if no index associated with the given time point
View Source
@staticmethod
def get_annotation_index(tier: sppasTier, timepoint) -> int:
"""Return the index corresponding to the given time point.
:param tier: (sppasTier) The tier that contains the annotation who searched
:param timepoint: (sppasPoint or float) The time to search for the index
:return: (int) The index or -1 if no index associated with the given time point
"""
if tier is None:
annotation_index = -1
elif tier.is_point():
annotation_index = tier.index(timepoint)
else:
annotation_index = tier.mindex(timepoint, bound=2)
return annotation_index
get_annotation_index_starting_to
Return the next index corresponding to the given time point starting to the start_index parameter.
Parameters
- tier: (sppasTier) The tier that contains the annotation who searched
- timepoint: (sppasPoint or float) The time to search for the index
- start_index: (int) The index to start to search for the index
Returns
- (int) The next annotation index that corresponding with the given timepoint Or -1 if no annotation found
View Source
@staticmethod
def get_annotation_index_starting_to(tier: sppasTier, timepoint, start_index: int) -> int:
"""Return the next index corresponding to the given time point starting to the start_index parameter.
:param tier: (sppasTier) The tier that contains the annotation who searched
:param timepoint: (sppasPoint or float) The time to search for the index
:param start_index: (int) The index to start to search for the index
:return: (int) The next annotation index that corresponding with the given timepoint
Or -1 if no annotation found
"""
if tier is None:
return -1
if start_index >= len(tier):
return -1
elif start_index <= 0:
return CuedSpeechVideoTagger.get_annotation_index(tier, timepoint)
new_index = start_index
while new_index != -1:
if tier[new_index].get_lowest_localization() <= timepoint <= tier[new_index].get_highest_localization():
break
new_index += 1
if new_index == len(tier):
new_index = -1
if timepoint > tier[new_index].get_highest_localization():
new_index = -1
return new_index
Protected functions
__load_options
Check the activated options and load the corresponding resources.
View Source
def __load_options(self) -> None:
"""Check the activated options and load the corresponding resources."""
hand_set = self.__options.get('handsset')
if len(hand_set) > 0:
self.load_hands(hand_set)
logging.info(f"Successfully loaded the hands set '{hand_set}'.")
hand_filter = self.__options.get('handsfilter')
if len(hand_filter) > 0:
self._img_hand_tagger.apply_hands_filter(hand_filter)
logging.info(f"Successfully applied the hands filter '{hand_filter}'.")
else:
logging.info('No hands set was defined. The video is tagged with badges.')
__check_tiers
Find the tiers used to tag the video.
Searched tiers (on the order of the return list): - CS-ShapeProbas: probabilities of the shapes of the hand - CS-PosProbas: probabilities of the positions of the hand - CS-VowelsCoords: Coordinates of the position of the vowels - PhonAlign: Time-aligned phonemes
Parameters
- transcription
View Source
def __check_tiers(self, transcription: sppasTranscription) -> None:
"""Find the tiers used to tag the video.
Searched tiers (on the order of the return list):
- CS-ShapeProbas: probabilities of the shapes of the hand
- CS-PosProbas: probabilities of the positions of the hand
- CS-VowelsCoords: Coordinates of the position of the vowels
- PhonAlign: Time-aligned phonemes
"""
tiers = list()
tiers.append(transcription.find('CS-ShapeProbas'))
if tiers[0] is None:
logging.error('Missing tier with shape probabilities.')
tiers.append(transcription.find('CS-PosProbas'))
if tiers[1] is None:
logging.error('Missing tier with position probabilities.')
tiers.append(transcription.find('CS-VowelsCoords'))
if tiers[2] is None:
logging.error('Missing tier with vowels coords.')
tiers.append(transcription.find('PhonAlign'))
if tiers[3] is None:
logging.error('Missing tier with time-aligned phonemes.')
tiers.append(transcription.find('CS-TargetCoords'))
if tiers[4] is None:
logging.error('Missing tier with target coordinates.')
tiers.append(transcription.find('CS-HandAngle'))
if tiers[5] is None:
logging.error('Missing tier with angle values.')
tiers.append(transcription.find('CS-FaceHeight'))
if tiers[6] is None:
logging.error('Missing tier with face heights.')
if None in tiers:
raise sppasError('At least one of the expected tiers was not found in the transcription. Video tagging is aborted. See logging for details.')
__tag_buffer
Browse the buffer and tag the images.
Parameters
- start_time: (float) The time when the tag begin
- coords_tier: (sppasTier or None) The sights of the hand for the target, S0 and S9
- pos_tier: (sppasTier or None) Positions and probabilities (CS-PosProbas)
- shape_tier: (sppasTier or None) Shape codes and probabilities (CS-ShapeProbas)
- vowels_tier: (sppasTier or None) The vowels positions (CS-VowelsCoords)
- info_tier: (sppasTier or None) The phonemes, used to print on the image in debug mode (PhonAlign)
View Source
def __tag_buffer(self, start_time: float, coords_tier, shape_tier, pos_tier, vowels_tier, info_tier) -> None:
"""Browse the buffer and tag the images.
:param start_time: (float) The time when the tag begin
:param coords_tier: (sppasTier or None) The sights of the hand for the target, S0 and S9
:param pos_tier: (sppasTier or None) Positions and probabilities (CS-PosProbas)
:param shape_tier: (sppasTier or None) Shape codes and probabilities (CS-ShapeProbas)
:param vowels_tier: (sppasTier or None) The vowels positions (CS-VowelsCoords)
:param info_tier: (sppasTier or None) The phonemes, used to print on the image in debug mode (PhonAlign)
"""
image_duration = 1.0 / self.__video_buffer.get_framerate()
radius = 0.0005
pos_index = shape_index = vowels_index = info_index = coords_index = -1
(start_buffer_index, _) = self.__video_buffer.get_buffer_range()
iter_images = self.__video_buffer.__iter__()
for i in range(len(self.__video_buffer)):
video_image_number = start_buffer_index + i
img = next(iter_images)
if img is None:
logging.warning('No frame at index {:d}.'.format(video_image_number))
continue
img = img.ialpha(254)
s = start_time + i * image_duration
middle_time = s + image_duration / 2.0
point = sppasPoint(middle_time, radius)
coords_index = self.get_annotation_index_starting_to(coords_tier, point, coords_index)
pos_index = self.get_annotation_index_starting_to(pos_tier, point, pos_index)
shape_index = self.get_annotation_index_starting_to(shape_tier, point, shape_index)
vowels_index = self.get_annotation_index_starting_to(vowels_tier, point, vowels_index)
info_index = self.get_annotation_index_starting_to(info_tier, point, info_index)
sights = self.__extract_coords_data(coords_tier, coords_index)
(shape_text, shapes) = self.__extract_shape_data(shape_tier, shape_index)
(pos_text, score) = self.__extract_position_data(pos_tier, pos_index)
vowels_positions = self.__extract_vowels_data(vowels_tier, vowels_index)
if shape_index != -1 and pos_index != -1 and (len(sights) > 0) and (sights[0] is not None):
img = self._img_hand_tagger.slap_on(img, shapes, sights)
if self.__options['vowelspos'] is True and len(vowels_positions) > 0:
img = self._img_vowel_tagger.slap_on(img, vowels_positions)
if self.__options['infotext'] is True:
info_text = 'Info: '
if info_index != -1:
info_text += str(round(info_tier[info_index].get_lowest_localization().get_midpoint(), 3))
info_text += ' '
info_text += serialize_labels(info_tier[info_index].get_labels())
frame_text = f'Frame: {video_image_number} ({round(middle_time, 3)},{round(radius, 3)})'
self.__put_debug_texts(img, [frame_text, shape_text, pos_text, info_text])
self.__video_buffer.set_at(img.ibgra_to_bgr(), i)
__put_debug_texts
Put debug information on the given image.
Parameters
- img: (sppasImage) The image to put the text on it.
- texts: (list) The list that contains all texts to put in the image.
View Source
def __put_debug_texts(self, img: sppasImage, texts: list) -> None:
"""Put debug information on the given image.
:param img: (sppasImage) The image to put the text on it.
:param texts: (list) The list that contains all texts to put in the image.
"""
step = 30
for (i, text) in enumerate(texts):
img.put_text((100, (i + 1) * step), (10, 10, 10), 1, text)
__extract_coords_data
Extract the sights data to compute the hand scale factor.
If the index equal to -1, this function returns a list of None.
Parameters
- coords_tier: (sppasTier or None) The tier that contains the sights
- index: (int) The index of the annotation
Returns
- (list) A list of the coords for s0, s9, target
Example of returned list: [sppasFuzzyPoint: (368,780), sppasFuzzyPoint: (432,684), sppasFuzzyPoint: (540,573)]
View Source
@staticmethod
def __extract_coords_data(coords_tier, index) -> list:
"""Extract the sights data to compute the hand scale factor.
If the index equal to -1, this function returns a list of None.
:param coords_tier: (sppasTier or None) The tier that contains the sights
:param index: (int) The index of the annotation
:return: (list) A list of the coords for s0, s9, target
Example of returned list:
[sppasFuzzyPoint: (368,780), sppasFuzzyPoint: (432,684), sppasFuzzyPoint: (540,573)]
"""
sights = [None, None, None]
if coords_tier is not None and index != -1:
return coords_tier[index].get_labels()
return [None, None, None]
__extract_shape_data
Extract the shape data (shapes codes and scores) from a shape tier.
Parameters
- shape_tier: (sppasTier) The tier that contains our shape data
- index: (int) The index of the annotation
Returns
- (str, list) The shape text information and the list of shapes codes and scores
View Source
@staticmethod
def __extract_shape_data(shape_tier: sppasTier, index: int) -> tuple:
"""Extract the shape data (shapes codes and scores) from a shape tier.
:param shape_tier: (sppasTier) The tier that contains our shape data
:param index: (int) The index of the annotation
:return: (str, list) The shape text information and the list of shapes codes and scores
"""
shape_text = 'Shape: '
shapes = list()
for shape_label in shape_tier[index].get_labels():
for (tag, score) in shape_label:
shape_code = tag.get_typed_content()
if score is None:
score = 1.0
shape_text += shape_code
shapes.append((shape_code, score))
return (shape_text, shapes)
__extract_position_data
Extract the position data from the position tiers.
Parameters
- pos_tier: (sppasTier or None) The tier that contains the positions text information
- index: (int) The index of the annotation
Returns
- (str) The position text information and the position probability
View Source
@staticmethod
def __extract_position_data(pos_tier, index: int) -> tuple:
"""Extract the position data from the position tiers.
:param pos_tier: (sppasTier or None) The tier that contains the positions text information
:param index: (int) The index of the annotation
:return: (str) The position text information and the position probability
"""
pos_text = 'Pos: '
score = 1
if pos_tier is not None:
for pos_label in pos_tier[index].get_labels():
score = pos_label.get_score(pos_label.get_best())
for (tag, score) in pos_label:
pos_text += tag.get_typed_content()
return (pos_text, score)
__extract_vowels_data
Extract the vowels data from the vowels tier.
If the vowels tier is None or the index equal to -1 return an empty list by default.
Parameters
- vowels_tier: (sppasTier or None) The tier that contains our vowels data
- index: (int) The index of the annotation
Returns
- (list) A list of all vowels positions
View Source
@staticmethod
def __extract_vowels_data(vowels_tier, index: int) -> list:
"""Extract the vowels data from the vowels tier.
If the vowels tier is None or the index equal to -1 return an empty list by default.
:param vowels_tier: (sppasTier or None) The tier that contains our vowels data
:param index: (int) The index of the annotation
:return: (list) A list of all vowels positions
"""
vowels_positions = list()
if vowels_tier is not None and index != -1:
for vowel_label in vowels_tier[index].get_labels():
pos = vowel_label.get_best().get_typed_content()
vowels_positions.append(pos)
return vowels_positions
Overloads
__del__
View Source
def __del__(self):
self.close()
