Unsupervised_Metrics_CLD_Sy/code/utterances_cleaner.py

"""This module contains an implementation of a class that help /
    to clean orthographic or IPA transcripts of utterances. /
    Crucially, this class will clean utterances by removing or replacing /
    markers. See the file markers.json to see what kinds of markers are /
    accounted.
"""
import re
import string

class UtterancesCleaner :
    """
    This class will clean utterances from CHILDES,\
    by deleting words, patterns, ponctuation or replacing\
    or replacing them by other things.
    """
    def __init__(self, markers: dict) :
        self.delete_marker_pattern = '|'.join(markers["marker_to_delete"])
        self.word_contains_delete_pattern = '|'.join(markers["word_contains_delete"])
        self.poncts_to_delete_pattern = '|'.join(markers["poncts_to_delete"])
        self.delete_comments_pattern = r"(\(|\<|\*)(.+?)(\)|\>|\*)"
        self.replace_unk_pattern = r"xxx|yyy|www|[0-9]+|\*"
        self.pattern_letter = re.compile(r"(\s?)([^ ]*)\s\[x (\d+)\]")
        self.pattern_repetition = re.compile(r"(\s?)([^ ]*)\s\[x (\d+)\]")

    def replace_marker(self, utterance: str, pattern: str, replacement: str="∑") -> list:
        """
        Method that replace some markers by an other symbol

        Parameters
        ----------
        - utterance : str
            Utterance from which markers will be replaced
        - pattern : str
            Regex pattern containing markers to delete from the utterance
        - replacement :
            Symbol that will replace markers
        """
        return " ".join(re.sub(pattern, replacement, word) for word in utterance.split(" "))

    def delete_words(self, utterance: str) -> str:

        """
        Method that delete some words from a given utterance.

        Parameters
        ----------
        - utterance : str
            Utterance from which those words will be removed
        """
        return " ".join(word for word in utterance.split(" ") \
            if not re.match(self.word_contains_delete_pattern, word))

    def remove_ponctuations(self, utterance: str) -> str :
        """
        Remove ponctuations from a given utterance.

        Parameters
        ----------
        - utterance : str
            The utterance from which the punctuation will be removed.

        Returns
        -------
        str :
            The utterance without punctuations.
        """
        return utterance.translate(str.maketrans('', '', string.punctuation))

    def remove_brackets(self, utterance: str) -> str :
        """
        Remove brackets from a given utterance.

        Parameters
        ----------
        - utterance : str
            The utterance from which the brackets will be removed.

        Returns
        -------
        str :
            The utterance without brackets.
        """
        return re.sub(r"[\(\[].*?[\)\]]", '', utterance)

    def handle_repetitions(self, utterance: str) -> str:
        """
        This function will repeat n times some units from\
        a give utterance.

        Parameters
        ----------
        utterance: str
            Utterance from which some units will be repeated.
        """
        while True:
            matched = re.search(self.pattern_repetition, utterance)

            if not matched:
                break

            all_match = matched.group(0)
            separator = matched.group(1)
            word, repetitions = matched.group(2),matched.group(3)
            repeated_word = f"{separator}{' '.join([word] * int(repetitions))}"

            utterance = utterance.replace(all_match, repeated_word, 1)

        return utterance

    def remove_multiple_spaces(self, utterance: str) -> str :
        """
        Remove multiple spaces from a given utterance.

        Parameters
        ----------
        utterance: str
            Utterance from which multiple successive spaces\
            will be replaced.

        Returns
        -------
        - str
            Utterance without multiple successive spaces.
        """
        return re.sub(' +', ' ', utterance)

    def clean(self, utterance: str) -> str :

        """
        Method that clean utterances by deleting or replacing /
        markers.

        Parameters
        ----------
        - utterances : str
            Utterance to clean
        Returns
        -------
        - str
            Cleaned utterance
        """
        utterance = self.handle_repetitions(utterance)
        utterance = self.replace_marker(utterance, self.delete_marker_pattern, "")
        utterance = self.delete_words(utterance)
        utterance = self.replace_marker(utterance, self.poncts_to_delete_pattern, "")
        utterance = self.replace_marker(utterance, self.delete_comments_pattern, "")
        utterance = self.replace_marker(utterance, self.replace_unk_pattern, "")
        utterance = self.remove_brackets(utterance)
        utterance = self.remove_ponctuations(utterance)
        utterance = self.remove_multiple_spaces(utterance)
        utterance = utterance.strip()
        return utterance