Generic annotation computation

README.md

@@ -112,7 +112,7 @@ Note that the recording file names **should comply with the file naming conventi
 
 **(7)** Extract the acoustic annotations using the following command 
 ```bash
-python -u scripts/URUMETRICS-CODE/acoustic_annotations/compute_acoustic_annotations.py --path-vtc ./annotations/vtc/raw/VTC_FILE_FOR_WHICH_TO_DERIVE_ACOUSTIC_ANNOTATIONS_FOR.rttm --path-recordings ./recordings/raw/ --save-path ./annotations/acoustic/raw
+python -u scripts/URUMETRICS-CODE/compute_annotations/compute_acoustic_annotations.py --path-vtc ./annotations/vtc/raw/VTC_FILE_FOR_WHICH_TO_DERIVE_ACOUSTIC_ANNOTATIONS_FOR.rttm --path-recordings ./recordings/raw/ --save-path ./annotations/acoustic/raw
 ```
 
 This command will compute acoustic annotations (mean pitch, pitch range) for the VTC file passed as argument. The output file will have the same name as the input VTC file with the `rttm` extension replaced by `csv`. Of course, in the previous command replace `VTC_FILE_FOR_WHICH_TO_DERIVE_ACOUSTIC_ANNOTATIONS_FOR` by the name of the RTTM file which you want to compute acoustic annotations for.
@@ -133,7 +133,7 @@ This script can also take the additional (optional) parameter `--recording`. Whe
 **(9)** Compute the conversational annotations using the following command:
 
 ```bash
-python scripts/URUMETRICS-CODE/turn_annotations/compute_turn_annotations.py --save-path ./annotations/conversations/raw --save-name CONV_${today}
+python scripts/URUMETRICS-CODE/turn_annotations/compute_annotations.py --save-path ./annotations/conversations/raw --save-name CONV_${today}
 ```
 
 This command will only compute the conversational annotations for the newly imported VTC files.

acoustic_annotations/compute_acoustic_annotations.py

@@ -1,10 +1,10 @@
 #!usr/bin/env python
 # -*- coding: utf8 -*-
-
+#
 # -----------------------------------------------------------------------------
-#   File: compute_acoustic_annotations.py (as part of project URUMETRICS)
-#   Created: 01/06/2022 15:25
-#   Last Modified: 01/06/2022 15:25
+#   File: annotations_functions.py (as part of project URUMETRICS-CODE)
+#   Created: 08/09/2022 12:42
+#   Last Modified: 08/09/2022 12:42
 # -----------------------------------------------------------------------------
 #   Author: William N. Havard
 #           Postdoctoral Researcher
@@ -15,22 +15,34 @@
 #
 # ------------------------------------------------------------------------------
 #   Description: 
-#       • This files computes acoustic annotations for each segment identified
-#         by the VTC.
+#       • 
 # -----------------------------------------------------------------------------
 
-import logging
+import argparse
 import os
-from math import ceil, floor
 
 import pandas as pd
 
+from math import ceil, floor
+
 import utils_audio
-from utils import list_audio_files, read_vtc
+from conversational_settings import uru_conversations
 from utils_annotations import get_pitch
 from utils_audio import get_audio_slice, read_audio
 
-logger = logging.getLogger(__name__)
+def conversations_annotations(recording_filename, segments, project_path, parser_args=None):
+    return uru_conversations.get_interactional_sequences(segments).to_dataframe()
+
+def acoustic_annotations(recording_filename, segments, project_path, parser_args):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--target-sr', required=True, default=16_000, type=int,
+                        help='Audio files sampling rate.')
+
+    args = parser.parse_args(parser_args)
+    return _compute_file_acoustic_annotation(
+                audio_path=os.path.join(project_path, 'recordings', 'raw', recording_filename),
+                audio_segments=segments,
+                target_sr=args.target_sr)
 
 def _annotation_pitch(audio_segments, audio_time_series, sampling_rate):
     """
@@ -89,92 +101,4 @@ def _compute_file_acoustic_annotation(audio_path, audio_segments, target_sr):
                               'frame_offset'],
                      inplace=True)
 
-    return annotations
-
-
-def compute_acoustic_annotations(path_vtc, path_recordings, target_sr=16_000):
-    """
-    Compute the acoustic annotations for the recordings found in the VTC file
-    :param path_vtc: path to the VTC file to be read
-    :type path_vtc: str
-    :param path_recordings: path where the recordings are stored
-    :type path_recordings: str
-    :param target_sr: target sampling rate of the recordings
-    :type target_sr: int
-    :return: annotations
-    :rtype: pd.DataFrame
-    """
-    vtc_data = read_vtc(path_vtc, drop_na=True)
-    audio_file_list = list_audio_files(path_recordings)
-
-    annotations = []
-    vtc_audio_files = vtc_data.groupby(by='file') # Iterate over VTC annotations grouped by file
-    for audio_file_name, audio_segments in vtc_audio_files:
-        if not audio_file_list.get(audio_file_name, False): continue
-        file_anns = _compute_file_acoustic_annotation(audio_file_list[audio_file_name], audio_segments, target_sr)
-        annotations.append(file_anns)
-
-    df_annotations = pd.concat(annotations, axis=0)
-    return df_annotations
-
-
-def save_annotations(save_path, save_name, annotations):
-    """
-    Save the computed annotations
-    :param save_path: path where to save the annotations
-    :type save_path: str
-    :param save_name: name of the file
-    :type save_name: str
-    :param annotations: annotations to be saved
-    :type annotations: pd.DataFrame
-    :return: None
-    :rtype: None
-    """
-    full_save_path = os.path.join(save_path, '{}.csv'.format(save_name))
-    if os.path.exists(full_save_path):
-        raise FileExistsError('File {} already exists!'.format(full_save_path))
-
-    annotations.to_csv(full_save_path, index=False)
-    logger.info('Saved to {}.'.format(full_save_path))
-
-
-def main(path_vtc, path_recordings, save_path, target_sr):
-    assert os.path.exists(os.path.abspath(save_path)), IOError('Path {} does not exist!'.format(save_path))
-
-    annotations = compute_acoustic_annotations(path_vtc, path_recordings, target_sr)
-    save_name = os.path.splitext(os.path.split(path_vtc)[-1])[0]
-    save_annotations(save_path, save_name, annotations)
-
-
-def _parse_args(argv):
-    import argparse
-
-    parser = argparse.ArgumentParser(description='Compute acoustic annotations.')
-    parser.add_argument('--path-vtc', required=True,
-                        help='Path to the VTC files for which acoustic annotations be computed.')
-    parser.add_argument('--path-recordings', required=True,
-                        help='Path to the recordings corresponding to the recording filenames contained '
-                             'in the VTC file.')
-    parser.add_argument('--save-path', required=True,
-                        help='Path were the annotations should be saved.')
-    parser.add_argument('--target-sr', required=False, default=16_000, type=int,
-                        help='Audio files sampling rate.')
-    args = parser.parse_args(argv)
-
-    return vars(args)
-
-
-if __name__ == '__main__':
-    import sys
-
-    pgrm_name, argv = sys.argv[0], sys.argv[1:]
-    args = _parse_args(argv)
-
-    logging.basicConfig(level=logging.INFO)
-
-    try:
-        main(**args)
-        sys.exit(0)
-    except Exception as e:
-        logger.exception(e)
-        sys.exit(1)
+    return annotations

turn_annotations/compute_turn_annotations.py

@@ -1,7 +1,7 @@
 #!usr/bin/env python
 # -*- coding: utf8 -*-
 # -----------------------------------------------------------------------------
-#   File: compute_turn_annotations.py (as part of project URUMETRICS-CODE)
+#   File: compute_annotations.py (as part of project URUMETRICS-CODE)
 #   Created: 03/08/2022 17:32
 #   Last Modified: 03/08/2022 17:32
 # -----------------------------------------------------------------------------
@@ -19,24 +19,21 @@
 
 import logging
 import os
+from functools import partial
 
 import pandas as pd
 
 from ChildProject.annotations import AnnotationManager
 from ChildProject.projects import ChildProject
 
-from conversational_settings import uru_conversations
+import annotations_functions
 
 logger = logging.getLogger(__name__)
 
-def compute_turn_annotations(data_path):
-    """
-    Computes conversational annotations for the ChildProject in directory data_path
-    :param data_path: path to ChildProject dataset
-    :type data_path: str
-    :return: annotations
-    :rtype: pd.DataFrame
-    """
+def _annotation_function_wrapper(func, parser_args, **kwargs):
+    return partial(func,parser_args=parser_args, **kwargs)
+
+def get_available_segments(data_path, set_name):
     project = ChildProject(data_path)
     am = AnnotationManager(project)
     am.read()
@@ -48,29 +45,43 @@ def compute_turn_annotations(data_path):
 
     # Get already existing conversation annotations and only compute annotations for the files that do not already
     # have conversational annotations
-    if 'conversations' in set(am.annotations['set']):
-        conversations_anns = am.annotations[am.annotations['set'] == 'conversations']
+    if set_name in set(am.annotations['set']):
+        conversations_anns = am.annotations[am.annotations['set'] == set_name]
         available_vtc_anns = available_vtc_anns[
             ~available_vtc_anns['recording_filename'].isin(conversations_anns['recording_filename'])]
 
     # Get the segments that are left
     data = am.get_segments(available_vtc_anns)
+    return data
+
+def _compute_annotations(data_path, annotation_function):
+    """
+    Computes conversational annotations for the ChildProject in directory data_path
+    :param data_path: path to ChildProject dataset
+    :type data_path: str
+    :return: annotations
+    :rtype: pd.DataFrame
+    """
+
+    data = get_available_segments(data_path, set_name='conversations')
     if not len(data):
         return pd.DataFrame()
 
     data = data[~data['speaker_type'].isnull()]
 
-    interactional_sequences = []
+    annotations = []
     data_grouped = data.groupby('recording_filename')
+
     for data_grouped_name, data_grouped_line in data_grouped:
-        df_interactional_sequences = uru_conversations.get_interactional_sequences(data_grouped_line).to_dataframe()
-        interactional_sequences.append(df_interactional_sequences)
+        df_annotations = annotation_function(recording_filename=data_grouped_name, segments=data_grouped_line,
+                                             project_path = data_path)
+        annotations.append(df_annotations)
 
-    output = pd.concat(interactional_sequences)
+    output = pd.concat(annotations, axis=0)
     return output
 
 
-def save_annotations(save_path, annotations):
+def save_annotations(save_path, annotations, annotation_type):
     """
     Save the computed annotations
     :param save_path: path where to save the annotations
@@ -85,9 +96,9 @@ def save_annotations(save_path, annotations):
 
     annotations_grouped = annotations.groupby('raw_filename')
     for annotation_group_name, annotation_group_data in annotations_grouped:
-        output_filename = 'CONV_{}'.format(annotation_group_name.replace('.rttm', ''))
+        output_filename = '{}'.format(annotation_group_name.replace('.rttm', '_{}.csv'.format(annotation_type.upper())))
 
-        full_save_path = os.path.join(save_path, '{}.csv'.format(output_filename))
+        full_save_path = os.path.join(save_path, output_filename)
         if os.path.exists(full_save_path):
             logger.warning('File {} already exists! If you want to recompute annotations for this file, '
                            'please delete it first!'.format(full_save_path))
@@ -112,48 +123,58 @@ def save_annotations(save_path, annotations):
         logger.info('Saved to {}.'.format(full_save_path))
 
 
-def main(save_path):
-    data_path = '.'
+def main(annotation_type, save_path, unknown_args):
+    data_path = ''#
 
     # Check if running the script from the root of the data set
     expected_annotation_path = os.path.join(data_path, 'annotations')
+    expected_recordings_path = os.path.join(data_path, 'recordings')
 
-    assert os.path.exists(expected_annotation_path), \
-        ValueError('Expected annotation ({}) path not found. Are you sure to be running this '
-                   'command from the root of the data set?'.format(expected_annotation_path))
+    assert os.path.exists(expected_annotation_path) and os.path.exists(expected_recordings_path), \
+        ValueError('Expected annotation ({}) or recording path ({}) not found. Are you sure to be running this '
+                   'command from the root of the data set?'.format(expected_annotation_path, expected_recordings_path))
 
     assert os.path.exists(os.path.abspath(save_path)), IOError('Path {} does not exist!'.format(save_path))
 
-    annotations = compute_turn_annotations(data_path)
+    assert hasattr(annotations_functions, '{}_annotations'.format(annotation_type.lower())), \
+        ValueError('Annotation function {}_annotations not found.'.format(annotation_type.lower()))
+
+    annotation_function = getattr(annotations_functions, '{}_annotations'.format(annotation_type.lower()))
+    annotations = _compute_annotations(data_path,
+                                      annotation_function=_annotation_function_wrapper(
+                                          func=annotation_function,
+                                          parser_args=unknown_args))
 
     if not len(annotations):
         logger.warning('Apparently nothing needs to be computed!')
         return
 
-    save_annotations(save_path, annotations)
+    save_annotations(save_path, annotations, annotation_type)
 
 
 def _parse_args(argv):
     import argparse
 
     parser = argparse.ArgumentParser(description='Compute acoustic annotations.')
+    parser.add_argument('--annotation-type', required=True,
+                        help='Which type of annotations should be computed.')
     parser.add_argument('--save-path', required=True,
                         help='Path were the annotations should be saved.')
-    args = parser.parse_args(argv)
+    args, unknown_args = parser.parse_known_args(argv)
 
-    return vars(args)
+    return vars(args), unknown_args
 
 
 if __name__ == '__main__':
     import sys
 
     pgrm_name, argv = sys.argv[0], sys.argv[1:]
-    args = _parse_args(argv)
+    args, unknown_args = _parse_args(argv)
 
     logging.basicConfig(level=logging.INFO)
 
     try:
-        main(**args)
+        main(unknown_args=unknown_args, **args)
         sys.exit(0)
     except Exception as e:
         logger.exception(e)

example/example_pipeline.sh

@@ -1,5 +1,3 @@
-set -e
-
 example_dir=/scratch2/whavard/CODE/LAAC/URUMETRICS/dat/data_set/recordings/original/renamed;
 dataset="URUMETRICS-TEST"
 
@@ -49,15 +47,15 @@ for dir_name in $example_dir/*; do
     python -u scripts/URUMETRICS-CODE/import_data/import_annotations.py --annotation-type VTC --annotation-file VTC_${today}.rttm
     python -u scripts/URUMETRICS-CODE/import_data/import_annotations.py --annotation-type VCM --annotation-file VTC_${today}.vcm
     python -u scripts/URUMETRICS-CODE/import_data/import_annotations.py --annotation-type ALICE --annotation-file ALICE_${today}.txt
-    python -u scripts/URUMETRICS-CODE/import_data/import_annotations.py --annotation-type ACOUSTIC --annotation-file VTC_${today}.csv
+    python -u scripts/URUMETRICS-CODE/import_data/import_annotations.py --annotation-type ACOUSTIC --annotation-file VTC_${today}.csv --recordings-from-annotation-file VTC_${today}.rttm
 
     # Compute turn annotations
     echo -e "\tComputing CONVERSATIONS annotations"
-    python scripts/URUMETRICS-CODE/turn_annotations/compute_turn_annotations.py --save-path ./annotations/conversations/raw --save-name CONV_${today}
+    python scripts/URUMETRICS-CODE/turn_annotations/compute_turn_annotations.py --save-path ./annotations/conversations/raw
 
     # Import turn annotations
     echo -e "\tImporting CONVERSATIONS annotations"
-    python -u scripts/URUMETRICS-CODE/import_data/import_annotations.py --annotation-type CONVERSATIONS --annotation-file CONV_${today}.csv
+    python -u scripts/URUMETRICS-CODE/import_data/import_annotations.py --annotation-type CONVERSATIONS --annotation-file CONV_VTC_${today}.csv --recordings-from-annotation-file VTC_${today}.rttm
 
     # Compute metrics
     echo -e "\tComputing metrics"

turn_annotations/__init__.py

@@ -1,19 +0,0 @@
-#!usr/bin/env python
-# -*- coding: utf8 -*-
-
-# -----------------------------------------------------------------------------
-#   File: __init__.py (as part of project URUMETRICS-CODE)
-#   Created: 03/08/2022 17:31
-#   Last Modified: 03/08/2022 17:31
-# -----------------------------------------------------------------------------
-#   Author: William N. Havard
-#           Postdoctoral Researcher
-#
-#   Mail  : william.havard@ens.fr / william.havard@gmail.com
-#  
-#   Institution: ENS / Laboratoire de Sciences Cognitives et Psycholinguistique
-#
-# ------------------------------------------------------------------------------
-#   Description: 
-#       • 
-# -----------------------------------------------------------------------------