From 96183f4034be9f5d2ba7df0ee509881b46c0dc25 Mon Sep 17 00:00:00 2001 From: JorisCos Date: Fri, 15 Jan 2021 10:13:04 +0100 Subject: [PATCH 1/9] Initial commit --- asteroid/data/__init__.py | 2 + asteroid/data/chime4_dataset.py | 70 ++++++++++ egs/chime4/ConvTasNet/eval.py | 127 ++++++++++++++++++ .../ConvTasNet/local/create_metadata.py | 106 +++++++++++++++ egs/chime4/ConvTasNet/run.sh | 75 +++++++++++ egs/chime4/ConvTasNet/utils | 1 + egs/chime4/README.md | 38 ++++++ 7 files changed, 419 insertions(+) create mode 100644 asteroid/data/chime4_dataset.py create mode 100644 egs/chime4/ConvTasNet/eval.py create mode 100644 egs/chime4/ConvTasNet/local/create_metadata.py create mode 100644 egs/chime4/ConvTasNet/run.sh create mode 120000 egs/chime4/ConvTasNet/utils create mode 100644 egs/chime4/README.md diff --git a/asteroid/data/__init__.py b/asteroid/data/__init__.py index 4698dd0db..28d3b5d10 100644 --- a/asteroid/data/__init__.py +++ b/asteroid/data/__init__.py @@ -9,6 +9,7 @@ from .kinect_wsj import KinectWsjMixDataset from .fuss_dataset import FUSSDataset from .dampvsep_dataset import DAMPVSEPSinglesDataset +from .chime4_dataset import CHiME4 __all__ = [ "AVSpeechDataset", @@ -22,4 +23,5 @@ "KinectWsjMixDataset", "FUSSDataset", "DAMPVSEPSinglesDataset", + "CHiME4", ] diff --git a/asteroid/data/chime4_dataset.py b/asteroid/data/chime4_dataset.py new file mode 100644 index 000000000..8a1662e9e --- /dev/null +++ b/asteroid/data/chime4_dataset.py @@ -0,0 +1,70 @@ +import pandas as pd +import soundfile as sf +import torch +from torch.utils.data import Dataset, DataLoader +import random as random +import os + + +class CHiME4(Dataset): + """Dataset class for CHiME4 source separation tasks. Only supports 'real' + data + + Args: + csv_dir (str): The path to the metadata file. + sample_rate (int) : The sample rate of the sources and mixtures. + segment (int) : The desired sources and mixtures length in s. + + References + Emmanuel Vincent, Shinji Watanabe, Aditya Arie Nugraha, Jon Barker, and Ricard Marxer + An analysis of environment, microphone and data simulation mismatches in robust speech recognition + Computer Speech and Language, 2017. + """ + + dataset_name = "CHiME4" + + def __init__(self, csv_dir, sample_rate=16000, segment=3): + self.csv_dir = csv_dir + # Get the csv corresponding to origin + self.segment = segment + self.sample_rate = sample_rate + self.csv_path = [f for f in os.listdir(csv_dir)] + # Open csv file and concatenate them + self.df = pd.read_csv(self.csv_path) + # Get rid of the utterances too short + if self.segment is not None: + max_len = len(self.df) + self.seg_len = int(self.segment * self.sample_rate) + # Ignore the file shorter than the desired_length + self.df = self.df[self.df["duration"] >= self.seg_len] + print( + f"Drop {max_len - len(self.df)} utterances from {max_len} " + f"(shorter than {segment} seconds)" + ) + else: + self.seg_len = None + + def __len__(self): + return len(self.df) + + def __getitem__(self, idx): + # Get the row in dataframe + row = self.df.iloc[idx] + # Get mixture path + self.mixture_path = row["mixture_path"] + # If there is a seg start point is set randomly + if self.seg_len is not None: + start = random.randint(0, row["length"] - self.seg_len) + stop = start + self.seg_len + else: + start = 0 + stop = None + + # Read the mixture + mixture, _ = sf.read(self.mixture_path, dtype="float32", + start=start, stop=stop) + # Convert to torch tensor + mixture = torch.from_numpy(mixture) + + return mixture + diff --git a/egs/chime4/ConvTasNet/eval.py b/egs/chime4/ConvTasNet/eval.py new file mode 100644 index 000000000..073a22aee --- /dev/null +++ b/egs/chime4/ConvTasNet/eval.py @@ -0,0 +1,127 @@ +import os +import random +import soundfile as sf +import torch +import yaml +import argparse +import numpy as np +import pandas as pd +from tqdm import tqdm +from pathlib import Path + +from asteroid.data.chime4_dataset import CHiME4 +from asteroid import ConvTasNet +from asteroid.utils import tensors_to_device +from asteroid.metrics import WERTracker, MockWERTracker + + +parser = argparse.ArgumentParser() +parser.add_argument( + "--test_dir", type=str, required=True, help="Test directory including the csv files" +) + +parser.add_argument( + "--use_gpu", type=int, default=0, help="Whether to use the GPU for model execution" +) +parser.add_argument("--exp_dir", default="exp/tmp", help="Experiment root") +parser.add_argument( + "--n_save_ex", type=int, default=10, help="Number of audio examples to save, -1 means all" +) +parser.add_argument( + "--compute_wer", type=int, default=0, help="Compute WER using ESPNet's pretrained model" +) + +ASR_MODEL_PATH = ( + "kamo-naoyuki/chime4_asr_train_asr_transformer3_raw_en_char_sp_valid.acc.ave" +) + + +def update_compute_metrics(compute_wer, metric_list): + if not compute_wer: + return metric_list + try: + from espnet2.bin.asr_inference import Speech2Text + from espnet_model_zoo.downloader import ModelDownloader + except ModuleNotFoundError: + import warnings + + warnings.warn("Couldn't find espnet installation. Continuing without.") + return metric_list + return metric_list + ["wer"] + + +def main(conf): + compute_metrics = update_compute_metrics(conf["compute_wer"]) + anno_df = pd.read_csv(Path(conf["test_dir"]).parent.parent.parent / "test_annotations.csv") + wer_tracker = ( + MockWERTracker() if not conf["compute_wer"] else WERTracker(ASR_MODEL_PATH, anno_df) + ) + model_path = os.path.join(conf["exp_dir"], "best_model.pth") + model = ConvTasNet.from_pretrained(model_path) + # Handle device placement + if conf["use_gpu"]: + model.cuda() + model_device = next(model.parameters()).device + test_set = CHiME4( + csv_dir=conf["test_dir"], + sample_rate=conf["sample_rate"], + segment=None, + ) # Uses all segment length + # Used to reorder sources only + + # Randomly choose the indexes of sentences to save. + eval_save_dir = os.path.join(conf["exp_dir"], conf["out_dir"]) + ex_save_dir = os.path.join(eval_save_dir, "examples/") + if conf["n_save_ex"] == -1: + conf["n_save_ex"] = len(test_set) + save_idx = random.sample(range(len(test_set)), conf["n_save_ex"]) + series_list = [] + torch.no_grad().__enter__() + for idx in tqdm(range(len(test_set))): + # Forward the network on the mixture. + mix = test_set[idx] + mix = mix.to(model_device) + est_sources = model(mix.unsqueeze(0)) + mix_np = mix.cpu().data.numpy() + est_sources_np = est_sources.squeeze(0).cpu().data.numpy() + + # Save some examples in a folder. Wav files and metrics as text. + if idx in save_idx: + local_save_dir = os.path.join(ex_save_dir, "ex_{}/".format(idx)) + os.makedirs(local_save_dir, exist_ok=True) + sf.write(local_save_dir + "mixture.wav", mix_np, conf["sample_rate"]) + # Loop over the sources and estimates + for src_idx, est_src in enumerate(est_sources_np): + est_src *= np.max(np.abs(mix_np)) / np.max(np.abs(est_src)) + sf.write( + local_save_dir + "s{}_estimate.wav".format(src_idx), + est_src, + conf["sample_rate"], + ) + + if conf["compute_wer"]: + print("\nWER report") + wer_card = wer_tracker.final_report_as_markdown() + print(wer_card) + # Save the report + with open(os.path.join(eval_save_dir, "final_wer.md"), "w") as f: + f.write(wer_card) + + +if __name__ == "__main__": + args = parser.parse_args() + arg_dic = dict(vars(args)) + # Load training config + conf_path = os.path.join(args.exp_dir, "conf.yml") + with open(conf_path) as f: + train_conf = yaml.safe_load(f) + arg_dic["sample_rate"] = train_conf["data"]["sample_rate"] + arg_dic["train_conf"] = train_conf + + if args.task != arg_dic["train_conf"]["data"]["task"]: + print( + "Warning : the task used to test is different than " + "the one from training, be sure this is what you want." + ) + + main(arg_dic) diff --git a/egs/chime4/ConvTasNet/local/create_metadata.py b/egs/chime4/ConvTasNet/local/create_metadata.py new file mode 100644 index 000000000..76f8e7e1c --- /dev/null +++ b/egs/chime4/ConvTasNet/local/create_metadata.py @@ -0,0 +1,106 @@ +import os +import shutil +import argparse +from glob import glob +import pandas as pd +import numpy as np + +# Command line arguments +parser = argparse.ArgumentParser() +parser.add_argument( + "--chime3_dir", type=str, default=None, + help="Path to CHiME3 root directory" +) + +# Set seed for random generation +SEED = 4 +np.random.seed(SEED) + + +def main(args): + chime3_dir = args.chime3_dir + create_local_metadata(chime3_dir) + + +def create_local_metadata(chime3_dir): + # Get CHiME-3 annotation files + c3_annot_files = [f for f in glob( + os.path.join(chime3_dir, "data", "annotations", "*real*.json"))] + # Get CHiME-4 annotation files + c4_annot_files = [f for f in glob( + os.path.join(chime3_dir, "data", "annotations", "*real*.list"))] + print(c3_annot_files) + print(c4_annot_files) + for c3_annot_file_path in c3_annot_files: + # Read CHiME-3 annotation file + c3_annot_file = pd.read_json(c3_annot_file_path) + # subsets : "tr" "dt" "et" origin "real" or "simu" + subset, origin = os.path.split(c3_annot_file_path)[1].replace('.json', + '').split( + '_') + # Look for associated CHiME-4 file + if c3_annot_file_path.replace('.json', + '_1ch_track.list') in c4_annot_files: + # Read CHiME-4 annotation file + c4_annot_file = pd.read_csv( + c3_annot_file_path.replace('.json', '_1ch_track.list'), + header=None, names=['path']) + else: + c4_annot_file = None + df = create_dataframe(chime3_dir, c3_annot_file, c4_annot_file, subset, origin) + write_dataframe(df, subset, origin) + + +def create_dataframe(chime3_dir, c3_annot_file, c4_annot_file, subset, origin): + + # Empty list for DataFrame creation + row_list = [] + for row in c3_annot_file.itertuples(): + speaker = row.speaker + ID = row.wsj_name + env = row.environment + # if we are not dealing with tr subset + if 'tr' not in subset: + mixture_path = c4_annot_file[ + c4_annot_file['path'].str.contains(ID + '_' + env)].values[ + 0][0] + mixture_path = os.path.join(chime3_dir, + "data/audio/16kHz/isolated/", + mixture_path) + + # if we are dealing with the tr subset + else: + channel = np.random.randint(1, 7) + mixture_path = os.path.join(chime3_dir, + "data/audio/16kHz/isolated/", + subset + '_' + env.lower() + '_' + origin, + speaker + '_' + ID + '_' + f'.CH{channel}' + '.wav') + + duration = row.end - row.start + temp_dict = {'ID': ID, 'subset': subset, 'origin': origin, + 'env': env, + 'mixture_ path': mixture_path, + 'duration': duration} + row_list.append(temp_dict) + + df = pd.DataFrame(row_list) + return df + + +def write_dataframe(df, subset, origin): + if 'et' in subset: + subdir = 'test' + elif 'dt' in subset: + subdir = 'val' + else: + subdir = 'train' + save_dir = os.path.join('data', subdir) + os.makedirs(save_dir, exist_ok=True) + save_path = os.path.join(save_dir, origin + '_1_ch_track.csv') + df.to_csv(save_path, index=False) + + +if __name__ == "__main__": + args = parser.parse_args() + main(args) diff --git a/egs/chime4/ConvTasNet/run.sh b/egs/chime4/ConvTasNet/run.sh new file mode 100644 index 000000000..31c9e62f4 --- /dev/null +++ b/egs/chime4/ConvTasNet/run.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +# Exit on error +set -e +set -o pipefail + +# If you haven't generated LibriMix start from stage 0 +# Main storage directory. You'll need disk space to store LibriSpeech, WHAM noises +# and LibriMix. This is about 500 Gb +storage_dir= + +# After running the recipe a first time, you can run it from stage 3 directly to train new models. + +# Path to the python you'll use for the experiment. Defaults to the current python +# You can run ./utils/prepare_python_env.sh to create a suitable python environment, paste the output here. +python_path=python + +# Example usage +# ./run.sh --stage 3 --tag my_tag --task sep_noisy --id 0,1 + +# General +stage=0 # Controls from which stage to start +tag="" # Controls the directory name associated to the experiment +# You can ask for several GPUs using id (passed to CUDA_VISIBLE_DEVICES) + +eval_use_gpu=1 +# Need to --compute_wer 1 --eval_mode max to be sure the user knows all the metrics +# are for the all mode. +compute_wer=0 +eval_mode= + +. utils/parse_options.sh + + +if [[ $stage -le 0 ]]; then + echo "Stage 0: Generating CHiME-4 dataset" + . local/create_metadata.py --chime3_dir $storage_dir/CHiME3 +fi + +# Generate a random ID for the run if no tag is specified +uuid=$($python_path -c 'import uuid, sys; print(str(uuid.uuid4())[:8])') +if [[ -z ${tag} ]]; then + tag=${uuid} +fi + +expdir=exp/train_convtasnet_${tag} +mkdir -p $expdir && echo $uuid >> $expdir/run_uuid.txt +echo "Results from the following experiment will be stored in $expdir" + +if [[ $stage -le 2 ]]; then + echo "Stage 2 : Evaluation" + + if [[ $compute_wer -eq 1 ]]; then + if [[ $eval_mode != "max" ]]; then + echo "Cannot compute WER without max mode. Start again with --stage 2 --compute_wer 1 --eval_mode max" + exit 1 + fi + + # Install espnet if not instaled + if ! python -c "import espnet" &> /dev/null; then + echo 'This recipe requires espnet. Installing requirements.' + $python_path -m pip install espnet_model_zoo + $python_path -m pip install jiwer + $python_path -m pip install tabulate + fi + fi + + $python_path eval.py \ + --exp_dir $expdir \ + --test_dir $test_dir \ + --use_gpu $eval_use_gpu \ + --compute_wer $compute_wer | tee logs/eval_${tag}.log + + cp logs/eval_${tag}.log $expdir/eval.log +fi diff --git a/egs/chime4/ConvTasNet/utils b/egs/chime4/ConvTasNet/utils new file mode 120000 index 000000000..00cd3a3b9 --- /dev/null +++ b/egs/chime4/ConvTasNet/utils @@ -0,0 +1 @@ +../../wham/ConvTasNet/utils/ \ No newline at end of file diff --git a/egs/chime4/README.md b/egs/chime4/README.md new file mode 100644 index 000000000..1754e2c0d --- /dev/null +++ b/egs/chime4/README.md @@ -0,0 +1,38 @@ +### The CHiME-4 dataset + +The CHiME-4 dataset is part of the 4th CHiME speech separation and recognition challenge. + +It was released in 2016 and revisits the datasets originally recorded for CHiME-3. + +All data and information are available [here](http://spandh.dcs.shef.ac.uk/chime_challenge/CHiME4/index.html). + +For now, this recipe only deals with the `real_1_ch_track` part of the dataset. +As the channel to use for the training set wasn't defined by +the challenge's rules, we will set it randomly. + +NOTE : +This dataset uses real noisy data. This means the clean speech from the noisy +utterances is not available. This makes it not suitable for the usual training +procedure. + + + +**References** +~~~BibTeX +@article{vincent:hal-01399180, + TITLE = {{An analysis of environment, microphone and data simulation mismatches in robust speech recognition}}, + AUTHOR = {Vincent, Emmanuel and Watanabe, Shinji and Nugraha, Aditya Arie and Barker, Jon and Marxer, Ricard}, + URL = {https://hal.inria.fr/hal-01399180}, + JOURNAL = {{Computer Speech and Language}}, + PUBLISHER = {{Elsevier}}, + VOLUME = {46}, + PAGES = {535-557}, + YEAR = {2017}, + MONTH = Jul, + DOI = {10.1016/j.csl.2016.11.005}, + KEYWORDS = {speech enhancement ; Robust ASR ; train/test mismatch ; microphone array}, + PDF = {https://hal.inria.fr/hal-01399180/file/vincent_CSL16.pdf}, + HAL_ID = {hal-01399180}, + HAL_VERSION = {v1}, +} +~~~ \ No newline at end of file From 13704af0d255bd640c4d1230596bf5972135d810 Mon Sep 17 00:00:00 2001 From: JorisCos Date: Mon, 18 Jan 2021 12:12:05 +0100 Subject: [PATCH 2/9] fix --- asteroid/data/chime4_dataset.py | 14 ++-- egs/chime4/ConvTasNet/eval.py | 79 ++++++++++++++----- .../ConvTasNet/local/create_metadata.py | 23 +++--- egs/chime4/ConvTasNet/run.sh | 28 +++---- 4 files changed, 91 insertions(+), 53 deletions(-) diff --git a/asteroid/data/chime4_dataset.py b/asteroid/data/chime4_dataset.py index 8a1662e9e..d3ba28d98 100644 --- a/asteroid/data/chime4_dataset.py +++ b/asteroid/data/chime4_dataset.py @@ -23,14 +23,15 @@ class CHiME4(Dataset): dataset_name = "CHiME4" - def __init__(self, csv_dir, sample_rate=16000, segment=3): + def __init__(self, csv_dir, sample_rate=16000, segment=3, return_id=False): self.csv_dir = csv_dir # Get the csv corresponding to origin self.segment = segment self.sample_rate = sample_rate - self.csv_path = [f for f in os.listdir(csv_dir)] + self.return_id = return_id + self.csv_path = [f for f in os.listdir(csv_dir) if 'annotations' not in f][0] # Open csv file and concatenate them - self.df = pd.read_csv(self.csv_path) + self.df = pd.read_csv(os.path.join(csv_dir,self.csv_path)) # Get rid of the utterances too short if self.segment is not None: max_len = len(self.df) @@ -65,6 +66,9 @@ def __getitem__(self, idx): start=start, stop=stop) # Convert to torch tensor mixture = torch.from_numpy(mixture) - - return mixture + fake_source = torch.vstack([mixture]) + if self.return_id: + id1 = row.ID + return mixture, fake_source, [id1] + return mixture, fake_source diff --git a/egs/chime4/ConvTasNet/eval.py b/egs/chime4/ConvTasNet/eval.py index 073a22aee..bf7e0701f 100644 --- a/egs/chime4/ConvTasNet/eval.py +++ b/egs/chime4/ConvTasNet/eval.py @@ -3,18 +3,19 @@ import soundfile as sf import torch import yaml +import json import argparse import numpy as np import pandas as pd from tqdm import tqdm -from pathlib import Path +from pprint import pprint from asteroid.data.chime4_dataset import CHiME4 from asteroid import ConvTasNet +from asteroid.models import save_publishable from asteroid.utils import tensors_to_device from asteroid.metrics import WERTracker, MockWERTracker - parser = argparse.ArgumentParser() parser.add_argument( "--test_dir", type=str, required=True, help="Test directory including the csv files" @@ -31,8 +32,8 @@ "--compute_wer", type=int, default=0, help="Compute WER using ESPNet's pretrained model" ) -ASR_MODEL_PATH = ( - "kamo-naoyuki/chime4_asr_train_asr_transformer3_raw_en_char_sp_valid.acc.ave" +COMPUTE_METRICS = [] +ASR_MODEL_PATH = ("kamo-naoyuki/wsj" ) @@ -51,8 +52,9 @@ def update_compute_metrics(compute_wer, metric_list): def main(conf): - compute_metrics = update_compute_metrics(conf["compute_wer"]) - anno_df = pd.read_csv(Path(conf["test_dir"]).parent.parent.parent / "test_annotations.csv") + compute_metrics = update_compute_metrics(conf["compute_wer"], COMPUTE_METRICS) + annot_path = [f for f in os.listdir(conf["test_dir"]) if 'annotations' in f][0] + anno_df = pd.read_csv(os.path.join(conf['test_dir'],annot_path)) wer_tracker = ( MockWERTracker() if not conf["compute_wer"] else WERTracker(ASR_MODEL_PATH, anno_df) ) @@ -66,11 +68,12 @@ def main(conf): csv_dir=conf["test_dir"], sample_rate=conf["sample_rate"], segment=None, + return_id=True, ) # Uses all segment length # Used to reorder sources only # Randomly choose the indexes of sentences to save. - eval_save_dir = os.path.join(conf["exp_dir"], conf["out_dir"]) + eval_save_dir = os.path.join(conf["exp_dir"], 'chime4') ex_save_dir = os.path.join(eval_save_dir, "examples/") if conf["n_save_ex"] == -1: conf["n_save_ex"] = len(test_set) @@ -79,18 +82,36 @@ def main(conf): torch.no_grad().__enter__() for idx in tqdm(range(len(test_set))): # Forward the network on the mixture. - mix = test_set[idx] - mix = mix.to(model_device) + mix, sources, ids = test_set[idx] + mix, sources = tensors_to_device([mix, sources], device=model_device) est_sources = model(mix.unsqueeze(0)) mix_np = mix.cpu().data.numpy() + sources_np = sources.cpu().data.numpy() est_sources_np = est_sources.squeeze(0).cpu().data.numpy() + # For each utterance, we get a dictionary with the mixture path, + # the input and output metrics + utt_metrics = {"mix_path": test_set.mixture_path} + utt_metrics.update( + **wer_tracker( + mix=mix_np, + clean=sources_np, + estimate=est_sources_np, + wav_id=ids, + sample_rate=conf["sample_rate"], + ) + ) + series_list.append(pd.Series(utt_metrics)) # Save some examples in a folder. Wav files and metrics as text. if idx in save_idx: local_save_dir = os.path.join(ex_save_dir, "ex_{}/".format(idx)) os.makedirs(local_save_dir, exist_ok=True) - sf.write(local_save_dir + "mixture.wav", mix_np, conf["sample_rate"]) + sf.write(local_save_dir + "mixture.wav", mix_np, + conf["sample_rate"]) # Loop over the sources and estimates + for src_idx, src in enumerate(sources_np): + sf.write(local_save_dir + "s{}.wav".format(src_idx), src, + conf["sample_rate"]) for src_idx, est_src in enumerate(est_sources_np): est_src *= np.max(np.abs(mix_np)) / np.max(np.abs(est_src)) sf.write( @@ -98,7 +119,24 @@ def main(conf): est_src, conf["sample_rate"], ) - + # Write local metrics to the example folder. + with open(local_save_dir + "metrics.json", "w") as f: + json.dump(utt_metrics, f, indent=0) + + # Save all metrics to the experiment folder. + all_metrics_df = pd.DataFrame(series_list) + all_metrics_df.to_csv(os.path.join(eval_save_dir, "all_metrics.csv")) + + # Print and save summary metrics + final_results = {} + for metric_name in compute_metrics: + input_metric_name = "input_" + metric_name + ldf = all_metrics_df[metric_name] - all_metrics_df[input_metric_name] + final_results[metric_name] = all_metrics_df[metric_name].mean() + final_results[metric_name + "_imp"] = ldf.mean() + + print("Overall metrics :") + pprint(final_results) if conf["compute_wer"]: print("\nWER report") wer_card = wer_tracker.final_report_as_markdown() @@ -107,6 +145,18 @@ def main(conf): with open(os.path.join(eval_save_dir, "final_wer.md"), "w") as f: f.write(wer_card) + with open(os.path.join(eval_save_dir, "final_metrics.json"), "w") as f: + json.dump(final_results, f, indent=0) + + model_dict = torch.load(model_path, map_location="cpu") + os.makedirs(os.path.join(conf["exp_dir"], "publish_dir"), exist_ok=True) + publishable = save_publishable( + os.path.join(conf["exp_dir"], "publish_dir"), + model_dict, + metrics=final_results, + train_conf=train_conf, + ) + if __name__ == "__main__": args = parser.parse_args() @@ -117,11 +167,4 @@ def main(conf): train_conf = yaml.safe_load(f) arg_dic["sample_rate"] = train_conf["data"]["sample_rate"] arg_dic["train_conf"] = train_conf - - if args.task != arg_dic["train_conf"]["data"]["task"]: - print( - "Warning : the task used to test is different than " - "the one from training, be sure this is what you want." - ) - main(arg_dic) diff --git a/egs/chime4/ConvTasNet/local/create_metadata.py b/egs/chime4/ConvTasNet/local/create_metadata.py index 76f8e7e1c..f0f69ae5d 100644 --- a/egs/chime4/ConvTasNet/local/create_metadata.py +++ b/egs/chime4/ConvTasNet/local/create_metadata.py @@ -1,5 +1,4 @@ import os -import shutil import argparse from glob import glob import pandas as pd @@ -29,8 +28,6 @@ def create_local_metadata(chime3_dir): # Get CHiME-4 annotation files c4_annot_files = [f for f in glob( os.path.join(chime3_dir, "data", "annotations", "*real*.list"))] - print(c3_annot_files) - print(c4_annot_files) for c3_annot_file_path in c3_annot_files: # Read CHiME-3 annotation file c3_annot_file = pd.read_json(c3_annot_file_path) @@ -47,14 +44,14 @@ def create_local_metadata(chime3_dir): header=None, names=['path']) else: c4_annot_file = None - df = create_dataframe(chime3_dir, c3_annot_file, c4_annot_file, subset, origin) - write_dataframe(df, subset, origin) + df, df_2 = create_dataframe(chime3_dir, c3_annot_file, c4_annot_file, subset, origin) + write_dataframe(df, df_2, subset, origin) def create_dataframe(chime3_dir, c3_annot_file, c4_annot_file, subset, origin): - # Empty list for DataFrame creation row_list = [] + row_list_2 = [] for row in c3_annot_file.itertuples(): speaker = row.speaker ID = row.wsj_name @@ -76,19 +73,21 @@ def create_dataframe(chime3_dir, c3_annot_file, c4_annot_file, subset, origin): subset + '_' + env.lower() + '_' + origin, speaker + '_' + ID + '_' + f'.CH{channel}' '.wav') - + dot = row.dot duration = row.end - row.start temp_dict = {'ID': ID, 'subset': subset, 'origin': origin, 'env': env, - 'mixture_ path': mixture_path, + 'mixture_path': mixture_path, 'duration': duration} + trans_dict = {'utt_id': ID, 'text':dot} row_list.append(temp_dict) - + row_list_2.append(trans_dict) df = pd.DataFrame(row_list) - return df + df_2 = pd.DataFrame(row_list_2) + return df, df_2 -def write_dataframe(df, subset, origin): +def write_dataframe(df, df2, subset, origin): if 'et' in subset: subdir = 'test' elif 'dt' in subset: @@ -99,6 +98,8 @@ def write_dataframe(df, subset, origin): os.makedirs(save_dir, exist_ok=True) save_path = os.path.join(save_dir, origin + '_1_ch_track.csv') df.to_csv(save_path, index=False) + save_path2 = os.path.join(save_dir, origin + '_1_ch_track_annotations.csv') + df2.to_csv(save_path2, index=False) if __name__ == "__main__": diff --git a/egs/chime4/ConvTasNet/run.sh b/egs/chime4/ConvTasNet/run.sh index 31c9e62f4..420faa22d 100644 --- a/egs/chime4/ConvTasNet/run.sh +++ b/egs/chime4/ConvTasNet/run.sh @@ -9,6 +9,8 @@ set -o pipefail # and LibriMix. This is about 500 Gb storage_dir= +# Directory containing the pretrained model +expdir= # After running the recipe a first time, you can run it from stage 3 directly to train new models. # Path to the python you'll use for the experiment. Defaults to the current python @@ -23,38 +25,28 @@ stage=0 # Controls from which stage to start tag="" # Controls the directory name associated to the experiment # You can ask for several GPUs using id (passed to CUDA_VISIBLE_DEVICES) -eval_use_gpu=1 +eval_use_gpu=0 # Need to --compute_wer 1 --eval_mode max to be sure the user knows all the metrics # are for the all mode. -compute_wer=0 -eval_mode= +compute_wer=1 + +test_dir=data/test . utils/parse_options.sh if [[ $stage -le 0 ]]; then echo "Stage 0: Generating CHiME-4 dataset" - . local/create_metadata.py --chime3_dir $storage_dir/CHiME3 -fi - -# Generate a random ID for the run if no tag is specified -uuid=$($python_path -c 'import uuid, sys; print(str(uuid.uuid4())[:8])') -if [[ -z ${tag} ]]; then - tag=${uuid} + $python_path local/create_metadata.py --chime3_dir $storage_dir/CHiME3/ fi -expdir=exp/train_convtasnet_${tag} mkdir -p $expdir && echo $uuid >> $expdir/run_uuid.txt echo "Results from the following experiment will be stored in $expdir" -if [[ $stage -le 2 ]]; then +if [[ $stage -le 1 ]]; then echo "Stage 2 : Evaluation" if [[ $compute_wer -eq 1 ]]; then - if [[ $eval_mode != "max" ]]; then - echo "Cannot compute WER without max mode. Start again with --stage 2 --compute_wer 1 --eval_mode max" - exit 1 - fi # Install espnet if not instaled if ! python -c "import espnet" &> /dev/null; then @@ -69,7 +61,5 @@ if [[ $stage -le 2 ]]; then --exp_dir $expdir \ --test_dir $test_dir \ --use_gpu $eval_use_gpu \ - --compute_wer $compute_wer | tee logs/eval_${tag}.log - - cp logs/eval_${tag}.log $expdir/eval.log + --compute_wer $compute_wer fi From 72039d27b1ec435e6a91e8aade0065790f0b407b Mon Sep 17 00:00:00 2001 From: JorisCos Date: Thu, 21 Jan 2021 16:37:36 +0100 Subject: [PATCH 3/9] add all transcriptions change model --- egs/chime4/ConvTasNet/eval.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/egs/chime4/ConvTasNet/eval.py b/egs/chime4/ConvTasNet/eval.py index bf7e0701f..e0fa59cd1 100644 --- a/egs/chime4/ConvTasNet/eval.py +++ b/egs/chime4/ConvTasNet/eval.py @@ -33,7 +33,7 @@ ) COMPUTE_METRICS = [] -ASR_MODEL_PATH = ("kamo-naoyuki/wsj" +ASR_MODEL_PATH = ("kamo-naoyuki/chime4_asr_train_asr_transformer3_raw_en_char_sp_valid.acc.ave" ) @@ -144,6 +144,9 @@ def main(conf): # Save the report with open(os.path.join(eval_save_dir, "final_wer.md"), "w") as f: f.write(wer_card) + all_transcriptions = wer_tracker.all_transcriptions() + with open(os.path.join(eval_save_dir, "all_transcriptions.json"), "w") as f: + json.dump(all_transcriptions, f, indent=4) with open(os.path.join(eval_save_dir, "final_metrics.json"), "w") as f: json.dump(final_results, f, indent=0) From 6f2149dff8a5f5d99e7e749993cef3604e0c88ff Mon Sep 17 00:00:00 2001 From: JorisCos Date: Thu, 21 Jan 2021 16:42:03 +0100 Subject: [PATCH 4/9] refactor fake_source --- asteroid/data/chime4_dataset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/asteroid/data/chime4_dataset.py b/asteroid/data/chime4_dataset.py index d3ba28d98..15e0c655e 100644 --- a/asteroid/data/chime4_dataset.py +++ b/asteroid/data/chime4_dataset.py @@ -66,9 +66,9 @@ def __getitem__(self, idx): start=start, stop=stop) # Convert to torch tensor mixture = torch.from_numpy(mixture) - fake_source = torch.vstack([mixture]) + mock_source = torch.vstack([mixture]) if self.return_id: id1 = row.ID - return mixture, fake_source, [id1] - return mixture, fake_source + return mixture, mock_source, [id1] + return mixture, mock_source From 4723e2a7127f1f690cdb3adeb1850f511a7466a2 Mon Sep 17 00:00:00 2001 From: JorisCos Date: Fri, 22 Jan 2021 11:31:20 +0100 Subject: [PATCH 5/9] add all possible models remove clipping before ASR --- egs/chime4/ConvTasNet/eval.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/egs/chime4/ConvTasNet/eval.py b/egs/chime4/ConvTasNet/eval.py index e0fa59cd1..2279e5518 100644 --- a/egs/chime4/ConvTasNet/eval.py +++ b/egs/chime4/ConvTasNet/eval.py @@ -33,8 +33,10 @@ ) COMPUTE_METRICS = [] -ASR_MODEL_PATH = ("kamo-naoyuki/chime4_asr_train_asr_transformer3_raw_en_char_sp_valid.acc.ave" -) +ASR_MODEL_PATH = ("kamo-naoyuki/chime4_asr_train_asr_transformer3_raw_en_char_sp_valid.acc.ave") +# ASR_MODEL_PATH = ("kamo-naoyuki/wsj") +# ASR_MODEL_PATH = ("kamo-naoyuki/wsj_transformer2") +# ASR_MODEL_PATH = ("kamo-naoyuki/dirha_wsj_asr_train_asr_transformer_cmvn_raw_char_rir_scpdatadirha_irwav.scp_noise_db_range10_17_noise_scpdatadirha_noisewav.scp_speech_volume_normalize1.0_num_workers2_rir_apply_prob1._sp_valid.acc.ave") def update_compute_metrics(compute_wer, metric_list): @@ -88,6 +90,7 @@ def main(conf): mix_np = mix.cpu().data.numpy() sources_np = sources.cpu().data.numpy() est_sources_np = est_sources.squeeze(0).cpu().data.numpy() + est_sources_np *= np.max(np.abs(mix_np)) / np.max(np.abs(est_sources_np)) # For each utterance, we get a dictionary with the mixture path, # the input and output metrics utt_metrics = {"mix_path": test_set.mixture_path} @@ -113,7 +116,7 @@ def main(conf): sf.write(local_save_dir + "s{}.wav".format(src_idx), src, conf["sample_rate"]) for src_idx, est_src in enumerate(est_sources_np): - est_src *= np.max(np.abs(mix_np)) / np.max(np.abs(est_src)) + # est_src *= np.max(np.abs(mix_np)) / np.max(np.abs(est_src)) sf.write( local_save_dir + "s{}_estimate.wav".format(src_idx), est_src, From dc2da6413d2a89aab144d8cb2698346580ea374d Mon Sep 17 00:00:00 2001 From: JorisCos Date: Tue, 2 Feb 2021 16:30:36 +0100 Subject: [PATCH 6/9] add asr_type --- egs/chime4/ConvTasNet/eval.py | 35 +++++--- .../ConvTasNet/local/create_metadata.py | 79 +++++++++---------- egs/chime4/ConvTasNet/run.sh | 18 ++--- 3 files changed, 70 insertions(+), 62 deletions(-) diff --git a/egs/chime4/ConvTasNet/eval.py b/egs/chime4/ConvTasNet/eval.py index 2279e5518..64cfe9fde 100644 --- a/egs/chime4/ConvTasNet/eval.py +++ b/egs/chime4/ConvTasNet/eval.py @@ -29,14 +29,17 @@ "--n_save_ex", type=int, default=10, help="Number of audio examples to save, -1 means all" ) parser.add_argument( - "--compute_wer", type=int, default=0, help="Compute WER using ESPNet's pretrained model" + "--compute_wer", type=int, default=1, help="Compute WER using ESPNet's pretrained model" ) +parser.add_argument( + "--asr_type", + default="noisy", + help="Choice for the ASR model whether trained on clean or noisy data. One of clean or noisy", +) + +# In CHiME 4 only the noisy data are available, hence no metrics. COMPUTE_METRICS = [] -ASR_MODEL_PATH = ("kamo-naoyuki/chime4_asr_train_asr_transformer3_raw_en_char_sp_valid.acc.ave") -# ASR_MODEL_PATH = ("kamo-naoyuki/wsj") -# ASR_MODEL_PATH = ("kamo-naoyuki/wsj_transformer2") -# ASR_MODEL_PATH = ("kamo-naoyuki/dirha_wsj_asr_train_asr_transformer_cmvn_raw_char_rir_scpdatadirha_irwav.scp_noise_db_range10_17_noise_scpdatadirha_noisewav.scp_speech_volume_normalize1.0_num_workers2_rir_apply_prob1._sp_valid.acc.ave") def update_compute_metrics(compute_wer, metric_list): @@ -54,11 +57,19 @@ def update_compute_metrics(compute_wer, metric_list): def main(conf): + + if conf["asr_type"] == "noisy": + asr_model_path = ( + "kamo-naoyuki/chime4_asr_train_asr_transformer3_raw_en_char_sp_valid.acc.ave" + ) + else: + asr_model_path = "kamo-naoyuki/wsj_transformer2" + compute_metrics = update_compute_metrics(conf["compute_wer"], COMPUTE_METRICS) - annot_path = [f for f in os.listdir(conf["test_dir"]) if 'annotations' in f][0] - anno_df = pd.read_csv(os.path.join(conf['test_dir'],annot_path)) + annot_path = [f for f in os.listdir(conf["test_dir"]) if "annotations" in f][0] + anno_df = pd.read_csv(os.path.join(conf["test_dir"], annot_path)) wer_tracker = ( - MockWERTracker() if not conf["compute_wer"] else WERTracker(ASR_MODEL_PATH, anno_df) + MockWERTracker() if not conf["compute_wer"] else WERTracker(asr_model_path, anno_df) ) model_path = os.path.join(conf["exp_dir"], "best_model.pth") model = ConvTasNet.from_pretrained(model_path) @@ -75,7 +86,7 @@ def main(conf): # Used to reorder sources only # Randomly choose the indexes of sentences to save. - eval_save_dir = os.path.join(conf["exp_dir"], 'chime4') + eval_save_dir = os.path.join(conf["exp_dir"], "chime4", conf["asr_type"]) ex_save_dir = os.path.join(eval_save_dir, "examples/") if conf["n_save_ex"] == -1: conf["n_save_ex"] = len(test_set) @@ -109,12 +120,10 @@ def main(conf): if idx in save_idx: local_save_dir = os.path.join(ex_save_dir, "ex_{}/".format(idx)) os.makedirs(local_save_dir, exist_ok=True) - sf.write(local_save_dir + "mixture.wav", mix_np, - conf["sample_rate"]) + sf.write(local_save_dir + "mixture.wav", mix_np, conf["sample_rate"]) # Loop over the sources and estimates for src_idx, src in enumerate(sources_np): - sf.write(local_save_dir + "s{}.wav".format(src_idx), src, - conf["sample_rate"]) + sf.write(local_save_dir + "s{}.wav".format(src_idx), src, conf["sample_rate"]) for src_idx, est_src in enumerate(est_sources_np): # est_src *= np.max(np.abs(mix_np)) / np.max(np.abs(est_src)) sf.write( diff --git a/egs/chime4/ConvTasNet/local/create_metadata.py b/egs/chime4/ConvTasNet/local/create_metadata.py index f0f69ae5d..7a474ea48 100644 --- a/egs/chime4/ConvTasNet/local/create_metadata.py +++ b/egs/chime4/ConvTasNet/local/create_metadata.py @@ -6,10 +6,7 @@ # Command line arguments parser = argparse.ArgumentParser() -parser.add_argument( - "--chime3_dir", type=str, default=None, - help="Path to CHiME3 root directory" -) +parser.add_argument("--chime3_dir", type=str, default=None, help="Path to CHiME3 root directory") # Set seed for random generation SEED = 4 @@ -23,25 +20,24 @@ def main(args): def create_local_metadata(chime3_dir): # Get CHiME-3 annotation files - c3_annot_files = [f for f in glob( - os.path.join(chime3_dir, "data", "annotations", "*real*.json"))] + c3_annot_files = [ + f for f in glob(os.path.join(chime3_dir, "data", "annotations", "*real*.json")) + ] # Get CHiME-4 annotation files - c4_annot_files = [f for f in glob( - os.path.join(chime3_dir, "data", "annotations", "*real*.list"))] + c4_annot_files = [ + f for f in glob(os.path.join(chime3_dir, "data", "annotations", "*real*.list")) + ] for c3_annot_file_path in c3_annot_files: # Read CHiME-3 annotation file c3_annot_file = pd.read_json(c3_annot_file_path) # subsets : "tr" "dt" "et" origin "real" or "simu" - subset, origin = os.path.split(c3_annot_file_path)[1].replace('.json', - '').split( - '_') + subset, origin = os.path.split(c3_annot_file_path)[1].replace(".json", "").split("_") # Look for associated CHiME-4 file - if c3_annot_file_path.replace('.json', - '_1ch_track.list') in c4_annot_files: + if c3_annot_file_path.replace(".json", "_1ch_track.list") in c4_annot_files: # Read CHiME-4 annotation file c4_annot_file = pd.read_csv( - c3_annot_file_path.replace('.json', '_1ch_track.list'), - header=None, names=['path']) + c3_annot_file_path.replace(".json", "_1ch_track.list"), header=None, names=["path"] + ) else: c4_annot_file = None df, df_2 = create_dataframe(chime3_dir, c3_annot_file, c4_annot_file, subset, origin) @@ -57,29 +53,32 @@ def create_dataframe(chime3_dir, c3_annot_file, c4_annot_file, subset, origin): ID = row.wsj_name env = row.environment # if we are not dealing with tr subset - if 'tr' not in subset: - mixture_path = c4_annot_file[ - c4_annot_file['path'].str.contains(ID + '_' + env)].values[ - 0][0] - mixture_path = os.path.join(chime3_dir, - "data/audio/16kHz/isolated/", - mixture_path) + if "tr" not in subset: + mixture_path = c4_annot_file[c4_annot_file["path"].str.contains(ID + "_" + env)].values[ + 0 + ][0] + mixture_path = os.path.join(chime3_dir, "data/audio/16kHz/isolated/", mixture_path) # if we are dealing with the tr subset else: channel = np.random.randint(1, 7) - mixture_path = os.path.join(chime3_dir, - "data/audio/16kHz/isolated/", - subset + '_' + env.lower() + '_' + origin, - speaker + '_' + ID + '_' + f'.CH{channel}' - '.wav') + mixture_path = os.path.join( + chime3_dir, + "data/audio/16kHz/isolated/", + subset + "_" + env.lower() + "_" + origin, + speaker + "_" + ID + "_" + f".CH{channel}" ".wav", + ) dot = row.dot duration = row.end - row.start - temp_dict = {'ID': ID, 'subset': subset, 'origin': origin, - 'env': env, - 'mixture_path': mixture_path, - 'duration': duration} - trans_dict = {'utt_id': ID, 'text':dot} + temp_dict = { + "ID": ID, + "subset": subset, + "origin": origin, + "env": env, + "mixture_path": mixture_path, + "duration": duration, + } + trans_dict = {"utt_id": ID, "text": dot} row_list.append(temp_dict) row_list_2.append(trans_dict) df = pd.DataFrame(row_list) @@ -88,17 +87,17 @@ def create_dataframe(chime3_dir, c3_annot_file, c4_annot_file, subset, origin): def write_dataframe(df, df2, subset, origin): - if 'et' in subset: - subdir = 'test' - elif 'dt' in subset: - subdir = 'val' + if "et" in subset: + subdir = "test" + elif "dt" in subset: + subdir = "val" else: - subdir = 'train' - save_dir = os.path.join('data', subdir) + subdir = "train" + save_dir = os.path.join("data", subdir) os.makedirs(save_dir, exist_ok=True) - save_path = os.path.join(save_dir, origin + '_1_ch_track.csv') + save_path = os.path.join(save_dir, origin + "_1_ch_track.csv") df.to_csv(save_path, index=False) - save_path2 = os.path.join(save_dir, origin + '_1_ch_track_annotations.csv') + save_path2 = os.path.join(save_dir, origin + "_1_ch_track_annotations.csv") df2.to_csv(save_path2, index=False) diff --git a/egs/chime4/ConvTasNet/run.sh b/egs/chime4/ConvTasNet/run.sh index 420faa22d..4b2aa4ddf 100644 --- a/egs/chime4/ConvTasNet/run.sh +++ b/egs/chime4/ConvTasNet/run.sh @@ -4,13 +4,11 @@ set -e set -o pipefail -# If you haven't generated LibriMix start from stage 0 -# Main storage directory. You'll need disk space to store LibriSpeech, WHAM noises -# and LibriMix. This is about 500 Gb +# The root directory containing CHiME3 storage_dir= # Directory containing the pretrained model -expdir= +exp_dir= # After running the recipe a first time, you can run it from stage 3 directly to train new models. # Path to the python you'll use for the experiment. Defaults to the current python @@ -30,6 +28,9 @@ eval_use_gpu=0 # are for the all mode. compute_wer=1 +# Choice for the ASR model whether trained on clean or noisy data. One of clean or noisy +asr_type=noisy + test_dir=data/test . utils/parse_options.sh @@ -40,11 +41,9 @@ if [[ $stage -le 0 ]]; then $python_path local/create_metadata.py --chime3_dir $storage_dir/CHiME3/ fi -mkdir -p $expdir && echo $uuid >> $expdir/run_uuid.txt -echo "Results from the following experiment will be stored in $expdir" - if [[ $stage -le 1 ]]; then echo "Stage 2 : Evaluation" + echo "Results from the following experiment will be stored in $exp_dir/chime4/$asr_type" if [[ $compute_wer -eq 1 ]]; then @@ -58,8 +57,9 @@ if [[ $stage -le 1 ]]; then fi $python_path eval.py \ - --exp_dir $expdir \ + --exp_dir $exp_dir \ --test_dir $test_dir \ --use_gpu $eval_use_gpu \ - --compute_wer $compute_wer + --compute_wer $compute_wer \ + --asr_type $asr_type fi From fce54f6886b3b911ca3147c7642120533cf81f2c Mon Sep 17 00:00:00 2001 From: JorisCos Date: Tue, 2 Feb 2021 16:35:36 +0100 Subject: [PATCH 7/9] black --- asteroid/data/chime4_dataset.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/asteroid/data/chime4_dataset.py b/asteroid/data/chime4_dataset.py index 15e0c655e..a12a86228 100644 --- a/asteroid/data/chime4_dataset.py +++ b/asteroid/data/chime4_dataset.py @@ -29,9 +29,9 @@ def __init__(self, csv_dir, sample_rate=16000, segment=3, return_id=False): self.segment = segment self.sample_rate = sample_rate self.return_id = return_id - self.csv_path = [f for f in os.listdir(csv_dir) if 'annotations' not in f][0] + self.csv_path = [f for f in os.listdir(csv_dir) if "annotations" not in f][0] # Open csv file and concatenate them - self.df = pd.read_csv(os.path.join(csv_dir,self.csv_path)) + self.df = pd.read_csv(os.path.join(csv_dir, self.csv_path)) # Get rid of the utterances too short if self.segment is not None: max_len = len(self.df) @@ -62,8 +62,7 @@ def __getitem__(self, idx): stop = None # Read the mixture - mixture, _ = sf.read(self.mixture_path, dtype="float32", - start=start, stop=stop) + mixture, _ = sf.read(self.mixture_path, dtype="float32", start=start, stop=stop) # Convert to torch tensor mixture = torch.from_numpy(mixture) mock_source = torch.vstack([mixture]) @@ -71,4 +70,3 @@ def __getitem__(self, idx): id1 = row.ID return mixture, mock_source, [id1] return mixture, mock_source - From f9b5c00cd21475e80f403e06099ab65f0896fb38 Mon Sep 17 00:00:00 2001 From: JorisCos Date: Tue, 9 Feb 2021 17:53:30 +0100 Subject: [PATCH 8/9] extend wertracker reformat create_metadata.py fix typo indent --- asteroid/data/__init__.py | 4 +- asteroid/data/chime4_dataset.py | 9 +-- asteroid/metrics.py | 49 ++++++++----- egs/chime4/ConvTasNet/eval.py | 17 ++--- .../ConvTasNet/local/create_metadata.py | 73 +++++++++++-------- egs/chime4/ConvTasNet/run.sh | 5 +- egs/chime4/README.md | 4 +- 7 files changed, 88 insertions(+), 73 deletions(-) diff --git a/asteroid/data/__init__.py b/asteroid/data/__init__.py index 28d3b5d10..2b9685150 100644 --- a/asteroid/data/__init__.py +++ b/asteroid/data/__init__.py @@ -9,7 +9,7 @@ from .kinect_wsj import KinectWsjMixDataset from .fuss_dataset import FUSSDataset from .dampvsep_dataset import DAMPVSEPSinglesDataset -from .chime4_dataset import CHiME4 +from .chime4_dataset import CHiME4Dataset __all__ = [ "AVSpeechDataset", @@ -23,5 +23,5 @@ "KinectWsjMixDataset", "FUSSDataset", "DAMPVSEPSinglesDataset", - "CHiME4", + "CHiME4Dataset", ] diff --git a/asteroid/data/chime4_dataset.py b/asteroid/data/chime4_dataset.py index a12a86228..028a0312c 100644 --- a/asteroid/data/chime4_dataset.py +++ b/asteroid/data/chime4_dataset.py @@ -6,7 +6,7 @@ import os -class CHiME4(Dataset): +class CHiME4Dataset(Dataset): """Dataset class for CHiME4 source separation tasks. Only supports 'real' data @@ -65,8 +65,7 @@ def __getitem__(self, idx): mixture, _ = sf.read(self.mixture_path, dtype="float32", start=start, stop=stop) # Convert to torch tensor mixture = torch.from_numpy(mixture) - mock_source = torch.vstack([mixture]) if self.return_id: - id1 = row.ID - return mixture, mock_source, [id1] - return mixture, mock_source + id1 = row.wsj_id + return mixture, [id1] + return mixture diff --git a/asteroid/metrics.py b/asteroid/metrics.py index af42b0ca7..91b3bdd86 100644 --- a/asteroid/metrics.py +++ b/asteroid/metrics.py @@ -177,13 +177,16 @@ def __call__( self.mix_counter += out_count local_mix_counter += out_count self.input_txt_list.append(dict(utt_id=tmp_id, text=txt)) - # Average WER for the clean pair - for wav, tmp_id in zip(clean, wav_id): - txt = self.predict_hypothesis(wav) - out_count = Counter(self.hsdi(truth=self.trans_dic[tmp_id], hypothesis=txt)) - self.clean_counter += out_count - local_clean_counter += out_count - self.clean_txt_list.append(dict(utt_id=tmp_id, text=txt)) + if clean is not None: + # Average WER for the clean pair + for wav, tmp_id in zip(clean, wav_id): + txt = self.predict_hypothesis(wav) + out_count = Counter(self.hsdi(truth=self.trans_dic[tmp_id], hypothesis=txt)) + self.clean_counter += out_count + local_clean_counter += out_count + self.clean_txt_list.append(dict(utt_id=tmp_id, text=txt)) + else: + self.clean_counter = None # Average WER for the estimate pair for est, tmp_id in zip(estimate, wav_id): txt = self.predict_hypothesis(est) @@ -191,11 +194,15 @@ def __call__( self.est_counter += out_count local_est_counter += out_count self.output_txt_list.append(dict(utt_id=tmp_id, text=txt)) - return dict( + + wer_dict = dict( input_wer=self.wer_from_hsdi(**dict(local_mix_counter)), - clean_wer=self.wer_from_hsdi(**dict(local_clean_counter)), wer=self.wer_from_hsdi(**dict(local_est_counter)), ) + if clean is not None: + wer_dict["clean_wer"] = self.wer_from_hsdi(**dict(local_clean_counter)) + + return wer_dict @staticmethod def wer_from_hsdi(hits=0, substitutions=0, deletions=0, insertions=0): @@ -228,31 +235,35 @@ def _df_to_dict(df): def final_df(self): """Generate a MarkDown table, as done by ESPNet.""" mix_n_word = sum(self.mix_counter[k] for k in ["hits", "substitutions", "deletions"]) - clean_n_word = sum(self.clean_counter[k] for k in ["hits", "substitutions", "deletions"]) est_n_word = sum(self.est_counter[k] for k in ["hits", "substitutions", "deletions"]) mix_wer = self.wer_from_hsdi(**dict(self.mix_counter)) - clean_wer = self.wer_from_hsdi(**dict(self.clean_counter)) est_wer = self.wer_from_hsdi(**dict(self.est_counter)) mix_hsdi = [ self.mix_counter[k] for k in ["hits", "substitutions", "deletions", "insertions"] ] - clean_hsdi = [ - self.clean_counter[k] for k in ["hits", "substitutions", "deletions", "insertions"] - ] est_hsdi = [ self.est_counter[k] for k in ["hits", "substitutions", "deletions", "insertions"] ] # Snt Wrd HSDI Err S.Err for_mix = [len(self.mix_counter), mix_n_word] + mix_hsdi + [mix_wer, "-"] - for_clean = [len(self.clean_counter), clean_n_word] + clean_hsdi + [clean_wer, "-"] for_est = [len(self.est_counter), est_n_word] + est_hsdi + [est_wer, "-"] - table = [ - ["test_clean / mixture"] + for_mix, - ["test_clean / clean"] + for_clean, - ["test_clean / separated"] + for_est, + ["ground_truth / mixture"] + for_mix, + ["ground_truth / separated"] + for_est, ] + + if self.clean_counter is not None: + clean_n_word = sum( + self.clean_counter[k] for k in ["hits", "substitutions", "deletions"] + ) + clean_wer = self.wer_from_hsdi(**dict(self.clean_counter)) + clean_hsdi = [ + self.clean_counter[k] for k in ["hits", "substitutions", "deletions", "insertions"] + ] + for_clean = [len(self.clean_counter), clean_n_word] + clean_hsdi + [clean_wer, "-"] + table.insert(1, ["ground_truth / clean"] + for_mix) + df = pd.DataFrame( table, columns=["dataset", "Snt", "Wrd", "Corr", "Sub", "Del", "Ins", "Err", "S.Err"] ) diff --git a/egs/chime4/ConvTasNet/eval.py b/egs/chime4/ConvTasNet/eval.py index 64cfe9fde..ac8f081d9 100644 --- a/egs/chime4/ConvTasNet/eval.py +++ b/egs/chime4/ConvTasNet/eval.py @@ -10,7 +10,7 @@ from tqdm import tqdm from pprint import pprint -from asteroid.data.chime4_dataset import CHiME4 +from asteroid.data.chime4_dataset import CHiME4Dataset from asteroid import ConvTasNet from asteroid.models import save_publishable from asteroid.utils import tensors_to_device @@ -26,7 +26,7 @@ ) parser.add_argument("--exp_dir", default="exp/tmp", help="Experiment root") parser.add_argument( - "--n_save_ex", type=int, default=10, help="Number of audio examples to save, -1 means all" + "--n_save_ex", type=int, default=1, help="Number of audio examples to save, -1 means all" ) parser.add_argument( "--compute_wer", type=int, default=1, help="Compute WER using ESPNet's pretrained model" @@ -77,7 +77,7 @@ def main(conf): if conf["use_gpu"]: model.cuda() model_device = next(model.parameters()).device - test_set = CHiME4( + test_set = CHiME4Dataset( csv_dir=conf["test_dir"], sample_rate=conf["sample_rate"], segment=None, @@ -95,11 +95,10 @@ def main(conf): torch.no_grad().__enter__() for idx in tqdm(range(len(test_set))): # Forward the network on the mixture. - mix, sources, ids = test_set[idx] - mix, sources = tensors_to_device([mix, sources], device=model_device) + mix, ids = test_set[idx] + mix = tensors_to_device(mix, device=model_device) est_sources = model(mix.unsqueeze(0)) mix_np = mix.cpu().data.numpy() - sources_np = sources.cpu().data.numpy() est_sources_np = est_sources.squeeze(0).cpu().data.numpy() est_sources_np *= np.max(np.abs(mix_np)) / np.max(np.abs(est_sources_np)) # For each utterance, we get a dictionary with the mixture path, @@ -108,7 +107,7 @@ def main(conf): utt_metrics.update( **wer_tracker( mix=mix_np, - clean=sources_np, + clean=None, estimate=est_sources_np, wav_id=ids, sample_rate=conf["sample_rate"], @@ -122,8 +121,6 @@ def main(conf): os.makedirs(local_save_dir, exist_ok=True) sf.write(local_save_dir + "mixture.wav", mix_np, conf["sample_rate"]) # Loop over the sources and estimates - for src_idx, src in enumerate(sources_np): - sf.write(local_save_dir + "s{}.wav".format(src_idx), src, conf["sample_rate"]) for src_idx, est_src in enumerate(est_sources_np): # est_src *= np.max(np.abs(mix_np)) / np.max(np.abs(est_src)) sf.write( @@ -156,7 +153,7 @@ def main(conf): # Save the report with open(os.path.join(eval_save_dir, "final_wer.md"), "w") as f: f.write(wer_card) - all_transcriptions = wer_tracker.all_transcriptions() + all_transcriptions = wer_tracker.trans_dic with open(os.path.join(eval_save_dir, "all_transcriptions.json"), "w") as f: json.dump(all_transcriptions, f, indent=4) diff --git a/egs/chime4/ConvTasNet/local/create_metadata.py b/egs/chime4/ConvTasNet/local/create_metadata.py index 7a474ea48..6f2499445 100644 --- a/egs/chime4/ConvTasNet/local/create_metadata.py +++ b/egs/chime4/ConvTasNet/local/create_metadata.py @@ -28,35 +28,44 @@ def create_local_metadata(chime3_dir): f for f in glob(os.path.join(chime3_dir, "data", "annotations", "*real*.list")) ] for c3_annot_file_path in c3_annot_files: - # Read CHiME-3 annotation file - c3_annot_file = pd.read_json(c3_annot_file_path) - # subsets : "tr" "dt" "et" origin "real" or "simu" - subset, origin = os.path.split(c3_annot_file_path)[1].replace(".json", "").split("_") - # Look for associated CHiME-4 file - if c3_annot_file_path.replace(".json", "_1ch_track.list") in c4_annot_files: - # Read CHiME-4 annotation file - c4_annot_file = pd.read_csv( - c3_annot_file_path.replace(".json", "_1ch_track.list"), header=None, names=["path"] - ) - else: - c4_annot_file = None - df, df_2 = create_dataframe(chime3_dir, c3_annot_file, c4_annot_file, subset, origin) - write_dataframe(df, df_2, subset, origin) + c3_annot_file, c4_annot_file, subset, origin = match_annotation_files( + c3_annot_file_path, c4_annot_files + ) + df_audio_path, df_annot = create_dataframe( + chime3_dir, c3_annot_file, c4_annot_file, subset, origin + ) + write_dataframe(df_audio_path, df_annot, subset, origin) + + +def match_annotation_files(c3_anno_path, c4_anno): + # Read CHiME-3 annotation file + c3_annot_file = pd.read_json(c3_anno_path) + # Extract subset and origin from /foo/bar/_.json + subset, origin = os.path.split(c3_anno_path)[1].replace(".json", "").split("_") + # Look for associated CHiME-4 file + if c3_anno_path.replace(".json", "_1ch_track.list") in c4_anno: + # Read CHiME-4 annotation file + c4_annot_file = pd.read_csv( + c3_anno_path.replace(".json", "_1ch_track.list"), header=None, names=["path"] + ) + else: + c4_annot_file = None + return c3_annot_file, c4_annot_file, subset, origin -def create_dataframe(chime3_dir, c3_annot_file, c4_annot_file, subset, origin): +def create_dataframe(chime3_dir, c3_anno, c4_anno, subset, origin): # Empty list for DataFrame creation - row_list = [] - row_list_2 = [] - for row in c3_annot_file.itertuples(): + row_path_list = [] + row_annot_list = [] + for row in c3_anno.itertuples(): speaker = row.speaker - ID = row.wsj_name + wsj_id = row.wsj_name env = row.environment - # if we are not dealing with tr subset - if "tr" not in subset: - mixture_path = c4_annot_file[c4_annot_file["path"].str.contains(ID + "_" + env)].values[ - 0 - ][0] + # if we are dealing with et or dt subset + if c4_anno is not None: + # Find current c3_annot_file wsj_id in c4_annot_file path list + # Path are stored like __real/_..wav + mixture_path = c4_anno[c4_anno["path"].str.contains(wsj_id + "_" + env)].values[0][0] mixture_path = os.path.join(chime3_dir, "data/audio/16kHz/isolated/", mixture_path) # if we are dealing with the tr subset @@ -66,24 +75,24 @@ def create_dataframe(chime3_dir, c3_annot_file, c4_annot_file, subset, origin): chime3_dir, "data/audio/16kHz/isolated/", subset + "_" + env.lower() + "_" + origin, - speaker + "_" + ID + "_" + f".CH{channel}" ".wav", + speaker + "_" + wsj_id + "_" + f".CH{channel}" ".wav", ) dot = row.dot duration = row.end - row.start temp_dict = { - "ID": ID, + "wsj_id": wsj_id, "subset": subset, "origin": origin, "env": env, "mixture_path": mixture_path, "duration": duration, } - trans_dict = {"utt_id": ID, "text": dot} - row_list.append(temp_dict) - row_list_2.append(trans_dict) - df = pd.DataFrame(row_list) - df_2 = pd.DataFrame(row_list_2) - return df, df_2 + trans_dict = {"utt_id": wsj_id, "text": dot} + row_path_list.append(temp_dict) + row_annot_list.append(trans_dict) + df_audio_path = pd.DataFrame(row_path_list) + df_annot = pd.DataFrame(row_annot_list) + return df_audio_path, df_annot def write_dataframe(df, df2, subset, origin): diff --git a/egs/chime4/ConvTasNet/run.sh b/egs/chime4/ConvTasNet/run.sh index 4b2aa4ddf..c5d095c0f 100644 --- a/egs/chime4/ConvTasNet/run.sh +++ b/egs/chime4/ConvTasNet/run.sh @@ -31,10 +31,9 @@ compute_wer=1 # Choice for the ASR model whether trained on clean or noisy data. One of clean or noisy asr_type=noisy -test_dir=data/test - . utils/parse_options.sh +test_dir=data/test if [[ $stage -le 0 ]]; then echo "Stage 0: Generating CHiME-4 dataset" @@ -42,7 +41,7 @@ if [[ $stage -le 0 ]]; then fi if [[ $stage -le 1 ]]; then - echo "Stage 2 : Evaluation" + echo "Stage 2 : Evaluation" echo "Results from the following experiment will be stored in $exp_dir/chime4/$asr_type" if [[ $compute_wer -eq 1 ]]; then diff --git a/egs/chime4/README.md b/egs/chime4/README.md index 1754e2c0d..7f501bff7 100644 --- a/egs/chime4/README.md +++ b/egs/chime4/README.md @@ -10,9 +10,9 @@ For now, this recipe only deals with the `real_1_ch_track` part of the dataset. As the channel to use for the training set wasn't defined by the challenge's rules, we will set it randomly. -NOTE : +**Note :** This dataset uses real noisy data. This means the clean speech from the noisy -utterances is not available. This makes it not suitable for the usual training +utterances is not available. This makes it unsuitable for the usual training procedure. From 0c58deb16e52c4b7568aa7f6e0833e156449183c Mon Sep 17 00:00:00 2001 From: JorisCos Date: Mon, 8 Mar 2021 11:34:21 +0100 Subject: [PATCH 9/9] fix metrics fix eval --- asteroid/metrics.py | 152 +++++++++++++++++++++++++++++++--- egs/chime4/ConvTasNet/eval.py | 2 +- 2 files changed, 143 insertions(+), 11 deletions(-) diff --git a/asteroid/metrics.py b/asteroid/metrics.py index 91b3bdd86..dcccb5675 100644 --- a/asteroid/metrics.py +++ b/asteroid/metrics.py @@ -1,8 +1,8 @@ +import json import warnings import traceback -from collections import Counter from typing import List - +from collections import Counter import pandas as pd import numpy as np from pb_bss_eval import InputMetrics, OutputMetrics @@ -30,7 +30,7 @@ def get_metrics( clean (np.array): reference array. estimate (np.array): estimate array. sample_rate (int): sampling rate of the audio clips. - metrics_list (Union [str, list]): List of metrics to compute. + metrics_list (Union[List[str], str): List of metrics to compute. Defaults to 'all' (['si_sdr', 'sdr', 'sir', 'sar', 'stoi', 'pesq']). average (bool): Return dict([float]) if True, else dict([array]). compute_permutation (bool): Whether to compute the permutation on @@ -115,6 +115,91 @@ def get_metrics( return utt_metrics +class MetricTracker: + """Metric tracker, subject to change. + + Args: + sample_rate (int): sampling rate of the audio clips. + metrics_list (Union[List[str], str): List of metrics to compute. + Defaults to 'all' (['si_sdr', 'sdr', 'sir', 'sar', 'stoi', 'pesq']). + average (bool): Return dict([float]) if True, else dict([array]). + compute_permutation (bool): Whether to compute the permutation on + estimate sources for the output metrics (default False) + ignore_metrics_errors (bool): Whether to ignore errors that occur in + computing the metrics. A warning will be printed instead. + """ + + def __init__( + self, + sample_rate, + metrics_list=tuple(ALL_METRICS), + average=True, + compute_permutation=False, + ignore_metrics_errors=False, + ): + self.sample_rate = sample_rate + # TODO: support WER in metrics_list when merged. + self.metrics_list = metrics_list + self.average = average + self.compute_permutation = compute_permutation + self.ignore_metrics_errors = ignore_metrics_errors + + self.series_list = [] + self._len_last_saved = 0 + self._all_metrics = pd.DataFrame() + + def __call__( + self, *, mix: np.ndarray, clean: np.ndarray, estimate: np.ndarray, filename=None, **kwargs + ): + """Compute metrics for mix/clean/estimate and log it to the class. + + Args: + mix (np.array): mixture array. + clean (np.array): reference array. + estimate (np.array): estimate array. + sample_rate (int): sampling rate of the audio clips. + filename (str, optional): If computing a metric fails, print this + filename along with the exception/warning message for debugging purposes. + **kwargs: Any key, value pair to log in the utterance metric (filename, speaker ID, etc...) + """ + utt_metrics = get_metrics( + mix, + clean, + estimate, + sample_rate=self.sample_rate, + metrics_list=self.metrics_list, + average=self.average, + compute_permutation=self.compute_permutation, + ignore_metrics_errors=self.ignore_metrics_errors, + filename=filename, + ) + utt_metrics.update(kwargs) + self.series_list.append(pd.Series(utt_metrics)) + + def as_df(self): + """Return dataframe containing the results (cached).""" + if self._len_last_saved == len(self.series_list): + return self._all_metrics + self._len_last_saved = len(self.series_list) + self._all_metrics = pd.DataFrame(self.series_list) + return pd.DataFrame(self.series_list) + + def final_report(self, dump_path: str = None): + """Return dict of average metrics. Dump to JSON if `dump_path` is not None.""" + final_results = {} + metrics_df = self.as_df() + for metric_name in self.metrics_list: + input_metric_name = "input_" + metric_name + ldf = metrics_df[metric_name] - metrics_df[input_metric_name] + final_results[metric_name] = metrics_df[metric_name].mean() + final_results[metric_name + "_imp"] = ldf.mean() + if dump_path is not None: + dump_path = dump_path + ".json" if not dump_path.endswith(".json") else dump_path + with open(dump_path, "w") as f: + json.dump(final_results, f, indent=0) + return final_results + + class MockWERTracker: def __init__(self, *args, **kwargs): pass @@ -139,6 +224,7 @@ def __init__(self, model_name, trans_df): from espnet2.bin.asr_inference import Speech2Text from espnet_model_zoo.downloader import ModelDownloader + import jiwer self.model_name = model_name d = ModelDownloader() @@ -146,12 +232,24 @@ def __init__(self, model_name, trans_df): self.input_txt_list = [] self.clean_txt_list = [] self.output_txt_list = [] + self.transcriptions = [] + self.true_txt_list = [] self.sample_rate = int(d.data_frame[d.data_frame["name"] == model_name]["fs"]) self.trans_df = trans_df self.trans_dic = self._df_to_dict(trans_df) self.mix_counter = Counter() self.clean_counter = Counter() self.est_counter = Counter() + self.transformation = jiwer.Compose( + [ + jiwer.ToLowerCase(), + jiwer.RemovePunctuation(), + jiwer.RemoveMultipleSpaces(), + jiwer.Strip(), + jiwer.SentencesToListOfWords(), + jiwer.RemoveEmptyStrings(), + ] + ) def __call__( self, @@ -172,29 +270,58 @@ def __call__( local_est_counter = Counter() # Count the mixture output for each speaker txt = self.predict_hypothesis(mix) + + # Dict to gather transcriptions and IDs + trans_dict = dict(mixture_txt={}, clean={}, estimates={}, truth={}) + # Get mixture transcription + trans_dict["mixture_txt"] = txt + # Get ground truth transcription and IDs + for i, tmp_id in enumerate(wav_id): + trans_dict["truth"][f"utt_id_{i}"] = tmp_id + trans_dict["truth"][f"txt_{i}"] = self.trans_dic[tmp_id] + self.true_txt_list.append(dict(utt_id=tmp_id, text=self.trans_dic[tmp_id])) + # Mixture for tmp_id in wav_id: - out_count = Counter(self.hsdi(truth=self.trans_dic[tmp_id], hypothesis=txt)) + out_count = Counter( + self.hsdi( + truth=self.trans_dic[tmp_id], hypothesis=txt, transformation=self.transformation + ) + ) self.mix_counter += out_count local_mix_counter += out_count self.input_txt_list.append(dict(utt_id=tmp_id, text=txt)) if clean is not None: # Average WER for the clean pair - for wav, tmp_id in zip(clean, wav_id): + for i, (wav, tmp_id) in enumerate(zip(clean, wav_id)): txt = self.predict_hypothesis(wav) - out_count = Counter(self.hsdi(truth=self.trans_dic[tmp_id], hypothesis=txt)) + out_count = Counter( + self.hsdi( + truth=self.trans_dic[tmp_id], hypothesis=txt, + transformation=self.transformation + ) + ) self.clean_counter += out_count local_clean_counter += out_count self.clean_txt_list.append(dict(utt_id=tmp_id, text=txt)) + trans_dict["clean"][f"utt_id_{i}"] = tmp_id + trans_dict["clean"][f"txt_{i}"] = txt else: self.clean_counter = None # Average WER for the estimate pair - for est, tmp_id in zip(estimate, wav_id): + for i, (est, tmp_id) in enumerate(zip(estimate, wav_id)): txt = self.predict_hypothesis(est) - out_count = Counter(self.hsdi(truth=self.trans_dic[tmp_id], hypothesis=txt)) + out_count = Counter( + self.hsdi( + truth=self.trans_dic[tmp_id], hypothesis=txt, transformation=self.transformation + ) + ) self.est_counter += out_count local_est_counter += out_count self.output_txt_list.append(dict(utt_id=tmp_id, text=txt)) + trans_dict["estimates"][f"utt_id_{i}"] = tmp_id + trans_dict["estimates"][f"txt_{i}"] = txt + self.transcriptions.append(trans_dict) wer_dict = dict( input_wer=self.wer_from_hsdi(**dict(local_mix_counter)), wer=self.wer_from_hsdi(**dict(local_est_counter)), @@ -210,11 +337,16 @@ def wer_from_hsdi(hits=0, substitutions=0, deletions=0, insertions=0): return wer @staticmethod - def hsdi(truth, hypothesis): + def hsdi(truth, hypothesis, transformation): from jiwer import compute_measures keep = ["hits", "substitutions", "deletions", "insertions"] - out = compute_measures(truth=truth, hypothesis=hypothesis).items() + out = compute_measures( + truth=truth, + hypothesis=hypothesis, + truth_transform=transformation, + hypothesis_transform=transformation, + ).items() return {k: v for k, v in out if k in keep} def predict_hypothesis(self, wav): diff --git a/egs/chime4/ConvTasNet/eval.py b/egs/chime4/ConvTasNet/eval.py index ac8f081d9..86a966a4d 100644 --- a/egs/chime4/ConvTasNet/eval.py +++ b/egs/chime4/ConvTasNet/eval.py @@ -153,7 +153,7 @@ def main(conf): # Save the report with open(os.path.join(eval_save_dir, "final_wer.md"), "w") as f: f.write(wer_card) - all_transcriptions = wer_tracker.trans_dic + all_transcriptions = wer_tracker.transcriptions with open(os.path.join(eval_save_dir, "all_transcriptions.json"), "w") as f: json.dump(all_transcriptions, f, indent=4)