From 2eb69485a9d4ba64256688f5545389d70b75082c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=98=89=E6=B8=8A?= Date: Thu, 11 May 2023 16:36:20 +0800 Subject: [PATCH] update repo --- ...{data_prep_librispeech.sh => data_prep.sh} | 0 .../conformer/local/download_and_untar.sh | 97 ++++++++++++++++++ egs/librispeech/conformer/local/spm_encode.py | 98 +++++++++++++++++++ egs/librispeech/conformer/local/spm_train.py | 12 +++ 4 files changed, 207 insertions(+) rename egs/librispeech/conformer/local/{data_prep_librispeech.sh => data_prep.sh} (100%) create mode 100755 egs/librispeech/conformer/local/download_and_untar.sh create mode 100755 egs/librispeech/conformer/local/spm_encode.py create mode 100755 egs/librispeech/conformer/local/spm_train.py diff --git a/egs/librispeech/conformer/local/data_prep_librispeech.sh b/egs/librispeech/conformer/local/data_prep.sh similarity index 100% rename from egs/librispeech/conformer/local/data_prep_librispeech.sh rename to egs/librispeech/conformer/local/data_prep.sh diff --git a/egs/librispeech/conformer/local/download_and_untar.sh b/egs/librispeech/conformer/local/download_and_untar.sh new file mode 100755 index 000000000..fe322e4af --- /dev/null +++ b/egs/librispeech/conformer/local/download_and_untar.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash + +# Copyright 2014 Johns Hopkins University (author: Daniel Povey) +# Apache 2.0 + +remove_archive=false + +if [ "$1" == --remove-archive ]; then + remove_archive=true + shift +fi + +if [ $# -ne 3 ]; then + echo "Usage: $0 [--remove-archive] " + echo "e.g.: $0 /export/a15/vpanayotov/data www.openslr.org/resources/11 dev-clean" + echo "With --remove-archive it will remove the archive after successfully un-tarring it." + echo " can be one of: dev-clean, test-clean, dev-other, test-other," + echo " train-clean-100, train-clean-360, train-other-500." + exit 1 +fi + +data=$1 +url=$2 +part=$3 + +if [ ! -d "$data" ]; then + echo "$0: no such directory $data" + exit 1 +fi + +part_ok=false +list="dev-clean test-clean dev-other test-other train-clean-100 train-clean-360 train-other-500" +for x in $list; do + if [ "$part" == $x ]; then part_ok=true; fi +done +if ! $part_ok; then + echo "$0: expected to be one of $list, but got '$part'" + exit 1 +fi + +if [ -z "$url" ]; then + echo "$0: empty URL base." + exit 1 +fi + +if [ -f $data/LibriSpeech/$part/.complete ]; then + echo "$0: data part $part was already successfully extracted, nothing to do." + exit 0 +fi + + +# sizes of the archive files in bytes. This is some older versions. +sizes_old="371012589 347390293 379743611 361838298 6420417880 23082659865 30626749128" +# sizes_new is the archive file sizes of the final release. Some of these sizes are of +# things we probably won't download. +sizes_new="337926286 314305928 695964615 297279345 87960560420 33373768 346663984 328757843 6387309499 23049477885 30593501606" + +if [ -f $data/$part.tar.gz ]; then + size=$(/bin/ls -l $data/$part.tar.gz | awk '{print $5}') + size_ok=false + for s in $sizes_old $sizes_new; do if [ $s == $size ]; then size_ok=true; fi; done + if ! $size_ok; then + echo "$0: removing existing file $data/$part.tar.gz because its size in bytes $size" + echo "does not equal the size of one of the archives." + rm $data/$part.tar.gz + else + echo "$data/$part.tar.gz exists and appears to be complete." + fi +fi + +if [ ! -f $data/$part.tar.gz ]; then + if ! which wget >/dev/null; then + echo "$0: wget is not installed." + exit 1 + fi + full_url=$url/$part.tar.gz + echo "$0: downloading data from $full_url. This may take some time, please be patient." + + if ! wget -P $data --no-check-certificate $full_url; then + echo "$0: error executing wget $full_url" + exit 1 + fi +fi + +if ! tar -C $data -xvzf $data/$part.tar.gz; then + echo "$0: error un-tarring archive $data/$part.tar.gz" + exit 1 +fi + +touch $data/LibriSpeech/$part/.complete + +echo "$0: Successfully downloaded and un-tarred $data/$part.tar.gz" + +if $remove_archive; then + echo "$0: removing $data/$part.tar.gz file since --remove-archive option was supplied." + rm $data/$part.tar.gz +fi diff --git a/egs/librispeech/conformer/local/spm_encode.py b/egs/librispeech/conformer/local/spm_encode.py new file mode 100755 index 000000000..9e1c15f0e --- /dev/null +++ b/egs/librispeech/conformer/local/spm_encode.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in +# https://github.com/pytorch/fairseq/blob/master/LICENSE + + +import argparse +import contextlib +import sys + +import sentencepiece as spm + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True, + help="sentencepiece model to use for encoding") + parser.add_argument("--inputs", nargs="+", default=['-'], + help="input files to filter/encode") + parser.add_argument("--outputs", nargs="+", default=['-'], + help="path to save encoded outputs") + parser.add_argument("--output_format", choices=["piece", "id"], default="piece") + parser.add_argument("--min-len", type=int, metavar="N", + help="filter sentence pairs with fewer than N tokens") + parser.add_argument("--max-len", type=int, metavar="N", + help="filter sentence pairs with more than N tokens") + args = parser.parse_args() + + assert len(args.inputs) == len(args.outputs), \ + "number of input and output paths should match" + + sp = spm.SentencePieceProcessor() + sp.Load(args.model) + + if args.output_format == "piece": + def encode(l): + return sp.EncodeAsPieces(l) + elif args.output_format == "id": + def encode(l): + return list(map(str, sp.EncodeAsIds(l))) + else: + raise NotImplementedError + + if args.min_len is not None or args.max_len is not None: + def valid(line): + return ( + (args.min_len is None or len(line) >= args.min_len) and + (args.max_len is None or len(line) <= args.max_len) + ) + else: + def valid(lines): + return True + + with contextlib.ExitStack() as stack: + inputs = [ + stack.enter_context(open(input, "r", encoding="utf-8")) + if input != "-" else sys.stdin + for input in args.inputs + ] + outputs = [ + stack.enter_context(open(output, "w", encoding="utf-8")) + if output != "-" else sys.stdout + for output in args.outputs + ] + + stats = { + "num_empty": 0, + "num_filtered": 0, + } + + def encode_line(line): + line = line.strip() + if len(line) > 0: + line = encode(line) + if valid(line): + return line + else: + stats["num_filtered"] += 1 + else: + stats["num_empty"] += 1 + return None + + for i, lines in enumerate(zip(*inputs), start=1): + enc_lines = list(map(encode_line, lines)) + if not any(enc_line is None for enc_line in enc_lines): + for enc_line, output_h in zip(enc_lines, outputs): + print(" ".join(enc_line), file=output_h) + if i % 10000 == 0: + print("processed {} lines".format(i), file=sys.stderr) + + print("skipped {} empty lines".format(stats["num_empty"]), file=sys.stderr) + print("filtered {} lines".format(stats["num_filtered"]), file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/egs/librispeech/conformer/local/spm_train.py b/egs/librispeech/conformer/local/spm_train.py new file mode 100755 index 000000000..134a0b1d3 --- /dev/null +++ b/egs/librispeech/conformer/local/spm_train.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# https://github.com/pytorch/fairseq/blob/master/LICENSE +import sys + +import sentencepiece as spm + +if __name__ == "__main__": + spm.SentencePieceTrainer.Train(" ".join(sys.argv[1:]))