From 6b7134d49167d4410b18a10eebd92d163d83828c Mon Sep 17 00:00:00 2001 From: nichongjia-2007 Date: Wed, 4 Jan 2023 16:49:51 +0800 Subject: [PATCH 1/2] modify export_models.py --- .../inverse_text_normalization/export_models.py | 17 +++++++++++++++-- fun_text_processing/version.txt | 2 +- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/fun_text_processing/inverse_text_normalization/export_models.py b/fun_text_processing/inverse_text_normalization/export_models.py index 7618f4905..be6394fd7 100644 --- a/fun_text_processing/inverse_text_normalization/export_models.py +++ b/fun_text_processing/inverse_text_normalization/export_models.py @@ -21,6 +21,12 @@ def parse_args(): parser.add_argument( "--language", help="language", choices=['de', 'en', 'es', 'fr', 'id', 'ja', 'ko', 'pt', 'ru', 'vi', 'zh'], default="en", type=str ) + + parser.add_argument( + "--token_and_classify_and_verbalize", help="export the single token&classify and verbalize or combined", choices=['single', 'combine'], + default="single", type=str + ) + parser.add_argument( "--export_dir", help="path to export directory. Default to current directory.", @@ -77,10 +83,17 @@ if __name__ == "__main__": os.makedirs(export_dir, exist_ok=True) tagger_far_file = os.path.join(export_dir, args.language + "_itn_tagger.far") verbalizer_far_file = os.path.join(export_dir, args.language + "_itn_verbalizer.far") + tager_and_verbalizer_far_file = os.path.join(export_dir, args.language, "model.far" ) start_time = perf_counter() tagger_fst, verbalizer_fst = get_grammars(args.language) - generator_main(tagger_far_file, {"tokenize_and_classify": tagger_fst}) - generator_main(verbalizer_far_file, {"verbalize": verbalizer_fst}) + + if args.token_and_classify_and_verbalize == 'single': + generator_main(tagger_far_file, {"tokenize_and_classify": tagger_fst}) + generator_main(verbalizer_far_file, {"verbalize": verbalizer_fst}) + elif args.token_and_classify_and_verbalize == 'combine': + if not os.path.exists(os.path.join(export_dir, args.language)): + os.makedirs(os.path.join(export_dir, args.language)) + generator_main(tager_and_verbalizer_far_file, {"tokenize_and_classify": tagger_fst, "verbalize": verbalizer_fst}) print(f'Time to generate graph: {round(perf_counter() - start_time, 2)} sec') diff --git a/fun_text_processing/version.txt b/fun_text_processing/version.txt index 6e8bf73aa..17e51c385 100644 --- a/fun_text_processing/version.txt +++ b/fun_text_processing/version.txt @@ -1 +1 @@ -0.1.0 +0.1.1 From 7836bee517a678192890a02449ef9aa7fe89f8fb Mon Sep 17 00:00:00 2001 From: Chong Zhang Date: Wed, 4 Jan 2023 16:58:07 +0800 Subject: [PATCH 2/2] Update export_models.py set the default value of "token_and_classify_and_verbalize" to "combine". --- fun_text_processing/inverse_text_normalization/export_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fun_text_processing/inverse_text_normalization/export_models.py b/fun_text_processing/inverse_text_normalization/export_models.py index be6394fd7..49149c257 100644 --- a/fun_text_processing/inverse_text_normalization/export_models.py +++ b/fun_text_processing/inverse_text_normalization/export_models.py @@ -24,7 +24,7 @@ def parse_args(): parser.add_argument( "--token_and_classify_and_verbalize", help="export the single token&classify and verbalize or combined", choices=['single', 'combine'], - default="single", type=str + default="combine", type=str ) parser.add_argument(